From d02e4bed6d2e591a22aa6b8e23932cee74acf6ac Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Mon, 10 May 2021 11:12:33 -0500
Subject: [PATCH 001/452] Add datatypes for Structural Materials Hexrd
 application

Hexrd ( https://github.com/HEXRD/hexrd ) uses H5 and compressed numpy files .npz for data formats.
---
 .../config/sample/datatypes_conf.xml.sample   |   9 +
 lib/galaxy/datatypes/binary.py                | 210 ++++++++++++++++++
 lib/galaxy/datatypes/test/hexrd.eta_ome.npz   | Bin 0 -> 2121 bytes
 lib/galaxy/datatypes/test/hexrd.images.npz    | Bin 0 -> 4322 bytes
 lib/galaxy/datatypes/test/hexrd.materials.h5  | Bin 0 -> 10712 bytes
 5 files changed, 219 insertions(+)
 create mode 100644 lib/galaxy/datatypes/test/hexrd.eta_ome.npz
 create mode 100644 lib/galaxy/datatypes/test/hexrd.images.npz
 create mode 100644 lib/galaxy/datatypes/test/hexrd.materials.h5

diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index 48060f2f97d..726f811dd08 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -861,6 +861,11 @@
     <datatype extension="ffindex" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" subclass="true" description_url="https://github.com/soedinglab/ffindex_soedinglab"/>
     <datatype extension="ffdata" type="galaxy.datatypes.data:Data" display_in_upload="true" subclass="true" description_url="https://github.com/soedinglab/ffindex_soedinglab"/>
     <datatype extension="pretext" type="galaxy.datatypes.binary:Pretext" display_in_upload="true" />
+    <!-- Structural Materials datatypes -->
+    <datatype extension="hexrd.materials.h5" type="galaxy.datatypes.binary:HexrdMaterials" display_in_upload="true"/>
+    <datatype extension="hexrd.images.npz" type="galaxy.datatypes.binary:HexrdImagesNpz" display_in_upload="true"/>
+    <datatype extension="hexrd.eta_ome.npz" type="galaxy.datatypes.binary:HexrdEtaOmeNpz" display_in_upload="true"/>
+    <!-- End Structural Materials datatypes -->
   </registration>
   <sniffers>
     <!--
@@ -913,7 +918,11 @@
     <sniffer type="galaxy.datatypes.binary:Loom"/>
     <sniffer type="galaxy.datatypes.binary:Anndata"/>
     <sniffer type="galaxy.datatypes.binary:Biom2"/>
+    <sniffer type="galaxy.datatypes.binary:HexrdMaterials"/>
     <sniffer type="galaxy.datatypes.binary:H5"/>
+    <sniffer type="galaxy.datatypes.binary:HexrdImagesNpz"/>
+    <sniffer type="galaxy.datatypes.binary:HexrdEtaOmeNpz"/>
+    <sniffer type="galaxy.datatypes.binary:Npz"/>
     <sniffer type="galaxy.datatypes.binary:Bam"/>
     <sniffer type="galaxy.datatypes.binary:BamQuerynameSorted"/>
     <sniffer type="galaxy.datatypes.binary:BamNative"/>
diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index e6578e0c0e3..342764ae98c 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -5,6 +5,7 @@ import gzip
 import io
 import json
 import logging
+import numpy as np
 import os
 import shutil
 import struct
@@ -1529,6 +1530,49 @@ class H5MLM(H5):
             return "HDF5 Model (%s)" % (nice_size(dataset.get_size()))
 
 
+class HexrdMaterials(H5):
+    """
+    Class describing a Hexrd Materials file: https://github.com/HEXRD/hexrd
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('hexrd.materials.h5')
+    >>> HexrdMaterials().sniff(fname)
+    True
+    >>> fname = get_test_fname('test.loom')
+    >>> HexrdMaterials().sniff(fname)
+    False
+    """
+    file_ext = "hexrd.materials.h5"
+    edam_format = "format_3590"
+
+    MetadataElement(name="materials", desc="materials", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None)
+
+    def sniff(self, filename):
+        if super().sniff(filename):
+            req = {'AtomData', 'Atomtypes', 'CrystalSystem', 'LatticeParameters'}
+            with h5py.File(filename, 'r') as mat_file:
+                for k in mat_file.keys():
+                    if set(mat_file[k].keys()) & req:
+                        return True
+        return False
+
+    def set_meta(self, dataset, overwrite=True, **kwd):
+        super().set_meta(dataset, overwrite=overwrite, **kwd)
+        try:
+            with h5py.File(dataset.file_name, 'r') as mat_file:
+                dataset.metadata.materials = list(mat_file.keys())
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Materials: %s" % (' '.join(dataset.metadata.materials))
+            dataset.blurb = dataset.peek
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+
 class Scf(Binary):
     """Class describing an scf binary sequence file"""
     edam_format = "format_1632"
@@ -3081,6 +3125,172 @@ class JP2(Binary):
             return "Binary JPEG 2000 file (%s)" % (nice_size(dataset.get_size()))
 
 
+class Npz(CompressedArchive):
+    """
+    Class describing an Numpy NPZ file
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('test.mz5')
+    >>> Npz().sniff(fname)
+    True
+    >>> fname = get_test_fname('interval.interval')
+    >>> Npz().sniff(fname)
+    False
+    """
+    file_ext = "npz"
+    # edam_format = "format_4003"
+
+    MetadataElement(name="nfiles", default=0, desc="nfiles", readonly=True, visible=True, no_value=0)
+
+    def __init__(self, **kwd):
+        super().__init__(**kwd)
+
+    def sniff(self, filename):
+        try:
+            npz = np.load(filename)
+            return isinstance(npz, np.lib.npyio.NpzFile)
+        except Exception as e:
+            return False
+        return False
+
+    def set_meta(self, dataset, **kwd):
+        try:
+            with np.load(dataset.file_name) as npz:
+                dataset.metadata.nfiles = len(npz.files)
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Binary Numpy npz file"
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except Exception:
+            return "Binary Numpy npz file (%s)" % (nice_size(dataset.get_size()))
+
+
+class HexrdImagesNpz(Npz):
+    """
+    Class describing an HEXRD Images Numpy NPZ file
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('hexrd.images.npz')
+    >>> Npz().sniff(fname)
+    True
+    >>> fname = get_test_fname('hexrd.estr(npz['panel_id'])ta_ome.npz')
+    >>> Npz().sniff(fname)
+    False
+    """
+    file_ext = "hexrd.images.npz"
+
+    MetadataElement(name="panel_id", default='', desc="Detector Panel ID", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value='')
+    MetadataElement(name="shape", default=(), desc="shape", param=metadata.ListParameter, readonly=True, visible=True, no_value=())
+    MetadataElement(name="nframes", default=0, desc="nframes", readonly=True, visible=True, no_value=0)
+    MetadataElement(name="omegas", desc="has omegas", default="False", visible=False)
+
+    def __init__(self, **kwd):
+        super().__init__(**kwd)
+
+    def sniff(self, filename):
+        if super().sniff(filename):
+            try:
+                req_files = {'0_row', '0_col', '0_data', 'shape', 'nframes',  'dtype'}
+                with np.load(filename) as npz:
+                    return set(npz.files) >= req_files
+            except Exception as e:
+                log.warning('%s, sniff Exception: %s', self, e)
+                return False
+        return False
+
+    def set_meta(self, dataset, **kwd):
+        super().set_meta(dataset, **kwd)
+        try:
+            with np.load(dataset.file_name) as npz:
+                if 'panel_id' in npz.files:
+                    dataset.metadata.panel_id = str(npz['panel_id'])
+                if 'omega' in npz.files:
+                    dataset.metadata.omegas = "True"
+                dataset.metadata.shape = npz['shape'].tolist()
+                dataset.metadata.nframes = npz['nframes'].tolist()
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Panel: %s Frames: %s Shape: %s" % (dataset.metadata.panel_id, dataset.metadata.nframes, dataset.metadata.shape)
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except Exception:
+            return "Binary Numpy npz file (%s)" % (nice_size(dataset.get_size()))
+
+
+class HexrdEtaOmeNpz(Npz):
+    """
+    Class describing an HEXRD Eta-Ome Numpy NPZ file
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname('hexrd.eta_ome.npz')
+    >>> Npz().sniff(fname)
+    True
+    >>> fname = get_test_fname('hexrd.images.npz')
+    >>> Npz().sniff(fname)
+    False
+    """
+    file_ext = "hexrd.eta_ome.npz"
+
+    MetadataElement(name="HKLs", default=(), desc="HKLs", param=metadata.ListParameter, readonly=True, visible=True, no_value=())
+    MetadataElement(name="nframes", default=0, desc="nframes", readonly=True, visible=True, no_value=0)
+
+    def __init__(self, **kwd):
+        super().__init__(**kwd)
+
+    def sniff(self, filename):
+        if super().sniff(filename):
+            try:
+                req_files = {'dataStore', 'etas', 'etaEdges', 'iHKLList', 'omegas', 'omeEdges', 'planeData_hkls'}
+                with np.load(filename) as npz:
+                    return set(npz.files) >= req_files
+            except Exception as e:
+                log.warning('%s, sniff Exception: %s', self, e)
+                return False
+        return False
+
+    def set_meta(self, dataset, **kwd):
+        super().set_meta(dataset, **kwd)
+        try:
+            with np.load(dataset.file_name) as npz:
+                dataset.metadata.HKLs = npz['iHKLList'].tolist()
+                dataset.metadata.nframes = len(npz['omegas'])
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Eta-Ome HKLs: %s Frames: %s" % (dataset.metadata.HKLsi, dataset.metadata.nframes)
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except Exception:
+            return "Binary Numpy npz file (%s)" % (nice_size(dataset.get_size()))
+
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod(sys.modules[__name__])
diff --git a/lib/galaxy/datatypes/test/hexrd.eta_ome.npz b/lib/galaxy/datatypes/test/hexrd.eta_ome.npz
new file mode 100644
index 0000000000000000000000000000000000000000..8f9cb23df075ef44c9277b6f0da07c784e9ff8c7
GIT binary patch
literal 2121
zcmWIWW@Zs#U|`??Vnv4fQ^!0#fh++=1_oXR5r&k+lEmPW{GwF7yn;$b1_7`HP@Vy#
z2n1%o5q~8uaPmaJv4FMnCPmGOS(3MOLEhxFh3ghCm^UpxBxuSE@%iynCUFUsd!Nzt
zW9<@8R}ovn^iwL$itE@cE3U0vDa?9pj%*A8DDLQFF#Q+{G=2sUa{%3uT9Q}{a{@>V
z1VHj2K(-T(S%DoO!m#d;<mTD4QZ4y4o>t$TuXCvKLDy^l*Q$5su%kK4a)Y;d8qmrG
zK+FSimTO9SD%@!xaS#Bh0Rgg|rUrAG9%t<I3HQ6^u=^R+-#egtvU0(%Yff>JjOUoG
zq?tFB+_2<lKyzckIy0*fpxqmQ805xG4{slz%;FMQ@PWiZ0Hg*4aJVs(h!}(hpB|e=
zLdzwefLW4Wn;kO4G<eZmH1+4aBe2Ni0=g(aH#MD>(W!TOr(^e>cj~v|t7HHEe7aNX
zciHskaY7Gszo9uyv#~Qh8W;o%fEW~1K!;(CPLLW9z!6l06FVY0*)%rHk=}CRZQVlK
zmb@+dzwP?<`E+qCy2G@Wq&#~9w9tT+fk6o9u!5Y#yi^xpUX0Jk&Vi>kkSqv*RD!_V
z$s4&21@O2=uT66OB7X0ih!uxYhsAD9JwCsn99PFRd%C)Ri2mCkdgj5z08{T5dr$7o
zymasR>6=MUtHs}EO*Xl2VQp5nN&e=E8@bQU>DjF_p1#DtMoRwEmII$-YrMLjzf&+e
zT&Hhj{yE=dZ}<<dbrtU&@38;RwJkgu>KcA|aXy3Wt046Q#+BbLw!atNGrj&r!8)me
zT)y1%x9mDEPhWV~`LODW=xzSK^J^K6-hX&)n|l6_+^Kh-^Zq?~?N#r5ab@Flc_W>V
z$+g}y7dKfwTx!p9FRVs2d*7s``}^($K3gDPdCe9r0vBdwC}skSs#8FW8cvBt>F{s@
z$$|h#B?w@R#<<7pF^a4*)FKNSi&GfZu|0dT=hB@fuf0$HEDG6s=hB=>XO1jdbi`}j
zk40CW{Mqwp(xOXGgw{E+b!e^IBhrzhxY*n|TXC{+a*$$iwW4=`;_5c@#pd4Giqav8
zUY@CrTH%V-EW*sp&HG&y-P7Hx71IM0r?;6GvnU6wSvKMJ`Sa&r+Y2;s=Kr4LG;hhd
z8%}Y(H=OKv%h?n2>b~Cnm48Rp<o`UIqHHGbEEeN{=&nL9#rh(*EZ4xUxX2{G%(yg9
zL-7fMa$G5eqFfnfVqDk6xH4TfFo-@^eYQ-D%e{bc=F`s9m6j_jU#d#BDkYdtZ4>Eo
z+rTJkx32D;qFif|;2H<M6DO`X6_)?@_Mg5ke)Tq{9G(-CoH{x96pRh^WhGR8?rfd3
z)brgp%~1Zk*E>U&PZRM{eaQ%nEJh{~W?Z!kBr-r`12|$(>Kb&ds1*XT)(#*CRz0BW
zLM^Y6b;$#*1nPokcXXYor6{t_!$6(LWh%N()Y1-FXDAZ`QrU;D54A8t*7pvg52ssE
rOABP3?aT~V$_{j!QF9Tp{`D-#`eAu0z?+o~q>2LwuLDCP2FwEh=Bu01

literal 0
HcmV?d00001

diff --git a/lib/galaxy/datatypes/test/hexrd.images.npz b/lib/galaxy/datatypes/test/hexrd.images.npz
new file mode 100644
index 0000000000000000000000000000000000000000..176070591f45d10de9c90a954107368b99f94390
GIT binary patch
literal 4322
zcmcIn3sh5A7JY;!N(dmokc0q3kwN5RAVq>eXzLWmVv!}5rjluXd@4jpjFnU)AHgI-
z2o-5XNx`<FLyHi~QqoQeA>V%R(~*&ADUihKs3=LTn2u5BJ*)*%x;o5i_gyzx@4UUw
zJNMjo-#RBN!}TdY005SPGZf&Nx=-q$D;EH81N;C2ZF^~1YzgO07hu_p5xfUL1PCpY
z0XG7deerpgKI>#_;}#k9SV{ZQlI)^mT^}56eTTMrQ&V$58?C7k=~I?56qyOH4JcfW
zJ_h>{SAbq~WDW8Z@``ILLbzG@j4%oH5oRax_4vy~M0`1&s@w6_^`{S}^8D@;U!5Ad
z5xp;(`06ywL|!rKwJ(u^r;PNL&>!el4jw<3qrIrU_~q%wZT{bEUCo_FPSYj_E<H$U
z`h=-y<?b=Xn{%#iy>eLh!51y6-8J`uOX1_Vevu%={_swuHOd<Gy(^G4Z~Xy>zwFh6
zOPgIktCa1$eux0!cUXVdpM@{u|DpK7Xf(c!?fdu=r=^z{U%|X$y2>b;u&dr^>}biB
zRv%`qJhZfKMIFIi>{opup51V%gfkA@*v}kLKWMreZ}6A44dExlsn<K(M{klwxMNpi
z8F#PkFe}gLzxr_S(}p)vS&=QVnvCPCsJLifQeRI+p4_x4??B!wc`i-~==tzZ$n)UZ
zpadXiUVuEt4u*3EAO?irBLjC(Zjxdw7Lyn!%U9~@XBEv1tmb({ZhC?KF6t(>oH$9R
zvt(yDb*AN-a$zrOkK{+m1#DMbXP)d!h2%LYH%fyQ#-I{0$K#I>wh*^5sAmXOwk6d_
zON{W74c|$wVM^i!^z%&FSDfG#yw=boT-UXLWzEr!Zp;`Vx}e!$m6}4&J8q%=C2`8d
z)cu2n>%mpN1y1NN@Lp$d@}HyvGD3g|5Ef3w!L9H8)?rA0d=~tG*mT6w%CDVbIM&nl
zVLIqu3{H12hLVEqjBWY6Cs9CV2oMFr{~g85dGvFEVgQ^%xUvpg|5wnr5S&mN@^-vA
zn+C`T0U|(HFo4&5*~fn?4L=3Y_0J)t&sw5W2>rExOUwej*Mk!UhLB02vDggW&uM_{
z5FieOUyI`PPUkQhIrBeFGhs9Vu><0vQSA@I=Hd5;hM7*$ytn*v_avC6$}Nu*Qo1d_
z@aH1}86iLf2=9I#dOo-KdA%**xF&DsnhljO7bO1Z`DlUa(s!=vO-9$I@Mp`Co(=do
z%gDRbeF(^(GL3DIAT}L-W%FM)=Lu`YwP7oto*B}=i)RMu{ikQf-WU4QzF?K#aXh*f
zIPTc><H;%aHqoO`%enV@rv$qmL|wWY7}C(ymOqmA@XCezZRJ0FTUzn`b=|j*Zqn}m
zd#tc*U?i=jeDnB&v5@lXj+C^o`W?F7$${QS<NeDTZ}lI!Y;tL^2XC~ytUd6b2Yj~*
zV7DEvl%$FGI6@r^hqrQXT5XyzP3r1!bF6nnDr?i~)2h?zUM<pEc^%%Gb0^YK18?Et
zs2TN`^0)-1y?t10+Et__Y^cZdH{;?Q+d*O!NQ49*f&bm@pr92OcclZK&W;mAo>GZp
zgsV=VF+DnK;EYZ7_@ZUz!jV%{<(-Qfcy|k3i`J}|VUMqmAu#JqQI82FGlaxV?h+%@
z{@E}r>@=o4ZHCu<9P*lVtbjt_=N-uI(O8#sV9^wY7Lhwia4?F5@ElCfAUYXU(R74S
z<)6-$>pV4QhEeXF1ZH*2PRv76f(8~PG1BXTgx&44ULt0_NV191a<$mww2@)IFh?0Z
zt4!N?My^7Jtu``6rkvSl(&m^0XB|3Tn5E<wg%F2&R6bFiab{Q`l~tEKX%(ARW#4aR
zNa62L81>!(#tB^RC_}UkRq^&^1Bw?d*CLB03Ke3+X!cA_VcKsFlM9+zRpB`lmKMHR
zp+}b)P2QSY3tAHwc6@L_$KNdIV)>~C>{SYVzyC!Hx5}fdM;Ek{ueF|BkUQQ!9jC#&
zNB5|4VX^YTgwqO*X9MOTc_9~^h=r_HuZ}%K1OqIiQ?3`gvpMxKx8<5JVa0}9EMXFw
z!rYgTPNi2Dv$>+6;5b3@LN2n}r{xW<LmY9ERIZA9hct*hL9w91&T^|V*qnXEqx8Dm
zQB!bo%7phu4Ee=djF2VGre!=lqly*H-Rqy(qsOMpEU-hy3FL@k5Z$NHr0(Ra!deNA
z>>{uGBo#{3Y+34#XYKdx<kTjdC7AI63N7Llh2G<|LKk$qN8=i7t1%^aq)u7@BAzJ?
zi{?neGv(^YizJPhN1gBqq?&MvMl-rhVes_gYkh@v!@<+sN~~v`!5dA|`$SMoe#J6|
zHYt{~-=m9DAIRp`C6BU1OH#~knr|1uH7p{Aip>8#5gW318x1Z6BrSd&Wdf5SqwmY^
z<O;p7DOBOB73zc%xi025ry(fVDvm%sYP_r=dlEFES4nD^fnt6tO2(`wN=OEHY|o&7
z7q>d5ge%%TX{}tB!my7lVsQhD*yY=c;66+}CEA0h3)c>DYv9rzty>OBhbzQzrS4Q_
zbrjWt$Q?C%-ZO$9Siai2nB9n2#BQv)>oaYQpSB@Att$)o9wNRlsnxQylwOs{mGsHA
z-;f5ADJFQPE%+ioq|gq8k+n%=HJq#uCl4ap%*ZwquFZm=(d$-m>$*9jZf--VT+_!_
zmGQOr_-ZfA!>~WrKl!E4YEqSwJ<7eV%3ADen$T8;^cqPWNSSyEZ;6Svt<JQ1=J2bs
zm7;_(1D323dvT==oT>)y{*Cg%6AE>qLVry$7+^GKjGCiHO~s>@z)5RNur(;ymfC7{
z6`tF)is7}271BKcbTfq`9C2Z@B}r;Up!vQAzI%qU-d9=e|JsD#6r=tgSLj7Bua#NC
zs1q!z*_UmN?6P9JY)K_n_-VUOpB-6d$KA6dk_B~31fq3<h6qQcsr*Lxq$z*W;*Xzr
zdB}?Mw4rz@2JbNytkjGtW$f?c?k$t6-DT#~P8<2G)g@gZ<_T)t1(FQGUJtU)pR7qB
z8$9||Xqp*KGkMZ1!Q*sMsjYI#DxR{{gb*Bsf2@B&>n^w5eC&Nbd<oy5s1&YE(csT<
zYZB$E7Lq=fV%{d32<o)LPVl{ol?_BerR|Qn!exFhMg02t=UxOWAPG33qK(5S$!F8H
z<~^xhAv*+!17X3Ut;DzJ7d4x6JNcw^d<KC)EL#nO$~705pX;x6wU3_${Z@ia4lvu}
zI?LuagAnyGKp7|`(0*zSon`Y|s3#)mX)Z8G_N1Y5*3EA>=IT6Lf!XH6SvJ2zm@7K~
z%I5S7&cgY1{am5nQ^0JM?<|{Ni_euE0%db6b7$rJx_PcL91Z|KSJBS0`R~zO*#=Kw
X-dmNG0Y^ah-N0c7w`dr=4*>rKI_s9t

literal 0
HcmV?d00001

diff --git a/lib/galaxy/datatypes/test/hexrd.materials.h5 b/lib/galaxy/datatypes/test/hexrd.materials.h5
new file mode 100644
index 0000000000000000000000000000000000000000..87ad66088aba226c411631efa545df271c206836
GIT binary patch
literal 10712
zcmeHNO-vI}5T0$pY%oT^AN)DECLS>H02LErVp>9xMzJ)s8jf9I15Ipqvn+`X2YbZC
z14j=YJb3Wv(IdwmIT%B{k{FL%J>h$AW?I<LKPuqQYtx<gX6DVj_wDR^vy;9~B_E#a
zyVeKtW17$d{i?~AJ^F0lMh^E`o~MREbe-tO2sR9mUw|vbA0Yc-z1-Z?6bMbW&y<G2
z#xS-_W@!3fA&{Dy$jAlH)2ca2s}yGp#$v%Px}`-GbCQ>0!4WCOWu#$T^5Zg2_PtBk
zs2<Uu9`<)3z;T(U4RDL5pYJ`mj)}l6ChfpxuIvY8&sibHr0>{4!7T|LN3yt|D-`Q}
z*BAWqN?^apimOu;dNXzq6c(I}?b}5saD29xwhubU`0kQWNeeCIG3(h2&b05Ydg;~T
zbH`UqajEmOj<8%>BEeIlyw7j&fK#T8&KE0+6HCy2FrZiM2|RBD@IWylxT2oW`ojIF
zIKiamfE5QkgAy=y7RL6-Pp{xgc_e}cO*-+S2C0KZ9*_(YTaMXTGxG<8C*IkNqsuVe
zjh_($wb#-CF6Vn)DpOx>zY1dNOE{#?&u@1hN4|WGhp7+&RGuZmRF2tiN8-PulmKq7
zeSbhrO*8@;0gZr0KqH_L_zehT)3ZrKy!B&F1Mov0_)R`^%tLCRiINaub_KB2LT{YR
zWCokt`_rYIU%sS?8k8UzhRkd*C)!V=BCC>;egQn@nwAuc9H>zqY0)j>%v7mEIFICc
zcqkS=a2U=4fb-a^VqP+@lRuQ9Ufn;k>v`-g@~^T%vJNNcf9#a~Cp+?gd~uMYd9)V?
z6}nU5c;v^NDaS?U?>F-_pOIQj$VQ{MJU(33JHo}Q;ex}ue?H!j|K`sJ?zd*GF16KS
z-^<WEGZ(1QfaHn4-9nG?U%uGD5V6;Z?zYf-OM0I#H@EkR#EGHidQ}z=bK`Kz`y8()
z9N&CDnRdS4zN7ou-g9<*^LXFZJPt>xHFtV)c1B@DVas@N6hE>{tF~XZVk39NFE+~M
zR#gKK=0`{F--}H=vD}gsR#ncaJc94`f+I>zrGlW0DwSto*_oe9su6hc&hP8!rba*`
zpb^jrXaqC@8Uc-fMnEH=5zq)|1T+F&A;5K%3R!7gN8$fL+^E&9hfElHMdR!B<M96=
dKD1TG!1@?htpPTvK*@E9@c%lxTKqpe_!n4N;cNf^

literal 0
HcmV?d00001


From f0cc1ec242a3730b0529f6caa205ba5da4c86a9d Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Mon, 10 May 2021 12:47:11 -0500
Subject: [PATCH 002/452] Update binary.py

Fix linting errors.
---
 lib/galaxy/datatypes/binary.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 342764ae98c..874d40bc724 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -5,7 +5,6 @@ import gzip
 import io
 import json
 import logging
-import numpy as np
 import os
 import shutil
 import struct
@@ -18,6 +17,7 @@ from json import dumps
 from typing import Optional
 
 import h5py
+import numpy as np
 import pysam
 import pysam.bcftools
 from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP
@@ -3149,7 +3149,7 @@ class Npz(CompressedArchive):
         try:
             npz = np.load(filename)
             return isinstance(npz, np.lib.npyio.NpzFile)
-        except Exception as e:
+        except Exception:
             return False
         return False
 
@@ -3200,7 +3200,7 @@ class HexrdImagesNpz(Npz):
     def sniff(self, filename):
         if super().sniff(filename):
             try:
-                req_files = {'0_row', '0_col', '0_data', 'shape', 'nframes',  'dtype'}
+                req_files = {'0_row', '0_col', '0_data', 'shape', 'nframes', 'dtype'}
                 with np.load(filename) as npz:
                     return set(npz.files) >= req_files
             except Exception as e:

From 68a95cccd25d0409ae046c1d739d4e9e425dfd10 Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Mon, 10 May 2021 14:39:13 -0500
Subject: [PATCH 003/452] Update binary.py

Fixes for tests.
---
 lib/galaxy/datatypes/binary.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 874d40bc724..4bc80850fdb 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -1552,7 +1552,7 @@ class HexrdMaterials(H5):
             req = {'AtomData', 'Atomtypes', 'CrystalSystem', 'LatticeParameters'}
             with h5py.File(filename, 'r') as mat_file:
                 for k in mat_file.keys():
-                    if set(mat_file[k].keys()) & req:
+                    if isinstance(mat_file[k], h5py._hl.group.Group) and set(mat_file[k].keys()) >= req:
                         return True
         return False
 
@@ -3130,7 +3130,7 @@ class Npz(CompressedArchive):
     Class describing an Numpy NPZ file
 
     >>> from galaxy.datatypes.sniff import get_test_fname
-    >>> fname = get_test_fname('test.mz5')
+    >>> fname = get_test_fname('hexrd.images.npz')
     >>> Npz().sniff(fname)
     True
     >>> fname = get_test_fname('interval.interval')
@@ -3183,7 +3183,7 @@ class HexrdImagesNpz(Npz):
     >>> fname = get_test_fname('hexrd.images.npz')
     >>> Npz().sniff(fname)
     True
-    >>> fname = get_test_fname('hexrd.estr(npz['panel_id'])ta_ome.npz')
+    >>> fname = get_test_fname('eta_ome.npz')
     >>> Npz().sniff(fname)
     False
     """

From d3a156ea0ffce754954b974ac70877280718639d Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Mon, 10 May 2021 15:05:28 -0500
Subject: [PATCH 004/452] Update binary.py

Fix tests
---
 lib/galaxy/datatypes/binary.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 4bc80850fdb..eb21078dfd5 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -3181,10 +3181,10 @@ class HexrdImagesNpz(Npz):
 
     >>> from galaxy.datatypes.sniff import get_test_fname
     >>> fname = get_test_fname('hexrd.images.npz')
-    >>> Npz().sniff(fname)
+    >>> HexrdImagesNpz().sniff(fname)
     True
     >>> fname = get_test_fname('eta_ome.npz')
-    >>> Npz().sniff(fname)
+    >>> HexrdImagesNpz().sniff(fname)
     False
     """
     file_ext = "hexrd.images.npz"
@@ -3242,10 +3242,10 @@ class HexrdEtaOmeNpz(Npz):
 
     >>> from galaxy.datatypes.sniff import get_test_fname
     >>> fname = get_test_fname('hexrd.eta_ome.npz')
-    >>> Npz().sniff(fname)
+    >>> HexrdEtaOmeNpz().sniff(fname)
     True
     >>> fname = get_test_fname('hexrd.images.npz')
-    >>> Npz().sniff(fname)
+    >>> HexrdEtaOmeNpz().sniff(fname)
     False
     """
     file_ext = "hexrd.eta_ome.npz"

From 0cd55c91a34f6da04a4012392393047627fd10d6 Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Mon, 10 May 2021 17:53:48 -0500
Subject: [PATCH 005/452] Update binary.py

Tighter requirement for sniffing npz files.
---
 lib/galaxy/datatypes/binary.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index eb21078dfd5..1faea327274 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -3148,7 +3148,10 @@ class Npz(CompressedArchive):
     def sniff(self, filename):
         try:
             npz = np.load(filename)
-            return isinstance(npz, np.lib.npyio.NpzFile)
+            if isinstance(npz, np.lib.npyio.NpzFile):
+                for f in npz.files:
+                    if isinstance(npz[f], np.ndarray):
+                        return True
         except Exception:
             return False
         return False

From acd249e7dae7bcbd41c3fc8ae9a28bc917c6dfb3 Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Fri, 14 May 2021 09:39:39 -0500
Subject: [PATCH 006/452] Update datatypes_conf.xml.sample

Forgot to add the generic numpy npz extension
---
 lib/galaxy/config/sample/datatypes_conf.xml.sample | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index 726f811dd08..e702ff930cb 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -863,6 +863,7 @@
     <datatype extension="pretext" type="galaxy.datatypes.binary:Pretext" display_in_upload="true" />
     <!-- Structural Materials datatypes -->
     <datatype extension="hexrd.materials.h5" type="galaxy.datatypes.binary:HexrdMaterials" display_in_upload="true"/>
+    <datatype extension="npz" type="galaxy.datatypes.binary:Npz" display_in_upload="true"/>
     <datatype extension="hexrd.images.npz" type="galaxy.datatypes.binary:HexrdImagesNpz" display_in_upload="true"/>
     <datatype extension="hexrd.eta_ome.npz" type="galaxy.datatypes.binary:HexrdEtaOmeNpz" display_in_upload="true"/>
     <!-- End Structural Materials datatypes -->

From c54e5fad403835ac68c6c41e6841658fc1189044 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 12:42:43 -0400
Subject: [PATCH 007/452] Replace legacy tool form wrapper with Vue component

---
 client/package.json                         |   1 +
 client/src/components/Form/Form.vue         |  81 +++-
 client/src/components/Form/FormCardTool.vue |   2 +
 client/src/components/Form/FormElement.vue  |  11 +-
 client/src/components/Tool/services.js      |  39 ++
 client/src/entry/analysis/AnalysisRouter.js |   4 +-
 client/src/mvc/form/form-input.js           |   4 +-
 client/src/mvc/form/form-view.js            |   2 +-
 client/src/mvc/tool/tool-form.js            | 472 --------------------
 client/tests/qunit/tests/form_tests.js      |  18 +-
 client/yarn.lock                            |  12 +
 11 files changed, 154 insertions(+), 492 deletions(-)
 delete mode 100644 client/src/mvc/tool/tool-form.js

diff --git a/client/package.json b/client/package.json
index b0ef323b5aa..2faba245fb7 100644
--- a/client/package.json
+++ b/client/package.json
@@ -81,6 +81,7 @@
     "vue-observe-visibility": "^1.0.0",
     "vue-router": "^3.3.1",
     "vue-rx": "^6.2.0",
+    "vue-scrollto": "^2.20.0",
     "vuedraggable": "2.24.1",
     "vueisotope": "^3.1.2",
     "vuex": "^3.4.0",
diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index 078671ffa80..a9bc63db550 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -3,6 +3,7 @@
 </template>
 
 <script>
+import { getGalaxyInstance } from "app";
 import Form from "mvc/form/form-view";
 
 export default {
@@ -27,18 +28,96 @@ export default {
             type: Boolean,
             default: false,
         },
+        validationErrors: {
+            type: Object,
+            default: null,
+        },
+    },
+    data() {
+        return {
+            formData: {},
+        };
     },
     watch: {
         id() {
             this.onRender();
         },
+        validationErrors() {
+            if (this.form) {
+                this.form.trigger("reset");
+                const matchedErrors = this.form.data.matchResponse(this.validationErrors);
+                for (const input_id in matchedErrors) {
+                    this.form.highlight(input_id, matchedErrors[input_id]);
+                }
+            }
+        },
+        validation() {
+            if (this.form) {
+                this.form.trigger("reset");
+                if (this.validation) {
+                    this.form.highlight(this.validation[0], this.validation[1], true);
+                }
+            }
+        },
     },
     mounted() {
         this.onRender();
     },
+    beforeDestroy() {
+        const Galaxy = getGalaxyInstance();
+        this.form.stopListening(Galaxy.currHistoryPanel.collection);
+    },
+    computed: {
+        validation() {
+            let batch_n = -1;
+            let batch_src = null;
+            for (const job_input_id in this.formData) {
+                const input_value = this.formData[job_input_id];
+                const input_id = this.form.data.match(job_input_id);
+                const input_field = this.form.field_list[input_id];
+                const input_def = this.form.input_list[input_id];
+                if (!input_id || !input_def || !input_field) {
+                    continue;
+                }
+                if (!input_def.optional && input_value == null && input_def.type != "hidden") {
+                    return [input_id, "Please provide a value for this option."];
+                }
+                if (input_field.validate) {
+                    const message = input_field.validate();
+                    if (message) {
+                        return [input_id, message];
+                    }
+                }
+                if (input_value && input_value.batch) {
+                    const n = input_value.values.length;
+                    const src = n > 0 && input_value.values[0] && input_value.values[0].src;
+                    if (src) {
+                        if (batch_src === null) {
+                            batch_src = src;
+                        } else if (batch_src !== src) {
+                            return [
+                                input_id,
+                                "Please select either dataset or dataset list fields for all batch mode fields.",
+                            ];
+                        }
+                    }
+                    if (batch_n === -1) {
+                        batch_n = n;
+                    } else if (batch_n !== n) {
+                        return [
+                            input_id,
+                            `Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>${n}</b> selection(s) while a previous field contains <b>${batch_n}</b>.`,
+                        ];
+                    }
+                }
+            }
+            return null;
+        },
+    },
     methods: {
         onChange() {
-            this.$emit("onChange", this.form.data.create());
+            this.formData = this.form.data.create();
+            this.$emit("onChange", this.formData);
         },
         onRender() {
             this.$nextTick(() => {
diff --git a/client/src/components/Form/FormCardTool.vue b/client/src/components/Form/FormCardTool.vue
index ffc3640f60d..228e4ce6a98 100644
--- a/client/src/components/Form/FormCardTool.vue
+++ b/client/src/components/Form/FormCardTool.vue
@@ -6,6 +6,7 @@
                     <b-dropdown
                         no-caret
                         right
+                        id="options"
                         role="button"
                         title="Options"
                         variant="link"
@@ -89,6 +90,7 @@
                 <slot name="body" />
             </div>
         </div>
+        <slot name="buttons" />
         <div class="m-1">
             <ToolHelp :content="options.help" />
             <ToolFooter
diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue
index 9235cbe2c38..2827231ab94 100644
--- a/client/src/components/Form/FormElement.vue
+++ b/client/src/components/Form/FormElement.vue
@@ -6,16 +6,19 @@
         </div>
         <div class="ui-form-title">{{ title }}</div>
         <div class="ui-form-field">
-            <FormInput :id="id" :value="value" :area="area" @onChange="onChange" />
+            <FormInput v-if="type == 'text'" :id="id" :value="value" :area="area" @onChange="onChange" />
+            <FormBoolean v-else-if="type == 'boolean'" :id="id" :value="value" @onChange="onChange" />
             <span class="ui-form-info form-text text-muted mt-2">{{ help }}</span>
         </div>
     </div>
 </template>
 <script>
-import FormInput from "./FormInput";
+import FormBoolean from "./Elements/FormBoolean";
+import FormInput from "./Elements/FormInput";
 
 export default {
     components: {
+        FormBoolean,
         FormInput,
     },
     props: {
@@ -43,6 +46,10 @@ export default {
             type: String,
             default: null,
         },
+        type: {
+            type: String,
+            default: "text",
+        },
     },
     computed: {
         hasError() {
diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index 7fc0c2f230e..d8a78e45779 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -1,6 +1,45 @@
+import $ from "jquery";
 import axios from "axios";
 import { rethrowSimple } from "utils/simple-error";
 import { getAppRoot } from "onload/loadConfig";
+import { getGalaxyInstance } from "app";
+
+/** Tools data request helper **/
+export async function getTool(tool_id, tool_version, job_id) {
+    const galaxy = getGalaxyInstance();
+    let url = "";
+    let data = {};
+
+    // build request url and collect request data
+    if (job_id) {
+        url = `${getAppRoot()}api/jobs/${job_id}/build_for_rerun`;
+    } else {
+        url = `${getAppRoot()}api/tools/${tool_id}/build`;
+        data = $.extend({}, galaxy.params);
+        data["tool_id"] && delete data["tool_id"];
+    }
+    tool_version && (data["tool_version"] = tool_version);
+
+    // attach data to request url
+    if (!$.isEmptyObject(data)) {
+        url += url.indexOf("?") == -1 ? "?" : "&";
+        url += $.param(data, true);
+    }
+
+    // request tool data
+    try {
+        const { data } = await axios.get(url);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}
+
+export async function submitJob(jobDetails) {
+    const url = `${getAppRoot()}api/tools`;
+    const { data } = await axios.post(url, jobDetails);
+    return data;
+}
 
 export async function addFavorite(userId, toolId) {
     const url = `${getAppRoot()}api/users/${userId}/favorites/tools`;
diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index 63e71476752..d0e31fa7618 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -15,7 +15,7 @@ import { getGalaxyInstance } from "app";
 import { getAppRoot } from "onload";
 import decodeUriComponent from "decode-uri-component";
 import Router from "layout/router";
-import ToolForm from "mvc/tool/tool-form";
+import ToolForm from "components/Tool/ToolForm";
 import FormWrapper from "mvc/form/form-wrapper";
 import Sharing from "components/Sharing.vue";
 import UserPreferences from "components/User/UserPreferences.vue";
@@ -427,7 +427,7 @@ export const getAnalysisRouter = (Galaxy) => {
             if (params.version) {
                 params.version = decodeUriComponent(params.version);
             }
-            this.page.display(new ToolForm.View(params));
+            this._display_vue_helper(ToolForm, params);
         },
 
         /** load the center panel iframe using the given url */
diff --git a/client/src/mvc/form/form-input.js b/client/src/mvc/form/form-input.js
index 8d337c2ccec..a4f84d75328 100644
--- a/client/src/mvc/form/form-input.js
+++ b/client/src/mvc/form/form-input.js
@@ -121,7 +121,7 @@ export default Backbone.View.extend({
         // render error messages
         var error_text = this.model.get("error_text");
         this.$error[error_text ? "show" : "hide"]();
-        this.$el[error_text ? "addClass" : "removeClass"]("alert alert-danger");
+        this.$el[error_text ? "addClass" : "removeClass"]("alert alert-info");
         this.$error_text.html(error_text);
         // render backdrop
         this.$backdrop[this.model.get("backdrop") ? "show" : "hide"]();
@@ -186,7 +186,7 @@ export default Backbone.View.extend({
             .append(
                 $("<div/>")
                     .addClass("ui-form-error")
-                    .append($("<span/>").addClass("fa fa-arrow-down mr-1"))
+                    .append($("<span/>").addClass("fa fa-exclamation mr-1"))
                     .append($("<span/>").addClass("ui-form-error-text"))
             )
             .append(
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index efaf8d6d6d6..e7b14bd92b1 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -54,7 +54,7 @@ export default Backbone.View.extend({
     highlight: function (input_id, message, silent) {
         var input_element = this.element_list[input_id];
         if (input_element) {
-            input_element.error(message || "Please verify this parameter.");
+            input_element.error(message || "Please verify this option.");
             this.portlet.expand();
             this.trigger("expand", input_id);
             if (!silent) {
diff --git a/client/src/mvc/tool/tool-form.js b/client/src/mvc/tool/tool-form.js
deleted file mode 100644
index e1bb04265ec..00000000000
--- a/client/src/mvc/tool/tool-form.js
+++ /dev/null
@@ -1,472 +0,0 @@
-/* This is the regular tool form */
-
-import _ from "underscore";
-import $ from "jquery";
-import Backbone from "backbone";
-import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
-import _l from "utils/localization";
-import Utils from "utils/utils";
-import Ui from "mvc/ui/ui-misc";
-import Modal from "mvc/ui/ui-modal";
-import ToolFormBase from "mvc/tool/tool-form-base";
-import Webhooks from "mvc/webhooks";
-import Vue from "vue";
-import ToolEntryPoints from "components/ToolEntryPoints/ToolEntryPoints";
-import ToolRecommendation from "components/ToolRecommendation";
-
-const View = Backbone.View.extend({
-    initialize: function (options) {
-        const Galaxy = getGalaxyInstance();
-        this.modal = Galaxy.modal || new Modal.View();
-        this.form = new ToolFormBase(
-            Utils.merge(
-                {
-                    setupToolMicrodata: true,
-                    listen_to_history: true,
-                    always_refresh: false,
-                    buildmodel: (process, form) => {
-                        const options = form.model.attributes;
-
-                        // build request url
-                        let build_url = "";
-                        let build_data = {};
-                        const job_id = options.job_id;
-                        if (job_id) {
-                            build_url = `${getAppRoot()}api/jobs/${job_id}/build_for_rerun`;
-                        } else {
-                            build_url = `${getAppRoot()}api/tools/${options.id}/build`;
-                            build_data = $.extend({}, Galaxy.params);
-                            build_data["tool_id"] && delete build_data["tool_id"];
-                        }
-                        options.version && (build_data["tool_version"] = options.version);
-
-                        // get initial model
-                        Utils.get({
-                            url: build_url,
-                            data: build_data,
-                            success: (data) => {
-                                if (!data.display) {
-                                    window.location = getAppRoot();
-                                    return;
-                                }
-
-                                // we can rely on tool_id, only if it's installed from toolshed
-                                // if no version provided and tool_id is not in toolshed style, don't show the warning
-                                const client_version =
-                                    options.version ||
-                                    (options.tool_id && options.tool_id.match(/^(toolshed)\./)
-                                        ? options.tool_id
-                                        : false);
-
-                                if (
-                                    data.versions &&
-                                    client_version &&
-                                    !data.versions.includes(client_version.replace(/.*\//, ""))
-                                ) {
-                                    data.message = `Requested version unavailable.`;
-                                }
-
-                                form.model.set(data);
-                                this._customize(form);
-                                Galaxy.emit.debug("tool-form-base::_buildModel()", "Initial tool model ready.", data);
-                                process.resolve();
-                            },
-                            error: (response, status) => {
-                                const error_message = (response && response.err_msg) || "Uncaught error.";
-                                if (status == 401) {
-                                    window.location = `${getAppRoot()}user/login?${$.param({
-                                        redirect: `${getAppRoot()}?tool_id=${options.id}`,
-                                    })}`;
-                                } else if (form.$el.is(":empty")) {
-                                    form.$el.prepend(
-                                        new Ui.Message({
-                                            message: error_message,
-                                            status: "danger",
-                                            persistent: true,
-                                        }).$el
-                                    );
-                                } else {
-                                    Galaxy.modal &&
-                                        Galaxy.modal.show({
-                                            title: _l("Tool request failed"),
-                                            body: error_message,
-                                            buttons: {
-                                                Close: () => {
-                                                    Galaxy.modal.hide();
-                                                },
-                                            },
-                                        });
-                                }
-                                Galaxy.emit.debug(
-                                    "tool-form-base::_buildModel()",
-                                    "Initial tool model request failed.",
-                                    response
-                                );
-                                process.reject();
-                            },
-                        });
-                    },
-                    postchange: (process, form) => {
-                        const current_state = {
-                            tool_id: form.model.get("id"),
-                            tool_version: form.model.get("version"),
-                            inputs: $.extend(true, {}, form.data.create()),
-                        };
-                        form.wait(true);
-                        Galaxy.emit.debug("tool-form::postchange()", "Sending current state.", current_state);
-                        Utils.request({
-                            type: "POST",
-                            url: `${getAppRoot()}api/tools/${form.model.get("id")}/build`,
-                            data: current_state,
-                            success: (data) => {
-                                form.update(data);
-                                form.wait(false);
-                                Galaxy.emit.debug("tool-form::postchange()", "Received new model.", data);
-                                process.resolve();
-                            },
-                            error: (response) => {
-                                Galaxy.emit.debug("tool-form::postchange()", "Refresh request failed.", response);
-                                process.reject();
-                            },
-                        });
-                    },
-                },
-                options
-            )
-        );
-        this.deferred = this.form.deferred;
-        this.setElement("<div/>");
-        this.$el.append(this.form.$el);
-    },
-
-    _customize: function (form) {
-        const Galaxy = getGalaxyInstance();
-        const options = form.model.attributes;
-        var inputs = options.inputs;
-        /* only append notification interface if
-           Galaxy has the server mail configured
-        */
-        if (Galaxy.config.server_mail_configured && !Galaxy.user.isAnonymous()) {
-            inputs.push({
-                name: `send_email_notification`,
-                label: _l("Email notification"),
-                type: "boolean",
-                value: "false",
-                ignore: "false",
-                help: _l("Send an email notification when the job completes."),
-            });
-        }
-
-        // build execute button
-        const execute_button = new Ui.Button({
-            icon: "fa-check",
-            tooltip: `Execute: ${options.name} (${options.version})`,
-            title: _l("Execute"),
-            cls: "btn btn-primary",
-            wait_cls: "btn btn-info",
-            onclick: () => {
-                execute_button.wait();
-                form.portlet.disable();
-                this.submit(options, () => {
-                    execute_button.unwait();
-                    form.portlet.enable();
-                });
-            },
-        });
-        options.buttons = { execute: execute_button };
-
-        // remap feature
-        if (options.job_id && options.job_remap) {
-            let label;
-            let help;
-            if (options.job_remap === "job_produced_collection_elements") {
-                label = "Replace elements in collection ?";
-                help =
-                    "The previous run of this tool failed. Use this option to replace the failed element(s) in the dataset collection that were produced during the previous tool run.";
-            } else {
-                label = "Resume dependencies from this job ?";
-                help =
-                    "The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.";
-            }
-            options.inputs.push({
-                label: label,
-                name: "rerun_remap_job_id",
-                type: "select",
-                display: "radio",
-                ignore: "__ignore__",
-                value: "__ignore__",
-                options: [
-                    ["Yes", options.job_id],
-                    ["No", "__ignore__"],
-                ],
-                help: help,
-            });
-        }
-
-        // Job Re-use Options
-        let extra_user_preferences = {};
-        if (Galaxy.user.attributes.preferences && "extra_user_preferences" in Galaxy.user.attributes.preferences) {
-            extra_user_preferences = JSON.parse(Galaxy.user.attributes.preferences.extra_user_preferences);
-        }
-        const use_cached_job =
-            "use_cached_job|use_cached_job_checkbox" in extra_user_preferences
-                ? extra_user_preferences["use_cached_job|use_cached_job_checkbox"]
-                : false;
-        if (use_cached_job === "true") {
-            options.inputs.push({
-                label: "BETA: Attempt to re-use jobs with identical parameters ?",
-                help: "This may skip executing jobs that you have already run",
-                name: "use_cached_job",
-                type: "select",
-                display: "radio",
-                ignore: "__ignore__",
-                value: "__ignore__",
-                options: [
-                    ["No", false],
-                    ["Yes", true],
-                ],
-            });
-        }
-    },
-
-    /** Submit a regular job.
-     * @param{dict}     options   - Specifies tool id and version
-     * @param{function} callback  - Called when request has completed
-     */
-    submit: function (options, callback) {
-        const Galaxy = getGalaxyInstance();
-        const history_id = Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.model.id;
-        const job_def = {
-            history_id: history_id,
-            tool_id: options.id,
-            tool_version: options.version,
-            inputs: this.form.data.create(),
-        };
-        this.form.trigger("reset");
-        if (!this.validate(job_def)) {
-            Galaxy.emit.debug("tool-form::submit()", "Submission canceled. Validation failed.");
-            callback && callback();
-            return;
-        }
-        if (options.action !== `${getAppRoot()}tool_runner/index`) {
-            const $f = $("<form/>").attr({
-                action: options.action,
-                method: options.method,
-                enctype: options.enctype,
-            });
-            _.each(job_def.inputs, (value, key) => {
-                $f.append($("<input/>").attr({ name: key, value: value }));
-            });
-            $f.hide().appendTo("body").submit().remove();
-            callback && callback();
-            return;
-        }
-        Galaxy.emit.debug("tool-form::submit()", "Validation complete.", job_def);
-        Utils.request({
-            type: "POST",
-            url: `${getAppRoot()}api/tools`,
-            data: job_def,
-            success: (response) => {
-                callback && callback();
-                this.$el.children().hide();
-                if (response.produces_entry_points) {
-                    for (const job of response.jobs) {
-                        const toolEntryPointsInstance = Vue.extend(ToolEntryPoints);
-                        const vm = document.createElement("div");
-                        this.$el.append(vm);
-                        const instance = new toolEntryPointsInstance({
-                            propsData: {
-                                jobId: job.id,
-                            },
-                        });
-                        instance.$mount(vm);
-                    }
-                }
-                this.$el.append(this._templateSuccess(response, job_def));
-                const enable_tool_recommendations = Galaxy.config.enable_tool_recommendations;
-                if (enable_tool_recommendations === true || enable_tool_recommendations === "true") {
-                    // show tool recommendations
-                    const ToolRecommendationInstance = Vue.extend(ToolRecommendation);
-                    const vm = document.createElement("div");
-                    this.$el.append(vm);
-                    const instance = new ToolRecommendationInstance({
-                        propsData: {
-                            toolId: job_def.tool_id,
-                        },
-                    });
-                    instance.$mount(vm);
-                }
-                this.$el.parent().scrollTop(0);
-                // Show Webhook if job is running
-                if (response.jobs && response.jobs.length > 0) {
-                    this.$el.append($("<div/>", { id: "webhook-view" }));
-                    new Webhooks.WebhookView({
-                        type: "tool",
-                        toolId: job_def.tool_id,
-                    });
-                }
-                if (Galaxy.currHistoryPanel) {
-                    this.form.stopListening(Galaxy.currHistoryPanel.collection);
-                    Galaxy.currHistoryPanel.refreshContents();
-                }
-            },
-            error: (response) => {
-                callback && callback();
-                Galaxy.emit.debug("tool-form::submit", "Submission failed.", response);
-                let input_found = false;
-                if (response && response.err_data) {
-                    const error_messages = this.form.data.matchResponse(response.err_data);
-                    for (const input_id in error_messages) {
-                        this.form.highlight(input_id, error_messages[input_id]);
-                        input_found = true;
-                        break;
-                    }
-                }
-                if (!input_found) {
-                    this.modal.show({
-                        title: _l("Job submission failed"),
-                        body: this._templateError(job_def, response && response.err_msg),
-                        buttons: {
-                            Close: () => {
-                                this.modal.hide();
-                            },
-                        },
-                    });
-                }
-            },
-        });
-    },
-
-    /** Validate job dictionary.
-     * @param{dict}     job_def   - Job execution dictionary
-     */
-    validate: function (job_def) {
-        const Galaxy = getGalaxyInstance();
-        const job_inputs = job_def.inputs;
-        let batch_n = -1;
-        let batch_src = null;
-        for (const job_input_id in job_inputs) {
-            const input_value = job_inputs[job_input_id];
-            const input_id = this.form.data.match(job_input_id);
-            const input_field = this.form.field_list[input_id];
-            const input_def = this.form.input_list[input_id];
-            if (!input_id || !input_def || !input_field) {
-                Galaxy.emit.debug("tool-form::validate()", "Retrieving input objects failed.");
-                continue;
-            }
-            if (!input_def.optional && input_value == null && input_def.type != "hidden") {
-                this.form.highlight(input_id, "This input requires a value.");
-                return false;
-            }
-            if (input_field.validate) {
-                const message = input_field.validate();
-                if (message) {
-                    this.form.highlight(input_id, message);
-                    return false;
-                }
-            }
-            if (input_value && input_value.batch) {
-                const n = input_value.values.length;
-                const src = n > 0 && input_value.values[0] && input_value.values[0].src;
-                if (src) {
-                    if (batch_src === null) {
-                        batch_src = src;
-                    } else if (batch_src !== src) {
-                        this.form.highlight(
-                            input_id,
-                            "Please select either dataset or dataset list fields for all batch mode fields."
-                        );
-                        return false;
-                    }
-                }
-                if (batch_n === -1) {
-                    batch_n = n;
-                } else if (batch_n !== n) {
-                    this.form.highlight(
-                        input_id,
-                        `Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>${n}</b> selection(s) while a previous field contains <b>${batch_n}</b>.`
-                    );
-                    return false;
-                }
-            }
-        }
-        return true;
-    },
-
-    _getInputs: function (job_def) {
-        const inputs = [];
-        const index = {};
-        for (const i in job_def.inputs) {
-            const input = job_def.inputs[i];
-            if (input && Array.isArray(input.values)) {
-                for (const j of input.values) {
-                    if (j.src && !index[j.id]) {
-                        inputs.push(j);
-                        index[j.id] = true;
-                    }
-                }
-            }
-        }
-        return inputs;
-    },
-
-    _templateRow: function (list, title) {
-        let blurb = "";
-        if (list.length > 0) {
-            blurb += `<p>${title}:</p>`;
-            list.sort((a, b) => {
-                b.hid - a.hid;
-            });
-            blurb += `<ul>`;
-            for (const item of list) {
-                const rowString = `${item.hid}: ${_.escape(item.name)}`;
-                blurb += `<li><b>${rowString}</b></li>`;
-            }
-            blurb += `</ul>`;
-        }
-        return blurb;
-    },
-
-    _templateSuccess: function (response, job_def) {
-        const njobs = response && response.jobs ? response.jobs.length : 0;
-        if (njobs > 0) {
-            const inputs = this._getInputs(job_def);
-            const ninputs = inputs.length;
-            const noutputs = response.outputs.length;
-            const njobsText = njobs > 1 ? `${njobs} jobs` : `1 job`;
-            const ninputsText = ninputs > 1 ? `${ninputs} inputs` : `this input`;
-            const noutputsText = noutputs > 1 ? `${noutputs} outputs` : `this output`;
-            const tool_name = this.form.model.get("name");
-            return `<div class="donemessagelarge">
-                        <p>
-                            Executed <b>${tool_name}</b> and successfully added ${njobsText} to the queue.
-                        </p>
-                        ${this._templateRow(inputs, `The tool uses ${ninputsText}`)}
-                        ${this._templateRow(response.outputs, `It produces ${noutputsText}`)}
-                        <p>
-                            You can check the status of queued jobs and view the resulting data by refreshing the History panel. When the job has been run the status will change from 'running' to 'finished' if completed successfully or 'error' if problems were encountered.
-                        </p>
-                    </div>`;
-        } else {
-            return this._templateError(response, "Invalid success response. No jobs found.");
-        }
-    },
-
-    _templateError: function (response, err_msg) {
-        return $("<div/>")
-            .addClass("errormessagelarge")
-            .append(
-                $("<p/>").text(
-                    `The server could not complete the request. Please contact the Galaxy Team if this error persists. ${
-                        err_msg || ""
-                    }`
-                )
-            )
-            .append($("<pre/>").text(JSON.stringify(response, null, 4)));
-    },
-});
-
-export default {
-    View: View,
-};
diff --git a/client/tests/qunit/tests/form_tests.js b/client/tests/qunit/tests/form_tests.js
index 2023ebb6f61..1968121230e 100644
--- a/client/tests/qunit/tests/form_tests.js
+++ b/client/tests/qunit/tests/form_tests.js
@@ -8,9 +8,10 @@ import testApp from "../test-app";
 import InputElement from "mvc/form/form-input";
 import Ui from "mvc/ui/ui-misc";
 import FormData from "mvc/form/form-data";
-import ToolForm from "mvc/tool/tool-form";
+import Form from "mvc/form/form-view";
 import Utils from "utils/utils";
 import { getAppRoot } from "onload/loadConfig";
+import toolsTestBuild from "../test-data/json/tools.test.build";
 
 QUnit.module("Form test", {
     beforeEach: function () {
@@ -32,16 +33,9 @@ QUnit.module("Form test", {
     },
 });
 
-QUnit.test("tool-form", function (assert) {
-    var toolform = new ToolForm.View({ id: "test" });
-    $("body").prepend(toolform.$el);
-    window.fakeserver.respond();
-
-    var form = toolform.form;
-    assert.ok(
-        form.$(".portlet-title-text").html() == `<b itemprop="name">_name</b> <span itemprop="description">_description</span> (Galaxy Version _version)`,
-        "Title correct"
-    );
+QUnit.test("form", function (assert) {
+    var form = new Form(toolsTestBuild);
+    $("body").prepend(form.$el);
     var tour_ids = [];
     $("[tour_id]").each(function () {
         tour_ids.push($(this).attr("tour_id"));
@@ -90,7 +84,7 @@ QUnit.test("data", function (assert) {
     Utils.get({
         url: getAppRoot() + "api/tools/test/build",
         success: function (response) {
-            FormData.visitInputs(response.inputs, function (node, name, context) {
+            FormData.visitInputs(response.inputs, function (node, name) {
                 visits.push({ name: name, node: node });
             });
         },
diff --git a/client/yarn.lock b/client/yarn.lock
index a6c52c2924a..69b5628a8a7 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -3239,6 +3239,11 @@ better-assert@~1.0.0:
   dependencies:
     callsite "1.0.0"
 
+bezier-easing@2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/bezier-easing/-/bezier-easing-2.1.0.tgz#c04dfe8b926d6ecaca1813d69ff179b7c2025d86"
+  integrity sha1-wE3+i5JtbsrKGBPWn/F5t8ICXYY=
+
 big.js@^3.1.3:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@@ -15173,6 +15178,13 @@ vue-rx@^6.2.0:
   resolved "https://registry.yarnpkg.com/vue-rx/-/vue-rx-6.2.0.tgz#c3b86462b252626edd16417385bb9054a3a23acb"
   integrity sha512-tpKUcqS5IUYS+HsdbR5TlE5LL9PK4zwlplEtmMeydnbqaUTa9ciLMplJXAtFSiQw1vuURoyEJmCXqMxaVEIloQ==
 
+vue-scrollto@^2.20.0:
+  version "2.20.0"
+  resolved "https://registry.yarnpkg.com/vue-scrollto/-/vue-scrollto-2.20.0.tgz#3ba52239a62710c97003d12d4393f1f3393cd5cc"
+  integrity sha512-7i+AGKJTThZnMEkhIPgrZjyAX+fXV7/rGdg+CV283uZZwCxwiMXaBLTmIc5RTA4uwGnT+E6eJle3mXQfM2OD3A==
+  dependencies:
+    bezier-easing "2.1.0"
+
 vue-style-loader@^4.1.0:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/vue-style-loader/-/vue-style-loader-4.1.2.tgz#dedf349806f25ceb4e64f3ad7c0a44fba735fcf8"

From e6645ec152b71a2ac6c605630e067cf5daeab173 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 12:45:31 -0400
Subject: [PATCH 008/452] Add new component files for tool form rewrite

---
 .../src/components/Common/ButtonSpinner.vue   |  36 +++
 client/src/components/Common/Webhook.vue      |  27 ++
 .../components/Form/Elements/FormBoolean.vue  |  34 +++
 .../Form/Elements/FormInput.test.js           |  38 +++
 .../components/Form/Elements/FormInput.vue    |  37 +++
 .../components/Form/Elements/Input.test.js    |  38 +++
 client/src/components/Tool/ToolForm.vue       | 275 ++++++++++++++++++
 client/src/components/Tool/ToolSuccess.vue    |  79 +++++
 client/src/components/Tool/utilities.js       |  13 +
 9 files changed, 577 insertions(+)
 create mode 100644 client/src/components/Common/ButtonSpinner.vue
 create mode 100644 client/src/components/Common/Webhook.vue
 create mode 100644 client/src/components/Form/Elements/FormBoolean.vue
 create mode 100644 client/src/components/Form/Elements/FormInput.test.js
 create mode 100644 client/src/components/Form/Elements/FormInput.vue
 create mode 100644 client/src/components/Form/Elements/Input.test.js
 create mode 100644 client/src/components/Tool/ToolForm.vue
 create mode 100644 client/src/components/Tool/ToolSuccess.vue
 create mode 100644 client/src/components/Tool/utilities.js

diff --git a/client/src/components/Common/ButtonSpinner.vue b/client/src/components/Common/ButtonSpinner.vue
new file mode 100644
index 00000000000..c316d4005ed
--- /dev/null
+++ b/client/src/components/Common/ButtonSpinner.vue
@@ -0,0 +1,36 @@
+<template>
+    <b-button v-if="wait" variant="info" disabled>
+        <font-awesome-icon icon="spinner" class="mr-2" spin />Please wait...
+    </b-button>
+    <b-button v-else variant="primary" @click="$emit('onClick')">
+        <font-awesome-icon icon="check" class="mr-2" />{{ title }}
+    </b-button>
+</template>
+<script>
+import Vue from "vue";
+import BootstrapVue from "bootstrap-vue";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faCheck, faSpinner } from "@fortawesome/free-solid-svg-icons";
+
+Vue.use(BootstrapVue);
+
+library.add(faCheck);
+library.add(faSpinner);
+
+export default {
+    components: {
+        FontAwesomeIcon,
+    },
+    props: {
+        title: {
+            type: String,
+            required: true,
+        },
+        wait: {
+            type: Boolean,
+            default: false,
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Common/Webhook.vue b/client/src/components/Common/Webhook.vue
new file mode 100644
index 00000000000..72cdbceadb8
--- /dev/null
+++ b/client/src/components/Common/Webhook.vue
@@ -0,0 +1,27 @@
+<template>
+    <div ref="webhook" id="webhook-view" />
+</template>
+<script>
+import Webhooks from "mvc/webhooks";
+
+export default {
+    props: {
+        type: {
+            type: String,
+            required: true,
+        },
+        toolId: {
+            type: String,
+            default: null,
+        },
+    },
+    mounted() {
+        const el = this.$refs["webhook"];
+        new Webhooks.WebhookView({
+            el,
+            type: "tool",
+            toolId: this.toolId,
+        });
+    },
+};
+</script>
diff --git a/client/src/components/Form/Elements/FormBoolean.vue b/client/src/components/Form/Elements/FormBoolean.vue
new file mode 100644
index 00000000000..6e3b3366ee7
--- /dev/null
+++ b/client/src/components/Form/Elements/FormBoolean.vue
@@ -0,0 +1,34 @@
+<template>
+    <b-form-checkbox v-model="currentValue" size="lg" switch @onChange="onChange" />
+</template>
+
+<script>
+import Vue from "vue";
+import BootstrapVue from "bootstrap-vue";
+
+Vue.use(BootstrapVue);
+
+export default {
+    props: {
+        value: {
+            type: Boolean,
+            required: true,
+        },
+    },
+    data() {
+        return {
+            currentValue: this.value,
+        };
+    },
+    watch: {
+        value(newValue) {
+            this.currentValue = newValue;
+        },
+    },
+    methods: {
+        onChange() {
+            this.$emit("onChange", this.currentValue);
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Form/Elements/FormInput.test.js b/client/src/components/Form/Elements/FormInput.test.js
new file mode 100644
index 00000000000..ebe24aa5960
--- /dev/null
+++ b/client/src/components/Form/Elements/FormInput.test.js
@@ -0,0 +1,38 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import FormInput from "./FormInput";
+
+jest.mock("app");
+
+const localVue = getLocalVue();
+
+describe("FormInput", () => {
+    let wrapper;
+
+    beforeEach(() => {
+        wrapper = mount(FormInput, {
+            propsData: {
+                id: "input",
+                value: "initial_value",
+            },
+            localVue,
+        });
+    });
+
+    it("check initial value and value change", async () => {
+        const input = wrapper.find("input");
+        expect(input.element.value).toBe("initial_value");
+        await input.setValue("new_value");
+        expect(input.element.value).toBe("new_value");
+        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
+    });
+
+    it("check switching to text area", async () => {
+        await wrapper.setProps({ area: true });
+        const input = wrapper.find("textarea");
+        expect(input.element.value).toBe("initial_value");
+        await input.setValue("new_value");
+        expect(input.element.value).toBe("new_value");
+        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
+    });
+});
diff --git a/client/src/components/Form/Elements/FormInput.vue b/client/src/components/Form/Elements/FormInput.vue
new file mode 100644
index 00000000000..ee344d00eb5
--- /dev/null
+++ b/client/src/components/Form/Elements/FormInput.vue
@@ -0,0 +1,37 @@
+<template>
+    <textarea v-if="area" class="ui-input" v-model="currentValue" @input="onChange" @change="onChange" />
+    <input v-else class="ui-input" v-model="currentValue" @input="onChange" @change="onChange" />
+</template>
+<script>
+export default {
+    props: {
+        id: {
+            type: String,
+            required: true,
+        },
+        value: {
+            type: String,
+            default: "",
+        },
+        area: {
+            type: Boolean,
+            default: false,
+        },
+    },
+    data() {
+        return {
+            currentValue: this.value,
+        };
+    },
+    watch: {
+        value(newValue) {
+            this.currentValue = newValue;
+        },
+    },
+    methods: {
+        onChange() {
+            this.$emit("onChange", this.currentValue);
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Form/Elements/Input.test.js b/client/src/components/Form/Elements/Input.test.js
new file mode 100644
index 00000000000..ebe24aa5960
--- /dev/null
+++ b/client/src/components/Form/Elements/Input.test.js
@@ -0,0 +1,38 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import FormInput from "./FormInput";
+
+jest.mock("app");
+
+const localVue = getLocalVue();
+
+describe("FormInput", () => {
+    let wrapper;
+
+    beforeEach(() => {
+        wrapper = mount(FormInput, {
+            propsData: {
+                id: "input",
+                value: "initial_value",
+            },
+            localVue,
+        });
+    });
+
+    it("check initial value and value change", async () => {
+        const input = wrapper.find("input");
+        expect(input.element.value).toBe("initial_value");
+        await input.setValue("new_value");
+        expect(input.element.value).toBe("new_value");
+        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
+    });
+
+    it("check switching to text area", async () => {
+        await wrapper.setProps({ area: true });
+        const input = wrapper.find("textarea");
+        expect(input.element.value).toBe("initial_value");
+        await input.setValue("new_value");
+        expect(input.element.value).toBe("new_value");
+        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
+    });
+});
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
new file mode 100644
index 00000000000..aa8d3f590c6
--- /dev/null
+++ b/client/src/components/Tool/ToolForm.vue
@@ -0,0 +1,275 @@
+<template>
+    <CurrentUser v-slot="{ user }">
+        <div>
+            <LoadingSpan v-if="showLoading" message="Loading Tool" />
+            <div v-if="showEntryPoints">
+                <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
+            </div>
+            <ToolSuccess v-if="showSuccess" :job-def="jobDef" :job-response="jobResponse" :tool-name="toolName" />
+            <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
+            <div v-if="showError" class="errormessagelarge">
+                <p>
+                    The server could not complete the request. Please contact the Galaxy Team if this error persists.
+                    {{ errorTitle | l }}
+                </p>
+                <pre>{{ errorContentPretty }}</pre>
+            </div>
+            <ToolRecommendation v-if="showRecommendation" :tool-id="id" />
+            <FormCardTool
+                v-if="showForm"
+                :id="formConfig.id"
+                :user="user"
+                :version="formConfig.version"
+                :title="formConfig.name"
+                :description="formConfig.description"
+                :options="formConfig"
+                :message-text="messageText"
+                :message-variant="messageVariant"
+                @onChangeVersion="onChangeVersion"
+                @onUpdateFavorites="onUpdateFavorites"
+                itemscope="itemscope"
+                itemtype="https://schema.org/CreativeWork"
+            >
+                <template v-slot:body>
+                    <Form
+                        :id="formConfig.id"
+                        :inputs="inputs"
+                        :validation-errors="validationErrors"
+                        @onChange="onChange"
+                    />
+                    <FormElement
+                        v-if="emailAllowed"
+                        id="send_email_notification"
+                        title="Email notification"
+                        help="Send an email notification when the job completes."
+                        type="boolean"
+                        value="false"
+                        @onChange="onChangeEmail"
+                    />
+                    <FormElement
+                        v-if="remapAllowed"
+                        id="rerun_remap_job_id"
+                        :title="remapTitle"
+                        :help="remapHelp"
+                        type="boolean"
+                        value="false"
+                        @onChange="onChangeRemap"
+                    />
+                    <FormElement
+                        v-if="reuseAllowed"
+                        id="use_cached_job"
+                        title="Attempt to re-use jobs with identical parameters?"
+                        help="This may skip executing jobs that you have already run."
+                        type="boolean"
+                        value="false"
+                        @onChange="onChangeReuse"
+                    />
+                </template>
+                <template v-slot:buttons>
+                    <ButtonSpinner id="execute" title="Execute" @onClick="onExecute" :wait="showExecuting" />
+                </template>
+            </FormCardTool>
+        </div>
+    </CurrentUser>
+</template>
+
+<script>
+import Scroller from "vue-scrollto";
+import { getAppRoot } from "onload/loadConfig";
+import { getGalaxyInstance } from "app";
+import { getTool, submitJob } from "./services";
+import { send } from "./utilities";
+import ButtonSpinner from "components/Common/ButtonSpinner";
+import CurrentUser from "components/providers/CurrentUser";
+import LoadingSpan from "components/LoadingSpan";
+import Form from "components/Form/Form";
+import FormCardTool from "components/Form/FormCardTool";
+import FormElement from "components/Form/FormElement";
+import ToolSuccess from "./ToolSuccess";
+import Webhook from "components/Common/Webhook";
+
+export default {
+    components: {
+        ButtonSpinner,
+        CurrentUser,
+        LoadingSpan,
+        Form,
+        FormCardTool,
+        FormElement,
+        ToolSuccess,
+        Webhook,
+    },
+    props: {
+        id: {
+            type: String,
+            default: null,
+        },
+        version: {
+            type: String,
+            default: null,
+        },
+        job_id: {
+            type: String,
+            default: null,
+        },
+    },
+    data() {
+        return {
+            showLoading: true,
+            showForm: false,
+            showEntryPoints: false,
+            showRecommendation: false,
+            showSuccess: false,
+            showError: false,
+            showExecuting: false,
+            error: null,
+            formConfig: {},
+            formData: {},
+            errorTitle: "",
+            errorContent: null,
+            messageVariant: "",
+            messageText: "",
+            reuse: false,
+            email: false,
+            remap: false,
+            entryPoints: [],
+            jobDef: {},
+            jobResponse: {},
+            validationErrors: {},
+        };
+    },
+    created() {
+        this.requestTool();
+    },
+    computed: {
+        toolName() {
+            return this.formConfig.name;
+        },
+        errorContentPretty() {
+            return JSON.stringify(this.errorContent, null, 4);
+        },
+        emailAllowed() {
+            const Galaxy = getGalaxyInstance();
+            return Galaxy.config.server_mail_configured && !Galaxy.user.isAnonymous();
+        },
+        inputs() {
+            return this.formConfig.inputs;
+        },
+        remapAllowed() {
+            return this.job_id && this.formConfig.job_remap;
+        },
+        remapLabel() {
+            if (this.formConfig.job_remap === "job_produced_collection_elements") {
+                return "Replace elements in collection ?";
+            } else {
+                return "Resume dependencies from this job ?";
+            }
+        },
+        remapHelp() {
+            if (this.formConfig.job_remap === "job_produced_collection_elements") {
+                return "The previous run of this tool failed. Use this option to replace the failed element(s) in the dataset collection that were produced during the previous tool run.";
+            } else {
+                return "The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.";
+            }
+        },
+        reuseAllowed() {
+            const Galaxy = getGalaxyInstance();
+            if (Galaxy.user.attributes.preferences && "extra_user_preferences" in Galaxy.user.attributes.preferences) {
+                const extra_user_preferences = JSON.parse(Galaxy.user.attributes.preferences.extra_user_preferences);
+                const keyCached = "use_cached_job|use_cached_job_checkbox";
+                const hasCachedJobs = keyCached in extra_user_preferences;
+                return hasCachedJobs ? extra_user_preferences[keyCached] : false;
+            } else {
+                return false;
+            }
+        },
+    },
+    methods: {
+        onChange(newData) {
+            this.formData = newData;
+        },
+        onChangeVersion(newVersion) {
+            this.requestTool(newVersion);
+        },
+        onChangeEmail(email) {
+            this.email = email;
+        },
+        onChangeRemap(remap) {
+            this.remap = remap ? this.job_id : null;
+        },
+        onChangeReuse(reuse) {
+            this.reuse = reuse;
+        },
+        onUpdateFavorites(user, newFavorites) {
+            user.preferences["favorites"] = newFavorites;
+        },
+        requestTool(newVersion) {
+            const version = newVersion || this.version;
+            getTool(this.id, version, this.job_id).then((data) => {
+                this.formConfig = data;
+                this.showLoading = false;
+                this.showForm = true;
+                if (newVersion) {
+                    this.messageVariant = "success";
+                    this.messageText = `Now you are using '${data.name}' version ${data.version}, id '${data.id}'.`;
+                }
+            });
+        },
+        onExecute() {
+            this.showExecuting = true;
+            const options = this.formData;
+            const Galaxy = getGalaxyInstance();
+            const history_id = Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.model.id;
+            const jobDef = {
+                history_id: history_id,
+                tool_id: this.formConfig.id,
+                tool_version: this.formConfig.version,
+                inputs: this.formData,
+            };
+            if (this.formConfig.action !== `${getAppRoot()}tool_runner/index`) {
+                send(options, jobDef);
+                return;
+            }
+            Galaxy.emit.debug("tool-form::submit()", "Validation complete.", jobDef);
+            submitJob(jobDef).then(
+                (jobResponse) => {
+                    if (Galaxy.currHistoryPanel) {
+                        Galaxy.currHistoryPanel.refreshContents();
+                    }
+                    this.showForm = false;
+                    if (jobResponse.produces_entry_points) {
+                        this.showEntryPoints = true;
+                        this.entryPoints = jobResponse.jobs;
+                    }
+                    const nJobs = jobResponse && jobResponse.jobs ? jobResponse.jobs.length : 0;
+                    if (nJobs > 0) {
+                        this.showSuccess = true;
+                        this.jobDef = jobDef;
+                        this.jobResponse = jobResponse;
+                    } else {
+                        this.showError = true;
+                        this.errorTitle = "Invalid success response. No jobs found.";
+                        this.errorContent = jobResponse;
+                    }
+                    if ([true, "true"].includes(Galaxy.config.enable_tool_recommendations)) {
+                        this.showRecommendation = true;
+                    }
+                    Scroller.scrollTo("body");
+                },
+                (e) => {
+                    this.showExecuting = false;
+                    const errorData = e && e.response && e.response.data && e.response.data.err_data;
+                    if (errorData) {
+                        this.validationErrors = errorData;
+                    } else {
+                        this.showError = true;
+                        this.showForm = false;
+                        this.errorTitle = "Job submission failed";
+                        this.errorContent = this.jobDef;
+                    }
+                }
+            );
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Tool/ToolSuccess.vue b/client/src/components/Tool/ToolSuccess.vue
new file mode 100644
index 00000000000..eeb9e8e3ed6
--- /dev/null
+++ b/client/src/components/Tool/ToolSuccess.vue
@@ -0,0 +1,79 @@
+<template>
+    <div class="donemessagelarge">
+        <p>
+            Executed <b>{{ toolName }}</b> and successfully added {{ nJobsText }} to the queue.
+        </p>
+        <p>The tool uses {{ nInputsText }}:</p>
+        <ul>
+            <li v-for="item of inputs" :key="item.hid">
+                <b>{{ item.hid }}: {{ item.name }}</b>
+            </li>
+        </ul>
+        <p>It produces {{ nOutputsText }}:</p>
+        <ul>
+            <li v-for="item of jobResponse.outputs" :key="item.hid">
+                <b>{{ item.hid }}: {{ item.name }}</b>
+            </li>
+        </ul>
+        <p>
+            You can check the status of queued jobs and view the resulting data by refreshing the History panel. When
+            the job has been run the status will change from 'running' to 'finished' if completed successfully or
+            'error' if problems were encountered.
+        </p>
+    </div>
+</template>
+
+<script>
+export default {
+    props: {
+        jobDef: {
+            type: Object,
+            required: true,
+        },
+        jobResponse: {
+            type: Object,
+            required: true,
+        },
+        toolName: {
+            type: String,
+            required: true,
+        },
+    },
+    computed: {
+        inputs() {
+            const inputs = [];
+            const index = {};
+            for (const i in this.jobDef.inputs) {
+                const input = this.jobDef.inputs[i];
+                if (input && Array.isArray(input.values)) {
+                    for (const j of input.values) {
+                        if (j.src && !index[j.id]) {
+                            inputs.push(j);
+                            index[j.id] = true;
+                        }
+                    }
+                }
+            }
+            return inputs;
+        },
+        nInputs() {
+            return this.inputs.length;
+        },
+        nOutputs() {
+            return this.jobResponse && this.jobResponse.outputs ? this.jobResponse.outputs.length : 0;
+        },
+        nJobs() {
+            return this.jobResponse && this.jobResponse.jobs ? this.jobResponse.jobs.length : 0;
+        },
+        nJobsText() {
+            return this.nJobs > 1 ? `${this.njobs} jobs` : `1 job`;
+        },
+        nInputsText() {
+            return this.nInputs > 1 ? `${this.nInputs} inputs` : `this input`;
+        },
+        nOutputsText() {
+            return this.nOutputs > 1 ? `${this.nOutputs} outputs` : `this output`;
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Tool/utilities.js b/client/src/components/Tool/utilities.js
new file mode 100644
index 00000000000..db25be8f17e
--- /dev/null
+++ b/client/src/components/Tool/utilities.js
@@ -0,0 +1,13 @@
+import $ from "jquery";
+
+export function send(options, job_def) {
+    const $f = $("<form/>").attr({
+        action: options.action,
+        method: options.method,
+        enctype: options.enctype,
+    });
+    Object.entries(job_def.inputs).forEach(([value, key]) => {
+        $f.append($("<input/>").attr({ name: key, value: value }));
+    });
+    $f.hide().appendTo("body").submit().remove();
+}

From aa320638ea3ef8de7e38daa5cf92ae51011c48f3 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 12:54:38 -0400
Subject: [PATCH 009/452] Fix webhook type parsing in webhook wrapper component

---
 client/src/components/Common/Webhook.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Common/Webhook.vue b/client/src/components/Common/Webhook.vue
index 72cdbceadb8..a3885849690 100644
--- a/client/src/components/Common/Webhook.vue
+++ b/client/src/components/Common/Webhook.vue
@@ -19,7 +19,7 @@ export default {
         const el = this.$refs["webhook"];
         new Webhooks.WebhookView({
             el,
-            type: "tool",
+            type: this.type,
             toolId: this.toolId,
         });
     },

From ac866bf206b9430c0d1d4e6e72c547281eac3ad5 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 13:35:21 -0400
Subject: [PATCH 010/452] Create parameters for tool request without jquery

---
 client/src/components/Tool/services.js | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index d8a78e45779..58d7194037a 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -1,4 +1,3 @@
-import $ from "jquery";
 import axios from "axios";
 import { rethrowSimple } from "utils/simple-error";
 import { getAppRoot } from "onload/loadConfig";
@@ -6,7 +5,7 @@ import { getGalaxyInstance } from "app";
 
 /** Tools data request helper **/
 export async function getTool(tool_id, tool_version, job_id) {
-    const galaxy = getGalaxyInstance();
+    const Galaxy = getGalaxyInstance();
     let url = "";
     let data = {};
 
@@ -15,15 +14,15 @@ export async function getTool(tool_id, tool_version, job_id) {
         url = `${getAppRoot()}api/jobs/${job_id}/build_for_rerun`;
     } else {
         url = `${getAppRoot()}api/tools/${tool_id}/build`;
-        data = $.extend({}, galaxy.params);
+        data = Object.assign({}, Galaxy.params);
         data["tool_id"] && delete data["tool_id"];
     }
     tool_version && (data["tool_version"] = tool_version);
 
     // attach data to request url
-    if (!$.isEmptyObject(data)) {
-        url += url.indexOf("?") == -1 ? "?" : "&";
-        url += $.param(data, true);
+    if (Object.entries(data).length != 0) {
+        const params = new URLSearchParams(data);
+        url = `${url}?${params.toString()}`;
     }
 
     // request tool data

From 076d1b155bf26ecf2e16ebe84753567d29ed7a55 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 13:41:52 -0400
Subject: [PATCH 011/452] Fix tool dropdown selector in selenium navigation

---
 client/src/components/Form/FormCardTool.vue | 3 +--
 lib/galaxy/selenium/navigation.yml          | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Form/FormCardTool.vue b/client/src/components/Form/FormCardTool.vue
index 228e4ce6a98..c19d2c272b0 100644
--- a/client/src/components/Form/FormCardTool.vue
+++ b/client/src/components/Form/FormCardTool.vue
@@ -6,12 +6,11 @@
                     <b-dropdown
                         no-caret
                         right
-                        id="options"
                         role="button"
                         title="Options"
                         variant="link"
                         aria-label="View all Options"
-                        class="float-right"
+                        class="tool-dropdown float-right"
                         size="sm"
                         v-b-tooltip.hover
                     >
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 547ad7d1a73..f22b87fb314 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -361,7 +361,7 @@ registration:
 
 tool_form:
   selectors:
-    options: '#options [data-toggle="dropdown"]'
+    options: '.tool-dropdown'
     execute: 'button#execute'
     parameter_div: 'div.ui-form-element[tour_id="${parameter}"]'
     parameter_textarea: 'div.ui-form-element[tour_id="${parameter}"] textarea'

From 21ff5238635c65115f0cd5f50431c9c888925049 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 13:53:01 -0400
Subject: [PATCH 012/452] Rename tool card for consistency

---
 .../components/{Form/FormCardTool.vue => Tool/ToolCard.vue} | 2 +-
 client/src/components/Tool/ToolForm.vue                     | 6 +++---
 client/src/components/Workflow/Editor/Forms/FormTool.vue    | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)
 rename client/src/components/{Form/FormCardTool.vue => Tool/ToolCard.vue} (99%)

diff --git a/client/src/components/Form/FormCardTool.vue b/client/src/components/Tool/ToolCard.vue
similarity index 99%
rename from client/src/components/Form/FormCardTool.vue
rename to client/src/components/Tool/ToolCard.vue
index c19d2c272b0..ac756f73f31 100644
--- a/client/src/components/Form/FormCardTool.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -106,7 +106,7 @@
 <script>
 import ariaAlert from "utils/ariaAlert";
 import { copyLink, downloadTool, openLink } from "./utilities";
-import FormMessage from "./FormMessage";
+import FormMessage from "components/Form/FormMessage";
 import ToolFooter from "components/Tool/ToolFooter";
 import ToolHelp from "components/Tool/ToolHelp";
 import Webhooks from "mvc/webhooks";
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index aa8d3f590c6..b3b8f92853c 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -15,7 +15,7 @@
                 <pre>{{ errorContentPretty }}</pre>
             </div>
             <ToolRecommendation v-if="showRecommendation" :tool-id="id" />
-            <FormCardTool
+            <ToolCard
                 v-if="showForm"
                 :id="formConfig.id"
                 :user="user"
@@ -68,7 +68,7 @@
                 <template v-slot:buttons>
                     <ButtonSpinner id="execute" title="Execute" @onClick="onExecute" :wait="showExecuting" />
                 </template>
-            </FormCardTool>
+            </ToolCard>
         </div>
     </CurrentUser>
 </template>
@@ -79,11 +79,11 @@ import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
 import { getTool, submitJob } from "./services";
 import { send } from "./utilities";
+import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
 import LoadingSpan from "components/LoadingSpan";
 import Form from "components/Form/Form";
-import FormCardTool from "components/Form/FormCardTool";
 import FormElement from "components/Form/FormElement";
 import ToolSuccess from "./ToolSuccess";
 import Webhook from "components/Common/Webhook";
diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index 45f81bdf325..54ff87df4ba 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -1,6 +1,6 @@
 <template>
     <CurrentUser v-slot="{ user }">
-        <FormCardTool
+        <ToolCard
             :id="node.config_form.id"
             :user="user"
             :version="node.config_form.version"
@@ -46,14 +46,14 @@
                     @onChangeOutputDatatype="onChangeOutputDatatype"
                 />
             </template>
-        </FormCardTool>
+        </ToolCard>
     </CurrentUser>
 </template>
 
 <script>
 import CurrentUser from "components/providers/CurrentUser";
 import Form from "components/Form/Form";
-import FormCardTool from "components/Form/FormCardTool";
+import ToolCard from "components/Tool/ToolCard";
 import FormSection from "./FormSection";
 import FormElement from "components/Form/FormElement";
 import { getModule } from "components/Workflow/Editor/modules/services";

From 201cbc8c890a077fc1e6a025d8bdc89bc9d6c8dc Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 14:49:12 -0400
Subject: [PATCH 013/452] Fix reference in workflow editor test case

---
 client/src/components/Workflow/Editor/Forms/FormTool.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index 54ff87df4ba..b5734bbae91 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -64,7 +64,7 @@ export default {
     components: {
         CurrentUser,
         Form,
-        FormCardTool,
+        ToolCard,
         FormElement,
         FormSection,
     },

From d093a705fcb86e0a9990663701581c5a32f3bcbb Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 15:40:40 -0400
Subject: [PATCH 014/452] Fix component import in Tool Form component

---
 client/src/components/Tool/ToolForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index b3b8f92853c..8f8e388ba57 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -94,7 +94,7 @@ export default {
         CurrentUser,
         LoadingSpan,
         Form,
-        FormCardTool,
+        ToolCard,
         FormElement,
         ToolSuccess,
         Webhook,

From a2025333baee69a74b6468892104bcd40915090e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 27 May 2021 16:43:41 -0400
Subject: [PATCH 015/452] Move utilities to tool card location

---
 client/src/components/Form/utilities.js | 14 --------------
 client/src/components/Tool/utilities.js | 14 ++++++++++++++
 lib/galaxy/selenium/navigation.yml      |  2 +-
 3 files changed, 15 insertions(+), 15 deletions(-)
 delete mode 100644 client/src/components/Form/utilities.js

diff --git a/client/src/components/Form/utilities.js b/client/src/components/Form/utilities.js
deleted file mode 100644
index d0d4e736c5b..00000000000
--- a/client/src/components/Form/utilities.js
+++ /dev/null
@@ -1,14 +0,0 @@
-import { getAppRoot } from "onload/loadConfig";
-import { copy } from "utils/clipboard";
-
-export function copyLink(toolId, message) {
-    copy(`${window.location.origin + getAppRoot()}root?tool_id=${toolId}`, message);
-}
-
-export function downloadTool(toolId) {
-    window.location.href = `${getAppRoot()}api/tools/${toolId}/download`;
-}
-
-export function openLink(url) {
-    window.open(url);
-}
diff --git a/client/src/components/Tool/utilities.js b/client/src/components/Tool/utilities.js
index db25be8f17e..0c8fb372b77 100644
--- a/client/src/components/Tool/utilities.js
+++ b/client/src/components/Tool/utilities.js
@@ -1,4 +1,18 @@
 import $ from "jquery";
+import { getAppRoot } from "onload/loadConfig";
+import { copy } from "utils/clipboard";
+
+export function copyLink(toolId, message) {
+    copy(`${window.location.origin + getAppRoot()}root?tool_id=${toolId}`, message);
+}
+
+export function downloadTool(toolId) {
+    window.location.href = `${getAppRoot()}api/tools/${toolId}/download`;
+}
+
+export function openLink(url) {
+    window.open(url);
+}
 
 export function send(options, job_def) {
     const $f = $("<form/>").attr({
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index f22b87fb314..81c9a798f51 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -369,7 +369,7 @@ tool_form:
     about: '.tool-footer'
 
   labels:
-    generate_tour: 'Generate Tour'
+    generate_tour: ' Generate Tour'
 
 workflows:
 

From 65f0335d96ebe684e42f8d81a1f13ba8fea34dd7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 28 May 2021 12:14:38 -0400
Subject: [PATCH 016/452] Unify dropdown item icon margins and used fixed icon
 width

---
 client/src/components/Tool/ToolCard.vue       | 28 +++++++++----------
 .../components/Workflow/Editor/Options.vue    | 12 ++++----
 client/src/style/scss/overrides.scss          |  5 ++++
 lib/galaxy/selenium/navigation.yml            |  2 +-
 4 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index ac756f73f31..aa43e3184da 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -17,19 +17,17 @@
                         <template v-slot:button-content>
                             <span class="fa fa-caret-down" />
                         </template>
-                        <b-dropdown-item @click="onCopyLink"> <span class="fa fa-chain" /> Copy Link </b-dropdown-item>
-                        <b-dropdown-item @click="onCopyId">
-                            <span class="fa fa-files-o" /> Copy Tool ID
-                        </b-dropdown-item>
-                        <b-dropdown-item v-if="showDownload" @click="onDownload">
-                            <span class="fa fa-download" /> Download
-                        </b-dropdown-item>
-                        <b-dropdown-item v-if="showLink" @click="onLink">
-                            <span class="fa fa-external-link" /> See in Tool Shed
-                        </b-dropdown-item>
-                        <b-dropdown-item v-for="w of webhookDetails" :key="w.title" @click="w.onclick">
-                            <span :class="w.icon" /> {{ w.title }}
-                        </b-dropdown-item>
+                        <b-dropdown-item @click="onCopyLink"> <icon class="fa-chain" />Copy Link</b-dropdown-item>
+                        <b-dropdown-item @click="onCopyId"><icon class="fa-files-o" />Copy Tool ID</b-dropdown-item>
+                        <b-dropdown-item v-if="showDownload" @click="onDownload"
+                            ><icon class="fa-download" />Download</b-dropdown-item
+                        >
+                        <b-dropdown-item v-if="showLink" @click="onLink"
+                            ><icon class="fa-external-link" />See in Tool Shed</b-dropdown-item
+                        >
+                        <b-dropdown-item v-for="w of webhookDetails" :key="w.title" @click="w.onclick"
+                            ><icon :class="w.icon" />{{ w.title }}</b-dropdown-item
+                        >
                     </b-dropdown>
                     <b-dropdown
                         v-if="showVersions"
@@ -47,8 +45,8 @@
                             <span class="fa fa-cubes" />
                         </template>
                         <b-dropdown-item v-for="v of availableVersions" :key="v" @click="$emit('onChangeVersion', v)">
-                            <span class="fa fa-cube" /> Switch to {{ v }}
-                        </b-dropdown-item>
+                            <icon class="fa-cube" />Switch to {{ v }}</b-dropdown-item
+                        >
                     </b-dropdown>
                     <b-button
                         v-if="showAddFavorite"
diff --git a/client/src/components/Workflow/Editor/Options.vue b/client/src/components/Workflow/Editor/Options.vue
index 552d1eda949..a4fd6d29edf 100644
--- a/client/src/components/Workflow/Editor/Options.vue
+++ b/client/src/components/Workflow/Editor/Options.vue
@@ -52,19 +52,17 @@
                 <span class="fa fa-cog" />
             </template>
             <b-dropdown-item href="#" @click="$emit('onSaveAs')"
-                ><span class="fa fa-floppy-o mr-1" />Save As...</b-dropdown-item
+                ><icon class="fa-floppy-o" />Save As...</b-dropdown-item
             >
             <b-dropdown-item href="#" @click="$emit('onLayout')"
-                ><span class="fa fa-align-left mr-1" />Auto Layout</b-dropdown-item
-            >
-            <b-dropdown-item href="#" @click="$emit('onLint')"
-                ><span class="fa fa-magic mr-1" />Best Practices</b-dropdown-item
+                ><icon class="fa-align-left" />Auto Layout</b-dropdown-item
             >
+            <b-dropdown-item href="#" @click="$emit('onLint')"><icon class="fa-magic" />Best Practices</b-dropdown-item>
             <b-dropdown-item href="#" @click="$emit('onUpgrade')"
-                ><span class="fa fa-recycle mr-1" />Upgrade Workflow</b-dropdown-item
+                ><icon class="fa-recycle" />Upgrade Workflow</b-dropdown-item
             >
             <b-dropdown-item href="#" @click="$emit('onDownload')"
-                ><span class="fa fa-download mr-1" />Download</b-dropdown-item
+                ><icon class="fa-download" />Download</b-dropdown-item
             >
         </b-dropdown>
         <b-button
diff --git a/client/src/style/scss/overrides.scss b/client/src/style/scss/overrides.scss
index 9593a7323db..6e6a9648dbb 100644
--- a/client/src/style/scss/overrides.scss
+++ b/client/src/style/scss/overrides.scss
@@ -35,6 +35,11 @@ pre {
 .dropdown-menu {
     .dropdown-item {
         text-decoration: none;
+        icon {
+            @extend .fa;
+            @extend .fa-fw;
+            @extend .mr-2;
+        }
     }
 }
 
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 81c9a798f51..f22b87fb314 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -369,7 +369,7 @@ tool_form:
     about: '.tool-footer'
 
   labels:
-    generate_tour: ' Generate Tour'
+    generate_tour: 'Generate Tour'
 
 workflows:
 

From 60dfd029c3fa5fee39352508d3e61ac9afd035c4 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 1 Jun 2021 12:02:57 -0400
Subject: [PATCH 017/452] Fix webhhook attachments to tool dropdown menu

---
 client/src/components/Tool/ToolCard.vue           | 14 +++++++-------
 client/src/components/Workflow/Editor/Options.vue | 12 +++++++-----
 client/src/style/scss/overrides.scss              |  3 +--
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index aa43e3184da..a1aa4b768ef 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -17,16 +17,16 @@
                         <template v-slot:button-content>
                             <span class="fa fa-caret-down" />
                         </template>
-                        <b-dropdown-item @click="onCopyLink"> <icon class="fa-chain" />Copy Link</b-dropdown-item>
-                        <b-dropdown-item @click="onCopyId"><icon class="fa-files-o" />Copy Tool ID</b-dropdown-item>
+                        <b-dropdown-item @click="onCopyLink"><span class="fa fa-chain" />Copy Link</b-dropdown-item>
+                        <b-dropdown-item @click="onCopyId"><span class="fa fa-files-o" />Copy Tool ID</b-dropdown-item>
                         <b-dropdown-item v-if="showDownload" @click="onDownload"
-                            ><icon class="fa-download" />Download</b-dropdown-item
+                            ><span class="fa fa-download" />Download</b-dropdown-item
                         >
                         <b-dropdown-item v-if="showLink" @click="onLink"
-                            ><icon class="fa-external-link" />See in Tool Shed</b-dropdown-item
+                            ><span class="fa fa-external-link" />See in Tool Shed</b-dropdown-item
                         >
                         <b-dropdown-item v-for="w of webhookDetails" :key="w.title" @click="w.onclick"
-                            ><icon :class="w.icon" />{{ w.title }}</b-dropdown-item
+                            ><span :class="w.icon" />{{ w.title }}</b-dropdown-item
                         >
                     </b-dropdown>
                     <b-dropdown
@@ -45,7 +45,7 @@
                             <span class="fa fa-cubes" />
                         </template>
                         <b-dropdown-item v-for="v of availableVersions" :key="v" @click="$emit('onChangeVersion', v)">
-                            <icon class="fa-cube" />Switch to {{ v }}</b-dropdown-item
+                            <i class="fa-cube" />Switch to {{ v }}</b-dropdown-item
                         >
                     </b-dropdown>
                     <b-button
@@ -204,7 +204,7 @@ export default {
                         this.webhookDetails.push({
                             icon: `fa ${webhook.config.icon}`,
                             title: webhook.config.title,
-                            onclick: function () {
+                            onclick: () => {
                                 const func = new Function("options", webhook.config.function);
                                 func(this.options);
                             },
diff --git a/client/src/components/Workflow/Editor/Options.vue b/client/src/components/Workflow/Editor/Options.vue
index a4fd6d29edf..da81a9dbc16 100644
--- a/client/src/components/Workflow/Editor/Options.vue
+++ b/client/src/components/Workflow/Editor/Options.vue
@@ -52,17 +52,19 @@
                 <span class="fa fa-cog" />
             </template>
             <b-dropdown-item href="#" @click="$emit('onSaveAs')"
-                ><icon class="fa-floppy-o" />Save As...</b-dropdown-item
+                ><span class="fa fa-floppy-o" />Save As...</b-dropdown-item
             >
             <b-dropdown-item href="#" @click="$emit('onLayout')"
-                ><icon class="fa-align-left" />Auto Layout</b-dropdown-item
+                ><span class="fa fa-align-left" />Auto Layout</b-dropdown-item
+            >
+            <b-dropdown-item href="#" @click="$emit('onLint')"
+                ><span class="fa fa-magic" />Best Practices</b-dropdown-item
             >
-            <b-dropdown-item href="#" @click="$emit('onLint')"><icon class="fa-magic" />Best Practices</b-dropdown-item>
             <b-dropdown-item href="#" @click="$emit('onUpgrade')"
-                ><icon class="fa-recycle" />Upgrade Workflow</b-dropdown-item
+                ><span class="fa fa-recycle" />Upgrade Workflow</b-dropdown-item
             >
             <b-dropdown-item href="#" @click="$emit('onDownload')"
-                ><icon class="fa-download" />Download</b-dropdown-item
+                ><span class="fa fa-download" />Download</b-dropdown-item
             >
         </b-dropdown>
         <b-button
diff --git a/client/src/style/scss/overrides.scss b/client/src/style/scss/overrides.scss
index 6e6a9648dbb..637584c2382 100644
--- a/client/src/style/scss/overrides.scss
+++ b/client/src/style/scss/overrides.scss
@@ -35,8 +35,7 @@ pre {
 .dropdown-menu {
     .dropdown-item {
         text-decoration: none;
-        icon {
-            @extend .fa;
+        .fa {
             @extend .fa-fw;
             @extend .mr-2;
         }

From 4fb49d6cb5a18de12e45502975bffb2ba2800986 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 2 Jun 2021 13:19:26 -0400
Subject: [PATCH 018/452] Add history listener to tool form component

---
 client/src/components/Tool/ToolForm.vue | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 8f8e388ba57..d0229e9e7e1 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -136,10 +136,17 @@ export default {
             jobDef: {},
             jobResponse: {},
             validationErrors: {},
+            currentVersion: this.version,
         };
     },
     created() {
         this.requestTool();
+        const Galaxy = getGalaxyInstance();
+        if (Galaxy.currHistoryPanel) {
+            Galaxy.currHistoryPanel.collection.on("change", () => {
+                this.requestTool();
+            });
+        }
     },
     computed: {
         toolName() {
@@ -204,8 +211,8 @@ export default {
             user.preferences["favorites"] = newFavorites;
         },
         requestTool(newVersion) {
-            const version = newVersion || this.version;
-            getTool(this.id, version, this.job_id).then((data) => {
+            this.currentVersion = newVersion || this.currentVersion;
+            getTool(this.id, currentVersion, this.job_id).then((data) => {
                 this.formConfig = data;
                 this.showLoading = false;
                 this.showForm = true;

From 7cdffe4b68bf9ae378a1a12945340b40467a6916 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 2 Jun 2021 20:54:32 -0400
Subject: [PATCH 019/452] Fix tool state update handling in tool form

---
 client/src/components/Form/Form.vue     | 10 ++++++----
 client/src/components/Tool/ToolForm.vue | 14 +++++++++++---
 client/src/components/Tool/services.js  | 15 +++++++++++++++
 client/src/mvc/form/form-view.js        |  4 ++--
 4 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index a9bc63db550..8563c483e22 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -64,8 +64,7 @@ export default {
         this.onRender();
     },
     beforeDestroy() {
-        const Galaxy = getGalaxyInstance();
-        this.form.stopListening(Galaxy.currHistoryPanel.collection);
+        this.form.off();
     },
     computed: {
         validation() {
@@ -128,13 +127,16 @@ export default {
                     initial_errors: this.initialErrors,
                     text_enable: this.textEnable,
                     text_disable: this.textDisable,
-                }).on("change", this.onChange);
+                    onchange: () => {
+                        this.onChange();
+                    },
+                });
                 this.onChange();
             });
         },
         parseUpdate(data) {
             this.$nextTick(() => {
-                this.form.update(data, false);
+                this.form.update(data);
             });
         },
         parseErrors(data) {
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index d0229e9e7e1..f93a7b2c0bd 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -36,6 +36,7 @@
                         :inputs="inputs"
                         :validation-errors="validationErrors"
                         @onChange="onChange"
+                        ref="form"
                     />
                     <FormElement
                         v-if="emailAllowed"
@@ -77,7 +78,7 @@
 import Scroller from "vue-scrollto";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
-import { getTool, submitJob } from "./services";
+import { getTool, submitJob, updateTool } from "./services";
 import { send } from "./utilities";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
@@ -144,7 +145,7 @@ export default {
         const Galaxy = getGalaxyInstance();
         if (Galaxy.currHistoryPanel) {
             Galaxy.currHistoryPanel.collection.on("change", () => {
-                this.requestTool();
+                this.onUpdate();
             });
         }
     },
@@ -194,6 +195,13 @@ export default {
     methods: {
         onChange(newData) {
             this.formData = newData;
+            this.onUpdate();
+        },
+        onUpdate() {
+            updateTool(this.id, this.currentVersion, this.formData).then((data) => {
+                const form = this.$refs["form"];
+                form.parseUpdate(data, true);
+            });
         },
         onChangeVersion(newVersion) {
             this.requestTool(newVersion);
@@ -212,7 +220,7 @@ export default {
         },
         requestTool(newVersion) {
             this.currentVersion = newVersion || this.currentVersion;
-            getTool(this.id, currentVersion, this.job_id).then((data) => {
+            getTool(this.id, this.currentVersion, this.job_id).then((data) => {
                 this.formConfig = data;
                 this.showLoading = false;
                 this.showForm = true;
diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index 58d7194037a..1c34dab0630 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -3,6 +3,21 @@ import { rethrowSimple } from "utils/simple-error";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
 
+export async function updateTool(tool_id, tool_version, inputs) {
+    const current_state = {
+        tool_id: tool_id,
+        tool_version: tool_version,
+        inputs: inputs,
+    };
+    const url = `${getAppRoot()}api/tools/${tool_id}/build`;
+    try {
+        const { data } = await axios.post(url, current_state);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}
+
 /** Tools data request helper **/
 export async function getTool(tool_id, tool_version, job_id) {
     const Galaxy = getGalaxyInstance();
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index e7b14bd92b1..78bd6dd1c6e 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -26,13 +26,13 @@ export default Backbone.View.extend({
     },
 
     /** Update available options */
-    update: function (new_model, triggerChange = true) {
+    update: function (new_model) {
         var self = this;
         this.data.matchModel(new_model, (node, input_id) => {
             var field = self.field_list[input_id];
             if (field.update) {
                 field.update(node);
-                triggerChange && field.trigger("change");
+                field.trigger("change");
                 const Galaxy = getGalaxyInstance();
                 Galaxy.emit.debug("form-view::update()", `Updating input: ${input_id}`);
             }

From 77db9ed2ac01aec22ed6424f3f51bc3dc28194c8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 2 Jun 2021 21:03:06 -0400
Subject: [PATCH 020/452] Add watcher to update tool state options for form
 components

---
 client/src/components/Form/Form.vue                      | 8 ++++++++
 client/src/components/Tool/ToolForm.vue                  | 5 ++---
 client/src/components/Workflow/Editor/Forms/FormTool.vue | 7 +++----
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index 8563c483e22..064f7b7872c 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -32,6 +32,10 @@ export default {
             type: Object,
             default: null,
         },
+        formConfig: {
+            type: Object,
+            default: null,
+        },
     },
     data() {
         return {
@@ -59,6 +63,10 @@ export default {
                 }
             }
         },
+        formConfig() {
+            this.parseUpdate(this.formConfig);
+            this.parseErrors(this.formConfig);
+        },
     },
     mounted() {
         this.onRender();
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index f93a7b2c0bd..a1a5a875029 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -35,8 +35,8 @@
                         :id="formConfig.id"
                         :inputs="inputs"
                         :validation-errors="validationErrors"
+                        :form-config="formConfig"
                         @onChange="onChange"
-                        ref="form"
                     />
                     <FormElement
                         v-if="emailAllowed"
@@ -199,8 +199,7 @@ export default {
         },
         onUpdate() {
             updateTool(this.id, this.currentVersion, this.formData).then((data) => {
-                const form = this.$refs["form"];
-                form.parseUpdate(data, true);
+                this.formConfig = data;
             });
         },
         onChangeVersion(newVersion) {
diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index b5734bbae91..27c73564b86 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -33,10 +33,10 @@
                     :id="id"
                     :inputs="inputs"
                     :initial-errors="true"
+                    :form-config="formConfig"
                     text-enable="Set in Advance"
                     text-disable="Set at Runtime"
                     @onChange="onChange"
-                    ref="form"
                 />
                 <FormSection
                     :id="id"
@@ -88,6 +88,7 @@ export default {
             sectionValues: {},
             messageVariant: "",
             messageText: "",
+            formConfig: {},
         };
     },
     computed: {
@@ -161,9 +162,7 @@ export default {
                 inputs: Object.assign({}, this.mainValues, this.sectionValues),
             }).then((data) => {
                 this.$emit("onSetData", this.node.id, data);
-                const form = this.$refs["form"];
-                form.parseUpdate(data.config_form);
-                form.parseErrors(data.config_form);
+                this.formConfig = data;
                 if (newVersion) {
                     const options = data.config_form;
                     this.messageVariant = "success";

From cd2ae5c01babaaa8deca56eb75a02c413a94964b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 2 Jun 2021 21:08:12 -0400
Subject: [PATCH 021/452] Fix style of version change dropdown

---
 client/src/components/Tool/ToolCard.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index a1aa4b768ef..0fafec27465 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -45,7 +45,7 @@
                             <span class="fa fa-cubes" />
                         </template>
                         <b-dropdown-item v-for="v of availableVersions" :key="v" @click="$emit('onChangeVersion', v)">
-                            <i class="fa-cube" />Switch to {{ v }}</b-dropdown-item
+                            <span class="fa fa-cube" />Switch to {{ v }}</b-dropdown-item
                         >
                     </b-dropdown>
                     <b-button

From 273fe60a2eb746a8c15fb20fee0ea399dbcea757 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 2 Jun 2021 21:20:15 -0400
Subject: [PATCH 022/452] Allow optional error highlighting on data change

---
 client/src/components/Form/Form.vue | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index 064f7b7872c..ddc7ff78f78 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -65,7 +65,9 @@ export default {
         },
         formConfig() {
             this.parseUpdate(this.formConfig);
-            this.parseErrors(this.formConfig);
+            if (this.initialErrors) {
+                this.parseErrors(this.formConfig);
+            }
         },
     },
     mounted() {

From c7b920e796f56fbfed88caa99e6e5c33c06a5fa9 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 2 Jun 2021 23:59:10 -0400
Subject: [PATCH 023/452] Remove unused reference to global galaxy object in
 form component

---
 client/src/components/Form/Form.vue | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index ddc7ff78f78..0a210b29860 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -3,7 +3,6 @@
 </template>
 
 <script>
-import { getGalaxyInstance } from "app";
 import Form from "mvc/form/form-view";
 
 export default {

From cbdca0b491f867d8e1ca56d113474ed1d8c3e597 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 4 Jun 2021 14:28:43 -0400
Subject: [PATCH 024/452] Fix potentially section key duplication, occurs
 during testing

---
 client/src/components/Panels/ToolBox.vue | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Panels/ToolBox.vue b/client/src/components/Panels/ToolBox.vue
index 73167474b32..fca9bd812ce 100644
--- a/client/src/components/Panels/ToolBox.vue
+++ b/client/src/components/Panels/ToolBox.vue
@@ -28,10 +28,10 @@
             <div class="toolMenuContainer">
                 <div class="toolMenu">
                     <tool-section
-                        v-for="section in sections"
+                        v-for="(section, key) in sections"
                         :category="section"
                         :query-filter="queryFilter"
-                        :key="section.id"
+                        :key="key"
                         @onClick="onOpen"
                     />
                 </div>

From baac74de6d26e9723ff6d5252911e3b2f350fa66 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 4 Jun 2021 15:25:48 -0400
Subject: [PATCH 025/452] Remove default bootstrap vue imports

---
 client/src/components/Common/ButtonSpinner.vue      | 3 ---
 client/src/components/Form/Elements/FormBoolean.vue | 5 -----
 2 files changed, 8 deletions(-)

diff --git a/client/src/components/Common/ButtonSpinner.vue b/client/src/components/Common/ButtonSpinner.vue
index c316d4005ed..a9f1d40109d 100644
--- a/client/src/components/Common/ButtonSpinner.vue
+++ b/client/src/components/Common/ButtonSpinner.vue
@@ -8,13 +8,10 @@
 </template>
 <script>
 import Vue from "vue";
-import BootstrapVue from "bootstrap-vue";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faCheck, faSpinner } from "@fortawesome/free-solid-svg-icons";
 
-Vue.use(BootstrapVue);
-
 library.add(faCheck);
 library.add(faSpinner);
 
diff --git a/client/src/components/Form/Elements/FormBoolean.vue b/client/src/components/Form/Elements/FormBoolean.vue
index 6e3b3366ee7..d135c5d7f82 100644
--- a/client/src/components/Form/Elements/FormBoolean.vue
+++ b/client/src/components/Form/Elements/FormBoolean.vue
@@ -3,11 +3,6 @@
 </template>
 
 <script>
-import Vue from "vue";
-import BootstrapVue from "bootstrap-vue";
-
-Vue.use(BootstrapVue);
-
 export default {
     props: {
         value: {

From 598c88a7056393e8d66ace46c5c35a8430e40fa3 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 4 Jun 2021 15:27:12 -0400
Subject: [PATCH 026/452] Remove duplicate input component files and tests

---
 .../components/Form/Elements/Input.test.js    | 38 -------------------
 client/src/components/Form/FormInput.test.js  | 38 -------------------
 client/src/components/Form/FormInput.vue      | 37 ------------------
 3 files changed, 113 deletions(-)
 delete mode 100644 client/src/components/Form/Elements/Input.test.js
 delete mode 100644 client/src/components/Form/FormInput.test.js
 delete mode 100644 client/src/components/Form/FormInput.vue

diff --git a/client/src/components/Form/Elements/Input.test.js b/client/src/components/Form/Elements/Input.test.js
deleted file mode 100644
index ebe24aa5960..00000000000
--- a/client/src/components/Form/Elements/Input.test.js
+++ /dev/null
@@ -1,38 +0,0 @@
-import { mount } from "@vue/test-utils";
-import { getLocalVue } from "jest/helpers";
-import FormInput from "./FormInput";
-
-jest.mock("app");
-
-const localVue = getLocalVue();
-
-describe("FormInput", () => {
-    let wrapper;
-
-    beforeEach(() => {
-        wrapper = mount(FormInput, {
-            propsData: {
-                id: "input",
-                value: "initial_value",
-            },
-            localVue,
-        });
-    });
-
-    it("check initial value and value change", async () => {
-        const input = wrapper.find("input");
-        expect(input.element.value).toBe("initial_value");
-        await input.setValue("new_value");
-        expect(input.element.value).toBe("new_value");
-        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
-    });
-
-    it("check switching to text area", async () => {
-        await wrapper.setProps({ area: true });
-        const input = wrapper.find("textarea");
-        expect(input.element.value).toBe("initial_value");
-        await input.setValue("new_value");
-        expect(input.element.value).toBe("new_value");
-        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
-    });
-});
diff --git a/client/src/components/Form/FormInput.test.js b/client/src/components/Form/FormInput.test.js
deleted file mode 100644
index ebe24aa5960..00000000000
--- a/client/src/components/Form/FormInput.test.js
+++ /dev/null
@@ -1,38 +0,0 @@
-import { mount } from "@vue/test-utils";
-import { getLocalVue } from "jest/helpers";
-import FormInput from "./FormInput";
-
-jest.mock("app");
-
-const localVue = getLocalVue();
-
-describe("FormInput", () => {
-    let wrapper;
-
-    beforeEach(() => {
-        wrapper = mount(FormInput, {
-            propsData: {
-                id: "input",
-                value: "initial_value",
-            },
-            localVue,
-        });
-    });
-
-    it("check initial value and value change", async () => {
-        const input = wrapper.find("input");
-        expect(input.element.value).toBe("initial_value");
-        await input.setValue("new_value");
-        expect(input.element.value).toBe("new_value");
-        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
-    });
-
-    it("check switching to text area", async () => {
-        await wrapper.setProps({ area: true });
-        const input = wrapper.find("textarea");
-        expect(input.element.value).toBe("initial_value");
-        await input.setValue("new_value");
-        expect(input.element.value).toBe("new_value");
-        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
-    });
-});
diff --git a/client/src/components/Form/FormInput.vue b/client/src/components/Form/FormInput.vue
deleted file mode 100644
index ed25eb4917b..00000000000
--- a/client/src/components/Form/FormInput.vue
+++ /dev/null
@@ -1,37 +0,0 @@
-<template>
-    <textarea v-if="area" class="ui-input" v-model="currentValue" @input="onChange" @change="onChange" />
-    <input v-else class="ui-input" v-model="currentValue" @input="onChange" @change="onChange" />
-</template>
-<script>
-export default {
-    props: {
-        id: {
-            type: String,
-            required: true,
-        },
-        value: {
-            type: String,
-            default: "",
-        },
-        area: {
-            type: Boolean,
-            default: false,
-        },
-    },
-    watch: {
-        value(newValue) {
-            this.currentValue = newValue;
-        },
-    },
-    data() {
-        return {
-            currentValue: this.value,
-        };
-    },
-    methods: {
-        onChange() {
-            this.$emit("onChange", this.currentValue);
-        },
-    },
-};
-</script>

From 857a7dbf274ab45851acad77fd5310c449abb763 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 4 Jun 2021 15:51:22 -0400
Subject: [PATCH 027/452] Remove default vue import from button component

---
 client/src/components/Common/ButtonSpinner.vue | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/components/Common/ButtonSpinner.vue b/client/src/components/Common/ButtonSpinner.vue
index a9f1d40109d..ed6ae7707c0 100644
--- a/client/src/components/Common/ButtonSpinner.vue
+++ b/client/src/components/Common/ButtonSpinner.vue
@@ -7,7 +7,6 @@
     </b-button>
 </template>
 <script>
-import Vue from "vue";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faCheck, faSpinner } from "@fortawesome/free-solid-svg-icons";

From 1efef4640c38d84b46cdeda5cd78f3eb271d1fb8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 4 Jun 2021 15:55:10 -0400
Subject: [PATCH 028/452] Fix masthead minor icon margins

---
 client/src/style/scss/base.scss | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/style/scss/base.scss b/client/src/style/scss/base.scss
index 49433ed6d73..f7b71f82d12 100644
--- a/client/src/style/scss/base.scss
+++ b/client/src/style/scss/base.scss
@@ -364,8 +364,8 @@ body {
                     .nav-note {
                         @extend .font-weight-bold;
                         @extend .position-absolute;
-                        left: 2.2rem;
-                        top: 2.2rem;
+                        left: 1.9rem;
+                        top: 1.9rem;
                         font-size: 0.6rem;
                     }
                 }

From 4f567ca475f2e1ad3df2ea47081202bde516eae7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 8 Jun 2021 14:48:13 -0400
Subject: [PATCH 029/452] Highlight one issue at a time in tool form, fix id
 parsing, remove legacy helpers

---
 client/src/components/Form/Form.vue     | 21 +++++++--------------
 client/src/components/Tool/ToolForm.vue |  4 ++--
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index 0a210b29860..5fadf091e78 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -51,6 +51,7 @@ export default {
                 const matchedErrors = this.form.data.matchResponse(this.validationErrors);
                 for (const input_id in matchedErrors) {
                     this.form.highlight(input_id, matchedErrors[input_id]);
+                    break;
                 }
             }
         },
@@ -63,10 +64,12 @@ export default {
             }
         },
         formConfig() {
-            this.parseUpdate(this.formConfig);
-            if (this.initialErrors) {
-                this.parseErrors(this.formConfig);
-            }
+            this.$nextTick(() => {
+                this.form.update(this.formConfig);
+                if (this.initialErrors) {
+                    this.form.errors(this.formConfig);
+                }
+            });
         },
     },
     mounted() {
@@ -143,16 +146,6 @@ export default {
                 this.onChange();
             });
         },
-        parseUpdate(data) {
-            this.$nextTick(() => {
-                this.form.update(data);
-            });
-        },
-        parseErrors(data) {
-            this.$nextTick(() => {
-                this.form.errors(data);
-            });
-        },
     },
 };
 </script>
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index a1a5a875029..6432369a917 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -14,7 +14,7 @@
                 </p>
                 <pre>{{ errorContentPretty }}</pre>
             </div>
-            <ToolRecommendation v-if="showRecommendation" :tool-id="id" />
+            <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
             <ToolCard
                 v-if="showForm"
                 :id="formConfig.id"
@@ -198,7 +198,7 @@ export default {
             this.onUpdate();
         },
         onUpdate() {
-            updateTool(this.id, this.currentVersion, this.formData).then((data) => {
+            updateTool(this.formConfig.id, this.currentVersion, this.formData).then((data) => {
                 this.formConfig = data;
             });
         },

From e1105576660e4c0c9327baf9ecaa3ec4e0db8a63 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 10 Jun 2021 16:44:54 -0400
Subject: [PATCH 030/452] Improve validation handling, in some cases the
 backend does not properly validate tool inputs

---
 client/src/components/Form/Form.vue     | 33 ++++++++++++++-----------
 client/src/components/Tool/ToolForm.vue | 13 +++++++++-
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/Form.vue
index 5fadf091e78..09d42bd6f16 100644
--- a/client/src/components/Form/Form.vue
+++ b/client/src/components/Form/Form.vue
@@ -27,6 +27,10 @@ export default {
             type: Boolean,
             default: false,
         },
+        validationScrollTo: {
+            type: Array,
+            default: null,
+        },
         validationErrors: {
             type: Object,
             default: null,
@@ -45,22 +49,17 @@ export default {
         id() {
             this.onRender();
         },
-        validationErrors() {
-            if (this.form) {
-                this.form.trigger("reset");
-                const matchedErrors = this.form.data.matchResponse(this.validationErrors);
-                for (const input_id in matchedErrors) {
-                    this.form.highlight(input_id, matchedErrors[input_id]);
-                    break;
-                }
-            }
+        validationScrollTo() {
+            this.onHighlight(this.validationScrollTo);
         },
         validation() {
-            if (this.form) {
-                this.form.trigger("reset");
-                if (this.validation) {
-                    this.form.highlight(this.validation[0], this.validation[1], true);
-                }
+            this.onHighlight(this.validation, true);
+            this.$emit("onValidation", this.validation);
+        },
+        validationErrors() {
+            const matchedErrors = Object.entries(this.form.data.matchResponse(this.validationErrors));
+            if (matchedErrors.length > 0) {
+                this.onHighlight(matchedErrors[0]);
             }
         },
         formConfig() {
@@ -146,6 +145,12 @@ export default {
                 this.onChange();
             });
         },
+        onHighlight(validation, silent = false) {
+            this.form.trigger("reset");
+            if (validation && validation.length == 2) {
+                this.form.highlight(validation[0], validation[1], silent);
+            }
+        },
     },
 };
 </script>
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 6432369a917..be0a1c50701 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -35,8 +35,10 @@
                         :id="formConfig.id"
                         :inputs="inputs"
                         :validation-errors="validationErrors"
+                        :validation-scroll-to="validationScrollTo"
                         :form-config="formConfig"
                         @onChange="onChange"
+                        @onValidation="onValidation"
                     />
                     <FormElement
                         v-if="emailAllowed"
@@ -136,7 +138,9 @@ export default {
             entryPoints: [],
             jobDef: {},
             jobResponse: {},
-            validationErrors: {},
+            validationInternal: null,
+            validationScrollTo: null,
+            validationErrors: null,
             currentVersion: this.version,
         };
     },
@@ -193,6 +197,9 @@ export default {
         },
     },
     methods: {
+        onValidation(validationInternal) {
+            this.validationInternal = validationInternal;
+        },
         onChange(newData) {
             this.formData = newData;
             this.onUpdate();
@@ -230,6 +237,10 @@ export default {
             });
         },
         onExecute() {
+            if (this.validationInternal) {
+                this.validationScrollTo = this.validationInternal.slice();
+                return;
+            }
             this.showExecuting = true;
             const options = this.formData;
             const Galaxy = getGalaxyInstance();

From 84b932e26f0986cabf8ed87dc3696c90e3270e39 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 10 Jun 2021 19:47:48 -0400
Subject: [PATCH 031/452] Fix checkbox event parsing, add test case

---
 .../Form/Elements/FormBoolean.test.js         | 34 +++++++++++++++++++
 .../components/Form/Elements/FormBoolean.vue  |  2 +-
 2 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 client/src/components/Form/Elements/FormBoolean.test.js

diff --git a/client/src/components/Form/Elements/FormBoolean.test.js b/client/src/components/Form/Elements/FormBoolean.test.js
new file mode 100644
index 00000000000..4198b752506
--- /dev/null
+++ b/client/src/components/Form/Elements/FormBoolean.test.js
@@ -0,0 +1,34 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import FormBoolean from "./FormBoolean";
+import Vue from "vue";
+
+jest.mock("app");
+
+const localVue = getLocalVue();
+
+describe("FormBoolean", () => {
+    let wrapper;
+
+    beforeEach(() => {
+        wrapper = mount(FormBoolean, {
+            propsData: {
+                value: false,
+            },
+            localVue,
+        });
+    });
+
+    it("check initial value and value change", async () => {
+        const input = wrapper.find("input");
+        expect(wrapper.vm.currentValue).toBe(false);
+        wrapper.setProps({ value: true });
+        await Vue.nextTick();
+        expect(wrapper.vm.currentValue).toBe(true);
+        input.trigger("click");
+        expect(input.element.checked).toBe(false);
+        input.trigger("click");
+        expect(input.element.checked).toBe(true);
+        expect(wrapper.emitted().onChange[0][0]).toBe(true);
+    });
+});
diff --git a/client/src/components/Form/Elements/FormBoolean.vue b/client/src/components/Form/Elements/FormBoolean.vue
index d135c5d7f82..0d18aaa0076 100644
--- a/client/src/components/Form/Elements/FormBoolean.vue
+++ b/client/src/components/Form/Elements/FormBoolean.vue
@@ -1,5 +1,5 @@
 <template>
-    <b-form-checkbox v-model="currentValue" size="lg" switch @onChange="onChange" />
+    <b-form-checkbox v-model="currentValue" size="lg" switch @change="onChange" />
 </template>
 
 <script>

From 33a34b172e960dcb7cae0a4cef9b1d85839ad00d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 10 Jun 2021 20:02:19 -0400
Subject: [PATCH 032/452] Add default form card component test case

---
 client/src/components/Form/FormCard.test.js | 31 +++++++++++++++++++++
 client/src/components/Form/FormCard.vue     |  5 ----
 2 files changed, 31 insertions(+), 5 deletions(-)
 create mode 100644 client/src/components/Form/FormCard.test.js

diff --git a/client/src/components/Form/FormCard.test.js b/client/src/components/Form/FormCard.test.js
new file mode 100644
index 00000000000..1d514d7c0c8
--- /dev/null
+++ b/client/src/components/Form/FormCard.test.js
@@ -0,0 +1,31 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import FormCard from "./FormCard";
+
+jest.mock("app");
+
+const localVue = getLocalVue();
+
+describe("FormCard", () => {
+    let wrapper;
+
+    beforeEach(() => {
+        wrapper = mount(FormCard, {
+            propsData: {
+                title: "title",
+                description: "description",
+                icon: "icon-class",
+            },
+            localVue,
+        });
+    });
+
+    it("check help text", async () => {
+        const title = wrapper.find(".portlet-title-text > b");
+        expect(title.text()).toBe("title");
+        const description = wrapper.find(".portlet-title-text > span");
+        expect(description.text()).toBe("description");
+        const icon = wrapper.find(".portlet-title-icon");
+        expect(icon.classes()).toContain("icon-class");
+    });
+});
diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 05443ec7630..6f145f5cd69 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -8,7 +8,6 @@
                 <i :class="['portlet-title-icon fa mr-1', icon]" style="display: inline"></i>
                 <span class="portlet-title-text">
                     <b itemprop="name">{{ title }}</b> <span itemprop="description">{{ description }}</span>
-                    {{ version }}
                 </span>
             </div>
         </div>
@@ -28,10 +27,6 @@ export default {
             type: String,
             required: false,
         },
-        version: {
-            type: String,
-            required: false,
-        },
         icon: {
             type: String,
             default: "fa-wrench",

From cd3094eeb3b05d8b537a974255a9b057a5edc1b6 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 11 Jun 2021 13:13:56 -0400
Subject: [PATCH 033/452] Add tool card rendering test case

---
 client/src/components/Form/FormCard.test.js   |  2 +-
 .../src/components/Form/FormElement.test.js   |  2 +-
 client/src/components/Tool/ToolCard.test.js   | 52 +++++++++++++++++++
 client/src/components/Tool/ToolFooter.vue     |  6 +--
 4 files changed, 57 insertions(+), 5 deletions(-)
 create mode 100644 client/src/components/Tool/ToolCard.test.js

diff --git a/client/src/components/Form/FormCard.test.js b/client/src/components/Form/FormCard.test.js
index 1d514d7c0c8..5603ebc8727 100644
--- a/client/src/components/Form/FormCard.test.js
+++ b/client/src/components/Form/FormCard.test.js
@@ -20,7 +20,7 @@ describe("FormCard", () => {
         });
     });
 
-    it("check help text", async () => {
+    it("check props", async () => {
         const title = wrapper.find(".portlet-title-text > b");
         expect(title.text()).toBe("title");
         const description = wrapper.find(".portlet-title-text > span");
diff --git a/client/src/components/Form/FormElement.test.js b/client/src/components/Form/FormElement.test.js
index 34f8c604732..90c4e295f01 100644
--- a/client/src/components/Form/FormElement.test.js
+++ b/client/src/components/Form/FormElement.test.js
@@ -22,7 +22,7 @@ describe("FormElement", () => {
         });
     });
 
-    it("check help text", async () => {
+    it("check props", async () => {
         const help = wrapper.find(".ui-form-info");
         expect(help.text()).toBe("help_text");
         const error = wrapper.find(".ui-form-error-text");
diff --git a/client/src/components/Tool/ToolCard.test.js b/client/src/components/Tool/ToolCard.test.js
new file mode 100644
index 00000000000..a4327befcbb
--- /dev/null
+++ b/client/src/components/Tool/ToolCard.test.js
@@ -0,0 +1,52 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import ToolCard from "./ToolCard";
+
+jest.mock("app");
+
+const localVue = getLocalVue();
+
+describe("ToolCard", () => {
+    let wrapper;
+
+    beforeEach(() => {
+        wrapper = mount(ToolCard, {
+            propsData: {
+                id: "identifier",
+                user: {
+                    id: "user.id",
+                    email: "user.email",
+                    is_admin: true,
+                    preferences: {},
+                },
+                version: "version",
+                title: "title",
+                description: "description",
+                sustainVersion: false,
+                options: {
+                    id: "options.id",
+                    versions: [],
+                    sharable_url: "options.sharable_url",
+                    help: "options.help",
+                    citations: null,
+                },
+                messageText: "messageText",
+                messageVariant: "warning",
+            },
+            localVue,
+        });
+    });
+
+    it("check props", async () => {
+        const title = wrapper.find(".portlet-title-text > b");
+        expect(title.text()).toBe("title");
+        const description = wrapper.find(".portlet-title-text > span");
+        expect(description.text()).toBe("description");
+        const icon = wrapper.find(".portlet-title-icon");
+        expect(icon.classes()).toContain("fa-wrench");
+        const dropdownHeader = wrapper.find(".tool-dropdown");
+        expect(dropdownHeader.attributes("title")).toBe("Options");
+        const dropdownItems = wrapper.findAll(".dropdown-item");
+        expect(dropdownItems.length).toBe(4);
+    });
+});
diff --git a/client/src/components/Tool/ToolFooter.vue b/client/src/components/Tool/ToolFooter.vue
index 22d097f2704..991b9788be3 100644
--- a/client/src/components/Tool/ToolFooter.vue
+++ b/client/src/components/Tool/ToolFooter.vue
@@ -76,9 +76,9 @@ import { faQuestion, faCopy, faAngleDoubleDown, faAngleDoubleUp } from "@fortawe
 library.add(faQuestion, faCopy, faAngleDoubleDown, faAngleDoubleUp);
 
 import { getCitations } from "components/Citation/services";
-import Citation from "components/Citation/Citation.vue";
-import License from "components/License/License.vue";
-import Creators from "components/SchemaOrg/Creators.vue";
+import Citation from "components/Citation/Citation";
+import License from "components/License/License";
+import Creators from "components/SchemaOrg/Creators";
 import { copy } from "utils/clipboard";
 
 export default {

From 7257de1722318226e7e621054e7009eed5400285 Mon Sep 17 00:00:00 2001
From: Alexandru Mahmoud <almahmoud@jhu.edu>
Date: Mon, 14 Jun 2021 16:57:02 -0400
Subject: [PATCH 034/452] Temporary fix for keycloak OIDC

---
 lib/galaxy/authnz/custos_authnz.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py
index f2a22191dd3..e5eef15b196 100644
--- a/lib/galaxy/authnz/custos_authnz.py
+++ b/lib/galaxy/authnz/custos_authnz.py
@@ -87,6 +87,7 @@ class CustosAuthnz(IdentityProvider):
         else:
             userinfo = self._get_userinfo(oauth2_session)
         email = userinfo['email']
+        username = userinfo.get('preferred_username', self._generate_username(trans, email))
         user_id = userinfo['sub']
 
         # Create or update custos_authnz_token record
@@ -110,9 +111,13 @@ class CustosAuthnz(IdentityProvider):
                         message = "There already exists a user with email %s.  To associate this external login, you must first be logged in as that existing account." % email
                         log.exception(message)
                         raise exceptions.AuthenticationFailed(message)
-                else:
-                    login_redirect_url = login_redirect_url + 'root/login?confirm=true&custos_token=' + json.dumps(token)
+                elif self.config['provider'] == 'custos':
+                    login_redirect_url = f"{login_redirect_url}root/login?confirm=true&custos_token={json.dumps(token)}"
                     return login_redirect_url, None
+                else:
+                    user = trans.app.user_manager.create(email=email, username=username)
+                    if trans.app.config.user_activation_on:
+                        trans.app.user_manager.send_activation_email(trans, email, username)
 
             custos_authnz_token = CustosAuthnzToken(user=user,
                                    external_user_id=user_id,
@@ -130,7 +135,7 @@ class CustosAuthnz(IdentityProvider):
             custos_authnz_token.refresh_expiration_time = refresh_expiration_time
         trans.sa_session.add(custos_authnz_token)
         trans.sa_session.flush()
-        return login_redirect_url, custos_authnz_token.user
+        return "/", custos_authnz_token.user
 
     def create_user(self, token, trans, login_redirect_url):
         token_dict = json.loads(token)

From 3a6271cbc56c3e79e60a8c5d110ba0432a090aaa Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 19:41:32 -0400
Subject: [PATCH 035/452] Fix typo in title

---
 client/src/components/Tool/ToolCard.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index 0fafec27465..809bc1eb122 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -77,7 +77,7 @@
                     <i class="portlet-title-icon fa mr-1 fa-wrench" style="display: inline"></i>
                     <span class="portlet-title-text">
                         <b itemprop="name">{{ title }}</b> <span itemprop="description">{{ description }}</span> (Galaxy
-                        Verson {{ version }})
+                        Version {{ version }})
                     </span>
                 </div>
             </div>

From df654abfd726dd167578617348d03fbcd7b12a2f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 24 Jun 2021 21:14:01 -0400
Subject: [PATCH 036/452] Use config and user provider in tool form component,
 add history id to tool request

---
 client/src/components/Tool/ToolForm.vue | 181 +++++++++++++-----------
 client/src/components/Tool/services.js  |   3 +-
 2 files changed, 98 insertions(+), 86 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index be0a1c50701..ecd502f18da 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -1,79 +1,82 @@
 <template>
-    <CurrentUser v-slot="{ user }">
-        <div>
-            <LoadingSpan v-if="showLoading" message="Loading Tool" />
-            <div v-if="showEntryPoints">
-                <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
+    <ConfigProvider v-slot="{ config }">
+        <CurrentUser v-slot="{ user }">
+            <div>
+                <LoadingSpan v-if="showLoading" message="Loading Tool" />
+                <div v-if="showEntryPoints">
+                    <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
+                </div>
+                <ToolSuccess v-if="showSuccess" :job-def="jobDef" :job-response="jobResponse" :tool-name="toolName" />
+                <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
+                <div v-if="showError" class="errormessagelarge">
+                    <p>
+                        The server could not complete the request. Please contact the Galaxy Team if this error
+                        persists.
+                        {{ errorTitle | l }}
+                    </p>
+                    <pre>{{ errorContentPretty }}</pre>
+                </div>
+                <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
+                <ToolCard
+                    v-if="showForm"
+                    :id="formConfig.id"
+                    :user="user"
+                    :version="formConfig.version"
+                    :title="formConfig.name"
+                    :description="formConfig.description"
+                    :options="formConfig"
+                    :message-text="messageText"
+                    :message-variant="messageVariant"
+                    @onChangeVersion="onChangeVersion"
+                    @onUpdateFavorites="onUpdateFavorites"
+                    itemscope="itemscope"
+                    itemtype="https://schema.org/CreativeWork"
+                >
+                    <template v-slot:body>
+                        <Form
+                            :id="formConfig.id"
+                            :inputs="inputs"
+                            :validation-errors="validationErrors"
+                            :validation-scroll-to="validationScrollTo"
+                            :form-config="formConfig"
+                            @onChange="onChange"
+                            @onValidation="onValidation"
+                        />
+                        <FormElement
+                            v-if="emailAllowed(config, user)"
+                            id="send_email_notification"
+                            title="Email notification"
+                            help="Send an email notification when the job completes."
+                            type="boolean"
+                            value="false"
+                            @onChange="onChangeEmail"
+                        />
+                        <FormElement
+                            v-if="remapAllowed"
+                            id="rerun_remap_job_id"
+                            :title="remapTitle"
+                            :help="remapHelp"
+                            type="boolean"
+                            value="false"
+                            @onChange="onChangeRemap"
+                        />
+                        <FormElement
+                            v-if="reuseAllowed(user)"
+                            id="use_cached_job"
+                            title="Attempt to re-use jobs with identical parameters?"
+                            help="This may skip executing jobs that you have already run."
+                            type="boolean"
+                            value="false"
+                            @onChange="onChangeReuse"
+                        />
+                    </template>
+                    <template v-slot:buttons>
+                        <ButtonSpinner id="execute" title="Execute" @onClick="onExecute" :wait="showExecuting" />
+                    </template>
+                </ToolCard>
             </div>
-            <ToolSuccess v-if="showSuccess" :job-def="jobDef" :job-response="jobResponse" :tool-name="toolName" />
-            <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
-            <div v-if="showError" class="errormessagelarge">
-                <p>
-                    The server could not complete the request. Please contact the Galaxy Team if this error persists.
-                    {{ errorTitle | l }}
-                </p>
-                <pre>{{ errorContentPretty }}</pre>
-            </div>
-            <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
-            <ToolCard
-                v-if="showForm"
-                :id="formConfig.id"
-                :user="user"
-                :version="formConfig.version"
-                :title="formConfig.name"
-                :description="formConfig.description"
-                :options="formConfig"
-                :message-text="messageText"
-                :message-variant="messageVariant"
-                @onChangeVersion="onChangeVersion"
-                @onUpdateFavorites="onUpdateFavorites"
-                itemscope="itemscope"
-                itemtype="https://schema.org/CreativeWork"
-            >
-                <template v-slot:body>
-                    <Form
-                        :id="formConfig.id"
-                        :inputs="inputs"
-                        :validation-errors="validationErrors"
-                        :validation-scroll-to="validationScrollTo"
-                        :form-config="formConfig"
-                        @onChange="onChange"
-                        @onValidation="onValidation"
-                    />
-                    <FormElement
-                        v-if="emailAllowed"
-                        id="send_email_notification"
-                        title="Email notification"
-                        help="Send an email notification when the job completes."
-                        type="boolean"
-                        value="false"
-                        @onChange="onChangeEmail"
-                    />
-                    <FormElement
-                        v-if="remapAllowed"
-                        id="rerun_remap_job_id"
-                        :title="remapTitle"
-                        :help="remapHelp"
-                        type="boolean"
-                        value="false"
-                        @onChange="onChangeRemap"
-                    />
-                    <FormElement
-                        v-if="reuseAllowed"
-                        id="use_cached_job"
-                        title="Attempt to re-use jobs with identical parameters?"
-                        help="This may skip executing jobs that you have already run."
-                        type="boolean"
-                        value="false"
-                        @onChange="onChangeReuse"
-                    />
-                </template>
-                <template v-slot:buttons>
-                    <ButtonSpinner id="execute" title="Execute" @onClick="onExecute" :wait="showExecuting" />
-                </template>
-            </ToolCard>
-        </div>
-    </CurrentUser>
+        </CurrentUser>
+    </ConfigProvider>
 </template>
 
 <script>
@@ -85,6 +88,7 @@ import { send } from "./utilities";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
+import ConfigProvider from "components/providers/ConfigProvider";
 import LoadingSpan from "components/LoadingSpan";
 import Form from "components/Form/Form";
 import FormElement from "components/Form/FormElement";
@@ -95,6 +99,7 @@ export default {
     components: {
         ButtonSpinner,
         CurrentUser,
+        ConfigProvider,
         LoadingSpan,
         Form,
         ToolCard,
@@ -115,6 +120,10 @@ export default {
             type: String,
             default: null,
         },
+        history_id: {
+            type: String,
+            default: null,
+        },
     },
     data() {
         return {
@@ -160,10 +169,6 @@ export default {
         errorContentPretty() {
             return JSON.stringify(this.errorContent, null, 4);
         },
-        emailAllowed() {
-            const Galaxy = getGalaxyInstance();
-            return Galaxy.config.server_mail_configured && !Galaxy.user.isAnonymous();
-        },
         inputs() {
             return this.formConfig.inputs;
         },
@@ -184,10 +189,18 @@ export default {
                 return "The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.";
             }
         },
-        reuseAllowed() {
-            const Galaxy = getGalaxyInstance();
-            if (Galaxy.user.attributes.preferences && "extra_user_preferences" in Galaxy.user.attributes.preferences) {
-                const extra_user_preferences = JSON.parse(Galaxy.user.attributes.preferences.extra_user_preferences);
+    },
+    methods: {
+        emailAllowed(config, user) {
+            return config.server_mail_configured && !user.isAnonymous();
+        },
+        reuseAllowed(user) {
+            if (
+                user.attributes &&
+                user.attributes.preferences &&
+                "extra_user_preferences" in user.attributes.preferences
+            ) {
+                const extra_user_preferences = JSON.parse(user.attributes.preferences.extra_user_preferences);
                 const keyCached = "use_cached_job|use_cached_job_checkbox";
                 const hasCachedJobs = keyCached in extra_user_preferences;
                 return hasCachedJobs ? extra_user_preferences[keyCached] : false;
@@ -195,8 +208,6 @@ export default {
                 return false;
             }
         },
-    },
-    methods: {
         onValidation(validationInternal) {
             this.validationInternal = validationInternal;
         },
@@ -226,7 +237,7 @@ export default {
         },
         requestTool(newVersion) {
             this.currentVersion = newVersion || this.currentVersion;
-            getTool(this.id, this.currentVersion, this.job_id).then((data) => {
+            getTool(this.id, this.currentVersion, this.job_id, this.history_id).then((data) => {
                 this.formConfig = data;
                 this.showLoading = false;
                 this.showForm = true;
diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index 1c34dab0630..09243581fd5 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -19,7 +19,7 @@ export async function updateTool(tool_id, tool_version, inputs) {
 }
 
 /** Tools data request helper **/
-export async function getTool(tool_id, tool_version, job_id) {
+export async function getTool(tool_id, tool_version, job_id, history_id) {
     const Galaxy = getGalaxyInstance();
     let url = "";
     let data = {};
@@ -32,6 +32,7 @@ export async function getTool(tool_id, tool_version, job_id) {
         data = Object.assign({}, Galaxy.params);
         data["tool_id"] && delete data["tool_id"];
     }
+    history_id && (data["history_id"] = history_id);
     tool_version && (data["tool_version"] = tool_version);
 
     // attach data to request url

From ce7af4cc3b063abe687cec31ce041cb040db4710 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 24 Jun 2021 21:16:15 -0400
Subject: [PATCH 037/452] Use updateToolFormData and getToolFormData as names
 in tool form services

---
 client/src/components/Tool/ToolForm.vue | 6 +++---
 client/src/components/Tool/services.js  | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index ecd502f18da..ee8638bffa8 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -83,7 +83,7 @@
 import Scroller from "vue-scrollto";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
-import { getTool, submitJob, updateTool } from "./services";
+import { getToolFormData, updateToolFormData, submitJob } from "./services";
 import { send } from "./utilities";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
@@ -216,7 +216,7 @@ export default {
             this.onUpdate();
         },
         onUpdate() {
-            updateTool(this.formConfig.id, this.currentVersion, this.formData).then((data) => {
+            updateToolFormData(this.formConfig.id, this.currentVersion, this.formData).then((data) => {
                 this.formConfig = data;
             });
         },
@@ -237,7 +237,7 @@ export default {
         },
         requestTool(newVersion) {
             this.currentVersion = newVersion || this.currentVersion;
-            getTool(this.id, this.currentVersion, this.job_id, this.history_id).then((data) => {
+            getToolFormData(this.id, this.currentVersion, this.job_id, this.history_id).then((data) => {
                 this.formConfig = data;
                 this.showLoading = false;
                 this.showForm = true;
diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index 09243581fd5..ca718530fe6 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -3,7 +3,7 @@ import { rethrowSimple } from "utils/simple-error";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
 
-export async function updateTool(tool_id, tool_version, inputs) {
+export async function updateToolFormData(tool_id, tool_version, inputs) {
     const current_state = {
         tool_id: tool_id,
         tool_version: tool_version,
@@ -19,7 +19,7 @@ export async function updateTool(tool_id, tool_version, inputs) {
 }
 
 /** Tools data request helper **/
-export async function getTool(tool_id, tool_version, job_id, history_id) {
+export async function getToolFormData(tool_id, tool_version, job_id, history_id) {
     const Galaxy = getGalaxyInstance();
     let url = "";
     let data = {};

From 7897b1f4e7a7b616a13a8f34d7d16beacac733d4 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 24 Jun 2021 21:30:11 -0400
Subject: [PATCH 038/452] Remove unused jest app handling in tests

---
 client/src/components/Form/Elements/FormBoolean.test.js | 2 --
 client/src/components/Form/FormCard.test.js             | 2 --
 client/src/components/Tool/ToolCard.test.js             | 2 --
 3 files changed, 6 deletions(-)

diff --git a/client/src/components/Form/Elements/FormBoolean.test.js b/client/src/components/Form/Elements/FormBoolean.test.js
index 4198b752506..92f83734326 100644
--- a/client/src/components/Form/Elements/FormBoolean.test.js
+++ b/client/src/components/Form/Elements/FormBoolean.test.js
@@ -3,8 +3,6 @@ import { getLocalVue } from "jest/helpers";
 import FormBoolean from "./FormBoolean";
 import Vue from "vue";
 
-jest.mock("app");
-
 const localVue = getLocalVue();
 
 describe("FormBoolean", () => {
diff --git a/client/src/components/Form/FormCard.test.js b/client/src/components/Form/FormCard.test.js
index 5603ebc8727..303e4f7c5cd 100644
--- a/client/src/components/Form/FormCard.test.js
+++ b/client/src/components/Form/FormCard.test.js
@@ -2,8 +2,6 @@ import { mount } from "@vue/test-utils";
 import { getLocalVue } from "jest/helpers";
 import FormCard from "./FormCard";
 
-jest.mock("app");
-
 const localVue = getLocalVue();
 
 describe("FormCard", () => {
diff --git a/client/src/components/Tool/ToolCard.test.js b/client/src/components/Tool/ToolCard.test.js
index a4327befcbb..6e3638b9530 100644
--- a/client/src/components/Tool/ToolCard.test.js
+++ b/client/src/components/Tool/ToolCard.test.js
@@ -2,8 +2,6 @@ import { mount } from "@vue/test-utils";
 import { getLocalVue } from "jest/helpers";
 import ToolCard from "./ToolCard";
 
-jest.mock("app");
-
 const localVue = getLocalVue();
 
 describe("ToolCard", () => {

From 67727619d4d68a6a60e2360d4641c97833d77d4b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 24 Jun 2021 21:36:21 -0400
Subject: [PATCH 039/452] Set default value for description in form card

---
 client/src/components/Form/FormCard.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 6f145f5cd69..1562b2e8161 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -25,7 +25,7 @@ export default {
         },
         description: {
             type: String,
-            required: false,
+            default: null,
         },
         icon: {
             type: String,

From b6a14f8bdb1f77586057a6fdbe9dac1a9cecca74 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 24 Jun 2021 21:42:18 -0400
Subject: [PATCH 040/452] Use localization directive for tool form dropdown
 options

---
 client/src/components/Tool/ToolCard.vue | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index 809bc1eb122..850cfd2050a 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -17,16 +17,22 @@
                         <template v-slot:button-content>
                             <span class="fa fa-caret-down" />
                         </template>
-                        <b-dropdown-item @click="onCopyLink"><span class="fa fa-chain" />Copy Link</b-dropdown-item>
-                        <b-dropdown-item @click="onCopyId"><span class="fa fa-files-o" />Copy Tool ID</b-dropdown-item>
+                        <b-dropdown-item @click="onCopyLink"
+                            ><span class="fa fa-chain" /><span v-localize>Copy Link</span>
+                        </b-dropdown-item>
+                        <b-dropdown-item @click="onCopyId"
+                            ><span class="fa fa-files-o" /><span v-localize>Copy Tool ID</span>
+                        </b-dropdown-item>
                         <b-dropdown-item v-if="showDownload" @click="onDownload"
-                            ><span class="fa fa-download" />Download</b-dropdown-item
-                        >
+                            ><span class="fa fa-download" /><span v-localize>Download</span>
+                        </b-dropdown-item>
                         <b-dropdown-item v-if="showLink" @click="onLink"
-                            ><span class="fa fa-external-link" />See in Tool Shed</b-dropdown-item
+                            ><span class="fa fa-external-link" /><span v-localize
+                                >See in Tool Shed</span
+                            ></b-dropdown-item
                         >
                         <b-dropdown-item v-for="w of webhookDetails" :key="w.title" @click="w.onclick"
-                            ><span :class="w.icon" />{{ w.title }}</b-dropdown-item
+                            ><span :class="w.icon" />{{ w.title | l }}</b-dropdown-item
                         >
                     </b-dropdown>
                     <b-dropdown
@@ -45,7 +51,7 @@
                             <span class="fa fa-cubes" />
                         </template>
                         <b-dropdown-item v-for="v of availableVersions" :key="v" @click="$emit('onChangeVersion', v)">
-                            <span class="fa fa-cube" />Switch to {{ v }}</b-dropdown-item
+                            <span class="fa fa-cube" /><span v-localize>Switch to </span>{{ v }}</b-dropdown-item
                         >
                     </b-dropdown>
                     <b-button

From 149eb84cfad1442717d77f6a237d5082936621b6 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 25 Jun 2021 13:11:31 -0400
Subject: [PATCH 041/452] Rename Form component to FormDisplay component

---
 client/src/components/Form/{Form.vue => FormDisplay.vue}    | 0
 client/src/components/Markdown/MarkdownVisualization.vue    | 6 +++---
 client/src/components/Tool/ToolForm.vue                     | 6 +++---
 client/src/components/Workflow/Editor/Forms/FormDefault.vue | 6 +++---
 client/src/components/Workflow/Editor/Forms/FormSection.vue | 4 ++--
 client/src/components/Workflow/Editor/Forms/FormTool.vue    | 6 +++---
 6 files changed, 14 insertions(+), 14 deletions(-)
 rename client/src/components/Form/{Form.vue => FormDisplay.vue} (100%)

diff --git a/client/src/components/Form/Form.vue b/client/src/components/Form/FormDisplay.vue
similarity index 100%
rename from client/src/components/Form/Form.vue
rename to client/src/components/Form/FormDisplay.vue
diff --git a/client/src/components/Markdown/MarkdownVisualization.vue b/client/src/components/Markdown/MarkdownVisualization.vue
index b9c5bcce32d..358508fa1bd 100644
--- a/client/src/components/Markdown/MarkdownVisualization.vue
+++ b/client/src/components/Markdown/MarkdownVisualization.vue
@@ -19,7 +19,7 @@
             @ok="onOk"
             @cancel="onCancel"
         >
-            <Form :inputs="formInputs" @onChange="onChange" />
+            <FormDisplay :inputs="formInputs" @onChange="onChange" />
         </b-modal>
     </span>
 </template>
@@ -29,7 +29,7 @@ import Vue from "vue";
 import BootstrapVue from "bootstrap-vue";
 import MarkdownSelector from "./MarkdownSelector";
 import DataDialog from "components/DataDialog/DataDialog";
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 
 Vue.use(BootstrapVue);
 
@@ -37,7 +37,7 @@ export default {
     components: {
         MarkdownSelector,
         DataDialog,
-        Form,
+        FormDisplay,
     },
     props: {
         argumentName: {
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index ee8638bffa8..61074740480 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -33,7 +33,7 @@
                     itemtype="https://schema.org/CreativeWork"
                 >
                     <template v-slot:body>
-                        <Form
+                        <FormDisplay
                             :id="formConfig.id"
                             :inputs="inputs"
                             :validation-errors="validationErrors"
@@ -90,7 +90,7 @@ import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
 import ConfigProvider from "components/providers/ConfigProvider";
 import LoadingSpan from "components/LoadingSpan";
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import FormElement from "components/Form/FormElement";
 import ToolSuccess from "./ToolSuccess";
 import Webhook from "components/Common/Webhook";
@@ -101,7 +101,7 @@ export default {
         CurrentUser,
         ConfigProvider,
         LoadingSpan,
-        Form,
+        FormDisplay,
         ToolCard,
         FormElement,
         ToolSuccess,
diff --git a/client/src/components/Workflow/Editor/Forms/FormDefault.vue b/client/src/components/Workflow/Editor/Forms/FormDefault.vue
index fa9d6be0061..ac46cad7a6c 100644
--- a/client/src/components/Workflow/Editor/Forms/FormDefault.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormDefault.vue
@@ -44,13 +44,13 @@
                 help="Add an annotation or notes to this step. Annotations are available when a workflow is viewed."
                 @onChange="onAnnotation"
             />
-            <Form :id="id" :inputs="inputs" @onChange="onChange" />
+            <FormDisplay :id="id" :inputs="inputs" @onChange="onChange" />
         </template>
     </FormCard>
 </template>
 
 <script>
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";
 import FormElement from "components/Form/FormElement";
 import FormMessage from "components/Form/FormMessage";
@@ -60,7 +60,7 @@ import WorkflowIcons from "components/Workflow/icons";
 
 export default {
     components: {
-        Form,
+        FormDisplay,
         FormCard,
         FormElement,
         FormMessage,
diff --git a/client/src/components/Workflow/Editor/Forms/FormSection.vue b/client/src/components/Workflow/Editor/Forms/FormSection.vue
index 8c5a082427c..0d2b71e5878 100644
--- a/client/src/components/Workflow/Editor/Forms/FormSection.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormSection.vue
@@ -1,9 +1,9 @@
 <template>
-    <Form :id="id" :inputs="inputs" ref="form" @onChange="onChange" />
+    <FormDisplay :id="id" :inputs="inputs" ref="form" @onChange="onChange" />
 </template>
 
 <script>
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import makeSection from "./makeSection";
 
 export default {
diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index 27c73564b86..4f82fec5aed 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -29,7 +29,7 @@
                     help="Add an annotation or notes to this step. Annotations are available when a workflow is viewed."
                     @onChange="onAnnotation"
                 />
-                <Form
+                <FormDisplay
                     :id="id"
                     :inputs="inputs"
                     :initial-errors="true"
@@ -52,7 +52,7 @@
 
 <script>
 import CurrentUser from "components/providers/CurrentUser";
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import ToolCard from "components/Tool/ToolCard";
 import FormSection from "./FormSection";
 import FormElement from "components/Form/FormElement";
@@ -63,7 +63,7 @@ import Utils from "utils/utils";
 export default {
     components: {
         CurrentUser,
-        Form,
+        FormDisplay,
         ToolCard,
         FormElement,
         FormSection,

From c105214d16ee8bfdcf9c7c4666285de9c2d300f7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 25 Jun 2021 14:37:34 -0400
Subject: [PATCH 042/452] Fix imported FormDisplay component name

---
 client/src/components/Workflow/Editor/Forms/FormSection.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Editor/Forms/FormSection.vue b/client/src/components/Workflow/Editor/Forms/FormSection.vue
index 0d2b71e5878..c49fe7768f6 100644
--- a/client/src/components/Workflow/Editor/Forms/FormSection.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormSection.vue
@@ -8,7 +8,7 @@ import makeSection from "./makeSection";
 
 export default {
     components: {
-        Form,
+        FormDisplay,
     },
     props: {
         id: {

From 2bd585dcf9dc1605ff646824ea9500d83d6e486f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 28 Jun 2021 10:45:19 -0400
Subject: [PATCH 043/452] Add optional tooltip to tool execution button

---
 client/src/components/Common/ButtonSpinner.vue |  6 +++++-
 client/src/components/Tool/ToolForm.vue        | 11 ++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Common/ButtonSpinner.vue b/client/src/components/Common/ButtonSpinner.vue
index ed6ae7707c0..56531098ef6 100644
--- a/client/src/components/Common/ButtonSpinner.vue
+++ b/client/src/components/Common/ButtonSpinner.vue
@@ -2,7 +2,7 @@
     <b-button v-if="wait" variant="info" disabled>
         <font-awesome-icon icon="spinner" class="mr-2" spin />Please wait...
     </b-button>
-    <b-button v-else variant="primary" @click="$emit('onClick')">
+    <b-button v-else variant="primary" v-b-tooltip.hover.bottom :title="tooltip" @click="$emit('onClick')">
         <font-awesome-icon icon="check" class="mr-2" />{{ title }}
     </b-button>
 </template>
@@ -27,6 +27,10 @@ export default {
             type: Boolean,
             default: false,
         },
+        tooltip: {
+            type: String,
+            default: null,
+        },
     },
 };
 </script>
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 61074740480..566c4808344 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -71,7 +71,13 @@
                         />
                     </template>
                     <template v-slot:buttons>
-                        <ButtonSpinner id="execute" title="Execute" @onClick="onExecute" :wait="showExecuting" />
+                        <ButtonSpinner
+                            id="execute"
+                            title="Execute"
+                            @onClick="onExecute"
+                            :wait="showExecuting"
+                            :tooltip="tooltip"
+                        />
                     </template>
                 </ToolCard>
             </div>
@@ -166,6 +172,9 @@ export default {
         toolName() {
             return this.formConfig.name;
         },
+        tooltip() {
+            return `Execute: ${this.formConfig.name} (${this.formConfig.version})`;
+        },
         errorContentPretty() {
             return JSON.stringify(this.errorContent, null, 4);
         },

From 82e82d99c33b8f00626be31e583b30a328b5778a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 28 Jun 2021 11:30:11 -0400
Subject: [PATCH 044/452] Access url parameters for tool form without using
 global galaxy object

---
 client/src/components/Tool/services.js | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index ca718530fe6..651bb7c7a57 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -1,7 +1,6 @@
 import axios from "axios";
 import { rethrowSimple } from "utils/simple-error";
 import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
 
 export async function updateToolFormData(tool_id, tool_version, inputs) {
     const current_state = {
@@ -20,7 +19,6 @@ export async function updateToolFormData(tool_id, tool_version, inputs) {
 
 /** Tools data request helper **/
 export async function getToolFormData(tool_id, tool_version, job_id, history_id) {
-    const Galaxy = getGalaxyInstance();
     let url = "";
     let data = {};
 
@@ -29,8 +27,13 @@ export async function getToolFormData(tool_id, tool_version, job_id, history_id)
         url = `${getAppRoot()}api/jobs/${job_id}/build_for_rerun`;
     } else {
         url = `${getAppRoot()}api/tools/${tool_id}/build`;
-        data = Object.assign({}, Galaxy.params);
-        data["tool_id"] && delete data["tool_id"];
+        const queryString = window.location.search;
+        const params = new URLSearchParams(queryString);
+        for (const [key, value] of params.entries()) {
+            if (key != "tool_id") {
+                data[key] = value;
+            }
+        }
     }
     history_id && (data["history_id"] = history_id);
     tool_version && (data["tool_version"] = tool_version);

From 973806309779198084b5db1ccbe5fe4fec5e6584 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 28 Jun 2021 20:43:38 +0200
Subject: [PATCH 045/452] reduce a few custom events

---
 .../components/Form/Elements/FormBoolean.vue  | 23 +++++++----------
 .../components/Form/Elements/FormInput.vue    | 25 ++++++++-----------
 client/src/components/Form/FormElement.vue    | 21 ++++++++++------
 client/src/components/Tool/ToolForm.vue       | 11 +++-----
 .../Workflow/Editor/Forms/FormDefault.vue     |  4 +--
 .../Workflow/Editor/Forms/FormTool.vue        |  4 +--
 6 files changed, 41 insertions(+), 47 deletions(-)

diff --git a/client/src/components/Form/Elements/FormBoolean.vue b/client/src/components/Form/Elements/FormBoolean.vue
index 0d18aaa0076..b926c2c3738 100644
--- a/client/src/components/Form/Elements/FormBoolean.vue
+++ b/client/src/components/Form/Elements/FormBoolean.vue
@@ -1,5 +1,5 @@
 <template>
-    <b-form-checkbox v-model="currentValue" size="lg" switch @change="onChange" />
+    <b-form-checkbox v-model="currentValue" size="lg" switch />
 </template>
 
 <script>
@@ -10,19 +10,14 @@ export default {
             required: true,
         },
     },
-    data() {
-        return {
-            currentValue: this.value,
-        };
-    },
-    watch: {
-        value(newValue) {
-            this.currentValue = newValue;
-        },
-    },
-    methods: {
-        onChange() {
-            this.$emit("onChange", this.currentValue);
+    computed: {
+        currentValue: {
+            get() {
+                return this.value;
+            },
+            set(val) {
+                this.$emit("input", val);
+            },
         },
     },
 };
diff --git a/client/src/components/Form/Elements/FormInput.vue b/client/src/components/Form/Elements/FormInput.vue
index ee344d00eb5..05758a19b28 100644
--- a/client/src/components/Form/Elements/FormInput.vue
+++ b/client/src/components/Form/Elements/FormInput.vue
@@ -1,6 +1,6 @@
 <template>
-    <textarea v-if="area" class="ui-input" v-model="currentValue" @input="onChange" @change="onChange" />
-    <input v-else class="ui-input" v-model="currentValue" @input="onChange" @change="onChange" />
+    <textarea v-if="area" class="ui-input" v-model="currentValue" />
+    <input v-else class="ui-input" v-model="currentValue" />
 </template>
 <script>
 export default {
@@ -18,19 +18,14 @@ export default {
             default: false,
         },
     },
-    data() {
-        return {
-            currentValue: this.value,
-        };
-    },
-    watch: {
-        value(newValue) {
-            this.currentValue = newValue;
-        },
-    },
-    methods: {
-        onChange() {
-            this.$emit("onChange", this.currentValue);
+    computed: {
+        currentValue: {
+            get() {
+                return this.value;
+            },
+            set(val) {
+                this.$emit("input", val);
+            },
         },
     },
 };
diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue
index 2827231ab94..ffd3d4d6711 100644
--- a/client/src/components/Form/FormElement.vue
+++ b/client/src/components/Form/FormElement.vue
@@ -6,8 +6,8 @@
         </div>
         <div class="ui-form-title">{{ title }}</div>
         <div class="ui-form-field">
-            <FormInput v-if="type == 'text'" :id="id" :value="value" :area="area" @onChange="onChange" />
-            <FormBoolean v-else-if="type == 'boolean'" :id="id" :value="value" @onChange="onChange" />
+            <FormInput v-if="type == 'text'" :id="id" :area="area" v-model="currentValue" />
+            <FormBoolean v-else-if="type == 'boolean'" :id="id" v-model="currentValue" />
             <span class="ui-form-info form-text text-muted mt-2">{{ help }}</span>
         </div>
     </div>
@@ -52,6 +52,18 @@ export default {
         },
     },
     computed: {
+        currentValue: {
+            get() {
+                if (this.type === "boolean") {
+                    return typeof this.value == "boolean" ? this.value : this.value === "true";
+                } else {
+                    return this.value;
+                }
+            },
+            set(val) {
+                this.$emit("input", val);
+            },
+        },
         hasError() {
             return !!this.error;
         },
@@ -63,10 +75,5 @@ export default {
             }
         },
     },
-    methods: {
-        onChange(value) {
-            this.$emit("onChange", value);
-        },
-    },
 };
 </script>
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 566c4808344..e6fcec4df10 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -49,7 +49,7 @@
                             help="Send an email notification when the job completes."
                             type="boolean"
                             value="false"
-                            @onChange="onChangeEmail"
+                            v-model="email"
                         />
                         <FormElement
                             v-if="remapAllowed"
@@ -58,7 +58,7 @@
                             :help="remapHelp"
                             type="boolean"
                             value="false"
-                            @onChange="onChangeRemap"
+                            @input="onChangeRemap"
                         />
                         <FormElement
                             v-if="reuseAllowed(user)"
@@ -67,7 +67,7 @@
                             help="This may skip executing jobs that you have already run."
                             type="boolean"
                             value="false"
-                            @onChange="onChangeReuse"
+                            v-model="reuse"
                         />
                     </template>
                     <template v-slot:buttons>
@@ -201,7 +201,7 @@ export default {
     },
     methods: {
         emailAllowed(config, user) {
-            return config.server_mail_configured && !user.isAnonymous();
+            return config.server_mail_configured && !user.isAnonymous;
         },
         reuseAllowed(user) {
             if (
@@ -232,9 +232,6 @@ export default {
         onChangeVersion(newVersion) {
             this.requestTool(newVersion);
         },
-        onChangeEmail(email) {
-            this.email = email;
-        },
         onChangeRemap(remap) {
             this.remap = remap ? this.job_id : null;
         },
diff --git a/client/src/components/Workflow/Editor/Forms/FormDefault.vue b/client/src/components/Workflow/Editor/Forms/FormDefault.vue
index ac46cad7a6c..226b2a11a82 100644
--- a/client/src/components/Workflow/Editor/Forms/FormDefault.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormDefault.vue
@@ -33,7 +33,7 @@
                 :value="node.label"
                 title="Label"
                 help="Add a step label."
-                @onChange="onLabel"
+                @input="onLabel"
                 :error="errorLabel"
             />
             <FormElement
@@ -42,7 +42,7 @@
                 title="Step Annotation"
                 :area="true"
                 help="Add an annotation or notes to this step. Annotations are available when a workflow is viewed."
-                @onChange="onAnnotation"
+                @input="onAnnotation"
             />
             <FormDisplay :id="id" :inputs="inputs" @onChange="onChange" />
         </template>
diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index 4f82fec5aed..70caeb2084d 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -18,7 +18,7 @@
                     :value="node.label"
                     title="Label"
                     help="Add a step label."
-                    @onChange="onLabel"
+                    @input="onLabel"
                     :error="errorLabel"
                 />
                 <FormElement
@@ -27,7 +27,7 @@
                     title="Step Annotation"
                     :area="true"
                     help="Add an annotation or notes to this step. Annotations are available when a workflow is viewed."
-                    @onChange="onAnnotation"
+                    @input="onAnnotation"
                 />
                 <FormDisplay
                     :id="id"

From 4a5fc2ac9760cc1fd8952cd4967b2fe4fd2a4639 Mon Sep 17 00:00:00 2001
From: Aysam Guerler <aysam.guerler@gmail.com>
Date: Tue, 29 Jun 2021 14:36:53 -0400
Subject: [PATCH 046/452] Update client/src/components/Tool/ToolForm.vue

Co-authored-by: Oleg Zharkov <jos.to.new@gmail.com>
---
 client/src/components/Tool/ToolForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 566c4808344..fdea342ffc8 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -201,7 +201,7 @@ export default {
     },
     methods: {
         emailAllowed(config, user) {
-            return config.server_mail_configured && !user.isAnonymous();
+            return config.server_mail_configured && !user.isAnonymous;
         },
         reuseAllowed(user) {
             if (

From 906e0df8bb0391e0fb46c17d6756c5293591305f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 29 Jun 2021 14:41:31 -0400
Subject: [PATCH 047/452] Adjust access to user preferences to differences
 between user provider and user object

---
 client/src/components/Tool/ToolForm.vue | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index fdea342ffc8..3daa9558605 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -205,11 +205,10 @@ export default {
         },
         reuseAllowed(user) {
             if (
-                user.attributes &&
-                user.attributes.preferences &&
-                "extra_user_preferences" in user.attributes.preferences
+                user.preferences &&
+                "extra_user_preferences" in user.preferences
             ) {
-                const extra_user_preferences = JSON.parse(user.attributes.preferences.extra_user_preferences);
+                const extra_user_preferences = JSON.parse(user.preferences.extra_user_preferences);
                 const keyCached = "use_cached_job|use_cached_job_checkbox";
                 const hasCachedJobs = keyCached in extra_user_preferences;
                 return hasCachedJobs ? extra_user_preferences[keyCached] : false;

From ad69622049c7190b47b5966f087ca721f46035e9 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Wed, 30 Jun 2021 17:31:43 +0200
Subject: [PATCH 048/452] fix tests

---
 client/src/components/Form/Elements/FormBoolean.test.js | 2 +-
 client/src/components/Form/Elements/FormInput.test.js   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Form/Elements/FormBoolean.test.js b/client/src/components/Form/Elements/FormBoolean.test.js
index 92f83734326..90149c2442c 100644
--- a/client/src/components/Form/Elements/FormBoolean.test.js
+++ b/client/src/components/Form/Elements/FormBoolean.test.js
@@ -27,6 +27,6 @@ describe("FormBoolean", () => {
         expect(input.element.checked).toBe(false);
         input.trigger("click");
         expect(input.element.checked).toBe(true);
-        expect(wrapper.emitted().onChange[0][0]).toBe(true);
+        expect(wrapper.emitted().input[0][0]).toBe(true);
     });
 });
diff --git a/client/src/components/Form/Elements/FormInput.test.js b/client/src/components/Form/Elements/FormInput.test.js
index ebe24aa5960..c9b7eb9247b 100644
--- a/client/src/components/Form/Elements/FormInput.test.js
+++ b/client/src/components/Form/Elements/FormInput.test.js
@@ -24,7 +24,7 @@ describe("FormInput", () => {
         expect(input.element.value).toBe("initial_value");
         await input.setValue("new_value");
         expect(input.element.value).toBe("new_value");
-        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
+        expect(wrapper.emitted().input[0][0]).toBe("new_value");
     });
 
     it("check switching to text area", async () => {
@@ -33,6 +33,6 @@ describe("FormInput", () => {
         expect(input.element.value).toBe("initial_value");
         await input.setValue("new_value");
         expect(input.element.value).toBe("new_value");
-        expect(wrapper.emitted().onChange[0][0]).toBe("new_value");
+        expect(wrapper.emitted().input[0][0]).toBe("new_value");
     });
 });

From 51ac6d512eba84fb1195ee55e6c902bf678226db Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Wed, 30 Jun 2021 17:45:09 +0200
Subject: [PATCH 049/452] fix lint

---
 client/src/components/Tool/ToolForm.vue | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index bdb52bd1501..68531edc9aa 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -204,10 +204,7 @@ export default {
             return config.server_mail_configured && !user.isAnonymous;
         },
         reuseAllowed(user) {
-            if (
-                user.preferences &&
-                "extra_user_preferences" in user.preferences
-            ) {
+            if (user.preferences && "extra_user_preferences" in user.preferences) {
                 const extra_user_preferences = JSON.parse(user.preferences.extra_user_preferences);
                 const keyCached = "use_cached_job|use_cached_job_checkbox";
                 const hasCachedJobs = keyCached in extra_user_preferences;

From 9cf5e6e99f77e6967c941a754d2af64323e4c278 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Wed, 30 Jun 2021 17:45:27 +0200
Subject: [PATCH 050/452] fix prop type

---
 client/src/components/Form/FormElement.vue | 2 +-
 client/src/components/Tool/ToolForm.vue    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue
index ffd3d4d6711..bef60f11d36 100644
--- a/client/src/components/Form/FormElement.vue
+++ b/client/src/components/Form/FormElement.vue
@@ -27,7 +27,7 @@ export default {
             required: true,
         },
         value: {
-            type: String,
+            type: [String, Boolean],
             default: "",
         },
         title: {
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 68531edc9aa..1089a45adc7 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -48,7 +48,7 @@
                             title="Email notification"
                             help="Send an email notification when the job completes."
                             type="boolean"
-                            value="false"
+                            :value="false"
                             v-model="email"
                         />
                         <FormElement
@@ -57,7 +57,7 @@
                             :title="remapTitle"
                             :help="remapHelp"
                             type="boolean"
-                            value="false"
+                            :value="false"
                             @input="onChangeRemap"
                         />
                         <FormElement
@@ -66,7 +66,7 @@
                             title="Attempt to re-use jobs with identical parameters?"
                             help="This may skip executing jobs that you have already run."
                             type="boolean"
-                            value="false"
+                            :value="false"
                             v-model="reuse"
                         />
                     </template>

From 6b8bbf99982b611e13a281fa1deaef379ea3f948 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Wed, 30 Jun 2021 19:24:26 +0200
Subject: [PATCH 051/452] validation

---
 client/src/components/Form/FormDisplay.vue | 2 +-
 client/src/components/Tool/ToolForm.vue    | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 09d42bd6f16..c86307d2778 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -54,7 +54,7 @@ export default {
         },
         validation() {
             this.onHighlight(this.validation, true);
-            this.$emit("onValidation", this.validation);
+            this.$emit("input", this.validation);
         },
         validationErrors() {
             const matchedErrors = Object.entries(this.form.data.matchResponse(this.validationErrors));
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 1089a45adc7..76c59c3ce28 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -40,7 +40,7 @@
                             :validation-scroll-to="validationScrollTo"
                             :form-config="formConfig"
                             @onChange="onChange"
-                            @onValidation="onValidation"
+                            v-model="validationInternal"
                         />
                         <FormElement
                             v-if="emailAllowed(config, user)"
@@ -213,9 +213,6 @@ export default {
                 return false;
             }
         },
-        onValidation(validationInternal) {
-            this.validationInternal = validationInternal;
-        },
         onChange(newData) {
             this.formData = newData;
             this.onUpdate();

From a85d6d47a858ed2c6f564999fba6ced3609a1145 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 30 Jun 2021 15:36:06 -0400
Subject: [PATCH 052/452] This is a legacy option in the code path which allows
 tools to submit the form data to an arbritrary url, this does not seemed to
 be used anywhere

---
 client/src/components/Tool/ToolForm.vue |  5 -----
 client/src/components/Tool/utilities.js | 12 ------------
 2 files changed, 17 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 76c59c3ce28..fe7e677f3e3 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -90,7 +90,6 @@ import Scroller from "vue-scrollto";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
-import { send } from "./utilities";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
@@ -261,10 +260,6 @@ export default {
                 tool_version: this.formConfig.version,
                 inputs: this.formData,
             };
-            if (this.formConfig.action !== `${getAppRoot()}tool_runner/index`) {
-                send(options, jobDef);
-                return;
-            }
             Galaxy.emit.debug("tool-form::submit()", "Validation complete.", jobDef);
             submitJob(jobDef).then(
                 (jobResponse) => {
diff --git a/client/src/components/Tool/utilities.js b/client/src/components/Tool/utilities.js
index 0c8fb372b77..9a40fecd5b3 100644
--- a/client/src/components/Tool/utilities.js
+++ b/client/src/components/Tool/utilities.js
@@ -13,15 +13,3 @@ export function downloadTool(toolId) {
 export function openLink(url) {
     window.open(url);
 }
-
-export function send(options, job_def) {
-    const $f = $("<form/>").attr({
-        action: options.action,
-        method: options.method,
-        enctype: options.enctype,
-    });
-    Object.entries(job_def.inputs).forEach(([value, key]) => {
-        $f.append($("<input/>").attr({ name: key, value: value }));
-    });
-    $f.hide().appendTo("body").submit().remove();
-}

From 74ac9fec969fb8b5bf116e84394886d949b5691c Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 30 Jun 2021 15:50:57 -0400
Subject: [PATCH 053/452] Fix linting, remove unused imports

---
 client/src/components/Tool/ToolForm.vue | 2 --
 client/src/components/Tool/services.js  | 2 +-
 client/src/components/Tool/utilities.js | 1 -
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index fe7e677f3e3..1c0b73049cb 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -87,7 +87,6 @@
 
 <script>
 import Scroller from "vue-scrollto";
-import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
 import ToolCard from "./ToolCard";
@@ -251,7 +250,6 @@ export default {
                 return;
             }
             this.showExecuting = true;
-            const options = this.formData;
             const Galaxy = getGalaxyInstance();
             const history_id = Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.model.id;
             const jobDef = {
diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index 651bb7c7a57..f8df815e275 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -20,7 +20,7 @@ export async function updateToolFormData(tool_id, tool_version, inputs) {
 /** Tools data request helper **/
 export async function getToolFormData(tool_id, tool_version, job_id, history_id) {
     let url = "";
-    let data = {};
+    const data = {};
 
     // build request url and collect request data
     if (job_id) {
diff --git a/client/src/components/Tool/utilities.js b/client/src/components/Tool/utilities.js
index 9a40fecd5b3..d0d4e736c5b 100644
--- a/client/src/components/Tool/utilities.js
+++ b/client/src/components/Tool/utilities.js
@@ -1,4 +1,3 @@
-import $ from "jquery";
 import { getAppRoot } from "onload/loadConfig";
 import { copy } from "utils/clipboard";
 

From 65f67f4fd378877f3a82d715ef05a726e832ce09 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 1 Jul 2021 11:42:28 -0400
Subject: [PATCH 054/452] Use on validation custom event for form display

---
 client/src/components/Form/FormDisplay.vue | 2 +-
 client/src/components/Tool/ToolForm.vue    | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index c86307d2778..09d42bd6f16 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -54,7 +54,7 @@ export default {
         },
         validation() {
             this.onHighlight(this.validation, true);
-            this.$emit("input", this.validation);
+            this.$emit("onValidation", this.validation);
         },
         validationErrors() {
             const matchedErrors = Object.entries(this.form.data.matchResponse(this.validationErrors));
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 1c0b73049cb..548239139bf 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -40,7 +40,7 @@
                             :validation-scroll-to="validationScrollTo"
                             :form-config="formConfig"
                             @onChange="onChange"
-                            v-model="validationInternal"
+                            @onValidation="validationInternal"
                         />
                         <FormElement
                             v-if="emailAllowed(config, user)"
@@ -211,6 +211,9 @@ export default {
                 return false;
             }
         },
+        onValidation(validationInternal) {
+            this.validationInternal = validationInternal;
+        },
         onChange(newData) {
             this.formData = newData;
             this.onUpdate();

From 6797a0381b5ee9591df8b79b19018bf3bedb36dd Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 2 Jul 2021 11:34:10 -0400
Subject: [PATCH 055/452] Fix validation handler, use parsed history id

---
 client/src/components/Tool/ToolForm.vue | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 548239139bf..625291b4c6c 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -40,7 +40,7 @@
                             :validation-scroll-to="validationScrollTo"
                             :form-config="formConfig"
                             @onChange="onChange"
-                            @onValidation="validationInternal"
+                            @onValidation="onValidation"
                         />
                         <FormElement
                             v-if="emailAllowed(config, user)"
@@ -254,9 +254,8 @@ export default {
             }
             this.showExecuting = true;
             const Galaxy = getGalaxyInstance();
-            const history_id = Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.model.id;
             const jobDef = {
-                history_id: history_id,
+                history_id: this.history_id,
                 tool_id: this.formConfig.id,
                 tool_version: this.formConfig.version,
                 inputs: this.formData,

From c534a53daaae4dbc3383ff1d229c09ca8ac61a2c Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 2 Jul 2021 12:00:22 -0400
Subject: [PATCH 056/452] Remove Galaxy object from Tool Form, user history
 provider instead

---
 client/src/components/Tool/ToolForm.vue | 186 ++++++++++++------------
 1 file changed, 92 insertions(+), 94 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 625291b4c6c..0e753c0b91e 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -1,93 +1,99 @@
 <template>
     <ConfigProvider v-slot="{ config }">
         <CurrentUser v-slot="{ user }">
-            <div>
-                <LoadingSpan v-if="showLoading" message="Loading Tool" />
-                <div v-if="showEntryPoints">
-                    <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
+            <UserHistories v-if="user" :user="user" v-slot="{ currentHistory }">
+                <div>
+                    <LoadingSpan v-if="showLoading" message="Loading Tool" />
+                    <div v-if="showEntryPoints">
+                        <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
+                    </div>
+                    <ToolSuccess
+                        v-if="showSuccess"
+                        :job-def="jobDef"
+                        :job-response="jobResponse"
+                        :tool-name="toolName"
+                    />
+                    <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
+                    <div v-if="showError" class="errormessagelarge">
+                        <p>
+                            The server could not complete the request. Please contact the Galaxy Team if this error
+                            persists.
+                            {{ errorTitle | l }}
+                        </p>
+                        <pre>{{ errorContentPretty }}</pre>
+                    </div>
+                    <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
+                    <ToolCard
+                        v-if="showForm"
+                        :id="formConfig.id"
+                        :user="user"
+                        :version="formConfig.version"
+                        :title="formConfig.name"
+                        :description="formConfig.description"
+                        :options="formConfig"
+                        :message-text="messageText"
+                        :message-variant="messageVariant"
+                        @onChangeVersion="onChangeVersion"
+                        @onUpdateFavorites="onUpdateFavorites"
+                        itemscope="itemscope"
+                        itemtype="https://schema.org/CreativeWork"
+                    >
+                        <template v-slot:body>
+                            <FormDisplay
+                                :id="formConfig.id"
+                                :inputs="inputs"
+                                :validation-errors="validationErrors"
+                                :validation-scroll-to="validationScrollTo"
+                                :form-config="formConfig"
+                                @onChange="onChange"
+                                @onValidation="onValidation"
+                            />
+                            <FormElement
+                                v-if="emailAllowed(config, user)"
+                                id="send_email_notification"
+                                title="Email notification"
+                                help="Send an email notification when the job completes."
+                                type="boolean"
+                                :value="false"
+                                v-model="email"
+                            />
+                            <FormElement
+                                v-if="remapAllowed"
+                                id="rerun_remap_job_id"
+                                :title="remapTitle"
+                                :help="remapHelp"
+                                type="boolean"
+                                :value="false"
+                                @input="onChangeRemap"
+                            />
+                            <FormElement
+                                v-if="reuseAllowed(user)"
+                                id="use_cached_job"
+                                title="Attempt to re-use jobs with identical parameters?"
+                                help="This may skip executing jobs that you have already run."
+                                type="boolean"
+                                :value="false"
+                                v-model="reuse"
+                            />
+                        </template>
+                        <template v-slot:buttons>
+                            <ButtonSpinner
+                                id="execute"
+                                title="Execute"
+                                @onClick="onExecute(config, currentHistory.id)"
+                                :wait="showExecuting"
+                                :tooltip="tooltip"
+                            />
+                        </template>
+                    </ToolCard>
                 </div>
-                <ToolSuccess v-if="showSuccess" :job-def="jobDef" :job-response="jobResponse" :tool-name="toolName" />
-                <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
-                <div v-if="showError" class="errormessagelarge">
-                    <p>
-                        The server could not complete the request. Please contact the Galaxy Team if this error
-                        persists.
-                        {{ errorTitle | l }}
-                    </p>
-                    <pre>{{ errorContentPretty }}</pre>
-                </div>
-                <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
-                <ToolCard
-                    v-if="showForm"
-                    :id="formConfig.id"
-                    :user="user"
-                    :version="formConfig.version"
-                    :title="formConfig.name"
-                    :description="formConfig.description"
-                    :options="formConfig"
-                    :message-text="messageText"
-                    :message-variant="messageVariant"
-                    @onChangeVersion="onChangeVersion"
-                    @onUpdateFavorites="onUpdateFavorites"
-                    itemscope="itemscope"
-                    itemtype="https://schema.org/CreativeWork"
-                >
-                    <template v-slot:body>
-                        <FormDisplay
-                            :id="formConfig.id"
-                            :inputs="inputs"
-                            :validation-errors="validationErrors"
-                            :validation-scroll-to="validationScrollTo"
-                            :form-config="formConfig"
-                            @onChange="onChange"
-                            @onValidation="onValidation"
-                        />
-                        <FormElement
-                            v-if="emailAllowed(config, user)"
-                            id="send_email_notification"
-                            title="Email notification"
-                            help="Send an email notification when the job completes."
-                            type="boolean"
-                            :value="false"
-                            v-model="email"
-                        />
-                        <FormElement
-                            v-if="remapAllowed"
-                            id="rerun_remap_job_id"
-                            :title="remapTitle"
-                            :help="remapHelp"
-                            type="boolean"
-                            :value="false"
-                            @input="onChangeRemap"
-                        />
-                        <FormElement
-                            v-if="reuseAllowed(user)"
-                            id="use_cached_job"
-                            title="Attempt to re-use jobs with identical parameters?"
-                            help="This may skip executing jobs that you have already run."
-                            type="boolean"
-                            :value="false"
-                            v-model="reuse"
-                        />
-                    </template>
-                    <template v-slot:buttons>
-                        <ButtonSpinner
-                            id="execute"
-                            title="Execute"
-                            @onClick="onExecute"
-                            :wait="showExecuting"
-                            :tooltip="tooltip"
-                        />
-                    </template>
-                </ToolCard>
-            </div>
+            </UserHistories>
         </CurrentUser>
     </ConfigProvider>
 </template>
 
 <script>
 import Scroller from "vue-scrollto";
-import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
@@ -97,6 +103,7 @@ import LoadingSpan from "components/LoadingSpan";
 import FormDisplay from "components/Form/FormDisplay";
 import FormElement from "components/Form/FormElement";
 import ToolSuccess from "./ToolSuccess";
+import UserHistories from "components/History/providers/UserHistories";
 import Webhook from "components/Common/Webhook";
 
 export default {
@@ -109,6 +116,7 @@ export default {
         ToolCard,
         FormElement,
         ToolSuccess,
+        UserHistories,
         Webhook,
     },
     props: {
@@ -159,12 +167,6 @@ export default {
     },
     created() {
         this.requestTool();
-        const Galaxy = getGalaxyInstance();
-        if (Galaxy.currHistoryPanel) {
-            Galaxy.currHistoryPanel.collection.on("change", () => {
-                this.onUpdate();
-            });
-        }
     },
     computed: {
         toolName() {
@@ -247,25 +249,21 @@ export default {
                 }
             });
         },
-        onExecute() {
+        onExecute(config, historyId) {
             if (this.validationInternal) {
                 this.validationScrollTo = this.validationInternal.slice();
                 return;
             }
             this.showExecuting = true;
-            const Galaxy = getGalaxyInstance();
             const jobDef = {
-                history_id: this.history_id,
+                history_id: historyId,
                 tool_id: this.formConfig.id,
                 tool_version: this.formConfig.version,
                 inputs: this.formData,
             };
-            Galaxy.emit.debug("tool-form::submit()", "Validation complete.", jobDef);
+            console.debug("toolForm::onExecute()", jobDef);
             submitJob(jobDef).then(
                 (jobResponse) => {
-                    if (Galaxy.currHistoryPanel) {
-                        Galaxy.currHistoryPanel.refreshContents();
-                    }
                     this.showForm = false;
                     if (jobResponse.produces_entry_points) {
                         this.showEntryPoints = true;
@@ -281,7 +279,7 @@ export default {
                         this.errorTitle = "Invalid success response. No jobs found.";
                         this.errorContent = jobResponse;
                     }
-                    if ([true, "true"].includes(Galaxy.config.enable_tool_recommendations)) {
+                    if ([true, "true"].includes(config.enable_tool_recommendations)) {
                         this.showRecommendation = true;
                     }
                     Scroller.scrollTo("body");

From 2f6dfc52fccd75d9aee165df5af8a87c07f805a0 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 2 Jul 2021 13:08:19 -0400
Subject: [PATCH 057/452] Attempt to resolve failing test case with current
 history

---
 client/src/components/Tool/ToolForm.vue | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 0e753c0b91e..f03787ba581 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -94,6 +94,7 @@
 
 <script>
 import Scroller from "vue-scrollto";
+import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
@@ -255,6 +256,7 @@ export default {
                 return;
             }
             this.showExecuting = true;
+            const Galaxy = getGalaxyInstance();
             const jobDef = {
                 history_id: historyId,
                 tool_id: this.formConfig.id,
@@ -264,6 +266,9 @@ export default {
             console.debug("toolForm::onExecute()", jobDef);
             submitJob(jobDef).then(
                 (jobResponse) => {
+                    if (Galaxy.currHistoryPanel) {
+                        Galaxy.currHistoryPanel.refreshContents();
+                    }
                     this.showForm = false;
                     if (jobResponse.produces_entry_points) {
                         this.showEntryPoints = true;

From 90fd09e95f945091129ffc8a23951fed7a4e6eb7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 2 Jul 2021 13:11:38 -0400
Subject: [PATCH 058/452] Remove unused helper in tool form

---
 client/src/components/Tool/ToolForm.vue | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index f03787ba581..6dd157cff1b 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -232,9 +232,6 @@ export default {
         onChangeRemap(remap) {
             this.remap = remap ? this.job_id : null;
         },
-        onChangeReuse(reuse) {
-            this.reuse = reuse;
-        },
         onUpdateFavorites(user, newFavorites) {
             user.preferences["favorites"] = newFavorites;
         },

From 06a6f8ea695e4c18c7b3f4b410f01c20d3ccb830 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 2 Jul 2021 14:45:03 -0400
Subject: [PATCH 059/452] Small fixes for beta history panel.

Issues discovered from Selenium tests. Exporting histories link does not work and the collection creation operations don't hide their items by default.
---
 client/src/components/History/HistoryDetails.vue             | 5 ++++-
 .../src/components/History/adapters/buildCollectionModal.js  | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/client/src/components/History/HistoryDetails.vue b/client/src/components/History/HistoryDetails.vue
index 7894f7a47a6..6a4fed2bcc1 100644
--- a/client/src/components/History/HistoryDetails.vue
+++ b/client/src/components/History/HistoryDetails.vue
@@ -84,7 +84,7 @@
                     key="export-history-to-file"
                     title="Export History to File"
                     icon="fas fa-file-archive"
-                    @click="iframeRedirect('/history/export_archive?preview=True')"
+                    @click="backboneRoute(exportLink)"
                 />
             </PriorityMenu>
         </div>
@@ -184,6 +184,9 @@ export default {
                 }
             },
         },
+        exportLink() {
+            return `histories/${this.history.id}/export`;
+        },
     },
     methods: {
         async updateFields(fields = {}) {
diff --git a/client/src/components/History/adapters/buildCollectionModal.js b/client/src/components/History/adapters/buildCollectionModal.js
index 3bf5599a6f2..521c3e9eb2e 100644
--- a/client/src/components/History/adapters/buildCollectionModal.js
+++ b/client/src/components/History/adapters/buildCollectionModal.js
@@ -15,7 +15,7 @@ import LIST_OF_PAIRS_COLLECTION_CREATOR from "../../Collections/PairedListCollec
 import RULE_BASED_COLLECTION_CREATOR from "../../Collections/RuleBasedCollectionCreatorModal";
 
 // stand-in for buildCollection from history-view-edit.js
-export async function buildCollectionModal(collectionType, history_id, selectedContent, hideSourceItems = false) {
+export async function buildCollectionModal(collectionType, history_id, selectedContent, hideSourceItems = true) {
     // select legacy function
     let createFunc;
     if (collectionType == "list") {

From f2d8c0aaf9c24943d08fc19dc52eddababe44a1d Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 5 Jul 2021 21:15:21 +0200
Subject: [PATCH 060/452] More f-strings, fixes string formatting bug
 introduced in a1056cd934e13d1dfde6cab7d24b28a1e63cb217

---
 lib/galaxy/tool_util/verify/interactor.py | 38 ++++++++---------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py
index f6d7c639086..2c193281126 100644
--- a/lib/galaxy/tool_util/verify/interactor.py
+++ b/lib/galaxy/tool_util/verify/interactor.py
@@ -178,9 +178,7 @@ class GalaxyInteractorApi:
                     break
 
             if not primary_output:
-                msg_template = "Failed to find primary dataset with designation [%s] for output with name [%s]"
-                msg_args = (designation, name)
-                raise Exception(msg_template % msg_args)
+                raise Exception(f"Failed to find primary dataset with designation [{designation}] for output with name [{name}]")
 
             primary_hda_id = primary_output["dataset"]["id"]
             try:
@@ -234,17 +232,13 @@ class GalaxyInteractorApi:
 
                     def compare(val, expected):
                         if str(val) != str(expected):
-                            msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
-                            msg_params = (key, value, dataset_value, dataset)
-                            msg = msg % msg_params
+                            msg = f"Dataset metadata verification for [{key}] failed, expected [{value}] but found [{dataset_value}]. Dataset API value was [{dataset}]."
                             raise Exception(msg)
 
                     if isinstance(dataset_value, list):
                         value = str(value).split(",")
                         if len(value) != len(dataset_value):
-                            msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s], lists differ in length. Dataset API value was [%s]."
-                            msg_params = (key, value, dataset_value, dataset)
-                            msg = msg % msg_params
+                            msg = f"Dataset metadata verification for [{key}] failed, expected [{value}] but found [{dataset_value}], lists differ in length. Dataset API value was [{dataset}]."
                             raise Exception(msg)
                         for val, expected in zip(dataset_value, value):
                             compare(val, expected)
@@ -369,13 +363,13 @@ class GalaxyInteractorApi:
                 if force_path_paste:
                     file_path = self.test_data_path(tool_id, file_name)
                     tool_input.update({
-                        "files_%d|url_paste" % i: "file://" + file_path
+                        f"files_{i}|url_paste": f"file://{file_path}"
                     })
                 else:
                     file_content = self.test_data_download(tool_id, file_name, is_output=False)
                     files["files_%s|file_data" % i] = file_content
                 tool_input.update({
-                    "files_%d|type" % i: "upload_dataset",
+                    f"files_{i}|type": "upload_dataset",
                 })
             name = test_data['name']
         else:
@@ -537,7 +531,7 @@ class GalaxyInteractorApi:
 
             print(ERROR_MESSAGE_DATASET_SEP)
             dataset_id = dataset.get('id', None)
-            print("| %d - %s (HID - NAME) " % (int(dataset['hid']), dataset['name']))
+            print(f"| {dataset['hid']} - {dataset['name']} (HID - NAME) ")
             if history_content['history_content_type'] == 'dataset_collection':
                 history_contents_json = self._get("histories/{}/contents/dataset_collections/{}".format(history_id, history_content["id"])).json()
                 print("| Dataset Collection: %s" % history_contents_json)
@@ -727,8 +721,7 @@ def ensure_tool_run_response_okay(submit_response_object, request_desc, inputs=N
             # invalid JSON content.
             pass
         if message is None:
-            template = "Request to %s failed - invalid JSON content returned from Galaxy server [%s]"
-            message = template % (request_desc, submit_response_object.text)
+            message = f"Request to {request_desc} failed - invalid JSON content returned from Galaxy server [{submit_response_object.text}]"
         raise RunToolException(message, inputs, dynamic_param_error=dynamic_param_error)
     submit_response = submit_response_object.json()
     return submit_response
@@ -779,16 +772,14 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
     if expected_collection_type:
         collection_type = data_collection["collection_type"]
         if expected_collection_type != collection_type:
-            template = "Output collection '%s': expected to be of type [%s], was of type [%s]."
-            message = template % (name, expected_collection_type, collection_type)
+            message = f"Output collection '{name}': expected to be of type [{expected_collection_type}], was of type [{collection_type}]."
             raise AssertionError(message)
 
     expected_element_count = output_collection_def.count
     if expected_element_count:
         actual_element_count = len(data_collection["elements"])
         if expected_element_count != actual_element_count:
-            template = "Output collection '%s': expected to have %s elements, but it had %s."
-            message = template % (name, expected_element_count, actual_element_count)
+            message = f"Output collection '{name}': expected to have {expected_element_count} elements, but it had {actual_element_count}."
             raise AssertionError(message)
 
     def get_element(elements, id):
@@ -812,8 +803,7 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
 
             element = get_element(element_objects, element_identifier)
             if not element:
-                template = "Output collection '%s': failed to find identifier '%s' in the tool generated elements %s"
-                message = template % (element_identifier, eo_ids)
+                message = f"Output collection '{name}': failed to find identifier '{element_identifier}' in the tool generated elements {eo_ids}"
                 raise AssertionError(message)
 
             element_type = element["element_type"]
@@ -834,8 +824,7 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
                         break
                     i += 1
                 if element is None:
-                    template = "Output collection '%s': identifier '%s' found out of order, expected order of %s for the tool generated collection elements %s"
-                    message = template % (name, element_identifier, expected_sort_order, eo_ids)
+                    message = f"Output collection '{name}': identifier '{element_identifier}' found out of order, expected order of {expected_sort_order} for the tool generated collection elements {eo_ids}"
                     raise AssertionError(message)
 
     verify_elements(data_collection["elements"], output_collection_def.element_tests)
@@ -1205,8 +1194,7 @@ def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_
             name = output_collection_def.name
             # TODO: data_collection_list is clearly a bad name for dictionary.
             if name not in data_collection_list:
-                template = "Failed to find output [%s], tool outputs include [%s]"
-                message = template % (name, ",".join(data_collection_list.keys()))
+                message = f"Failed to find output [{name}], tool outputs include [{','.join(data_collection_list.keys())}]"
                 raise AssertionError(message)
 
             # Data collection returned from submission, elements may have been populated after
@@ -1252,7 +1240,7 @@ class ToolTestDescription:
     def __init__(self, processed_test_dict):
         assert "test_index" in processed_test_dict, "Invalid processed test description, must have a 'test_index' for naming, etc.."
         test_index = processed_test_dict["test_index"]
-        name = processed_test_dict.get('name', 'Test-%d' % (test_index + 1))
+        name = processed_test_dict.get('name', f'Test-{test_index + 1}')
         maxseconds = processed_test_dict.get('maxseconds', DEFAULT_TOOL_TEST_WAIT)
         if maxseconds is not None:
             maxseconds = int(maxseconds)

From 2946bbf132c94815a9d4280be0faaece415a3d85 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 6 Jul 2021 10:11:36 -0400
Subject: [PATCH 061/452] Restore history update listener

---
 client/src/components/Tool/ToolForm.vue | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 6dd157cff1b..28d18f9a2ef 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -168,6 +168,12 @@ export default {
     },
     created() {
         this.requestTool();
+        const Galaxy = getGalaxyInstance();
+        if (Galaxy.currHistoryPanel) {
+            Galaxy.currHistoryPanel.collection.on("change", () => {
+                this.onUpdate();
+            });
+        }
     },
     computed: {
         toolName() {

From 4da9a03d6ac716938d451de292e5383cd3ba04b7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 6 Jul 2021 10:35:32 -0400
Subject: [PATCH 062/452] Attempt to resolve input type dependency in form
 element wrapper

---
 client/src/components/Form/FormElement.vue | 12 ++++--------
 client/src/components/Tool/ToolForm.vue    |  2 +-
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue
index bef60f11d36..89abca8ca1e 100644
--- a/client/src/components/Form/FormElement.vue
+++ b/client/src/components/Form/FormElement.vue
@@ -28,15 +28,15 @@ export default {
         },
         value: {
             type: [String, Boolean],
-            default: "",
+            default: null,
         },
         title: {
             type: String,
-            default: "",
+            default: null,
         },
         help: {
             type: String,
-            default: "",
+            default: null,
         },
         area: {
             type: Boolean,
@@ -54,11 +54,7 @@ export default {
     computed: {
         currentValue: {
             get() {
-                if (this.type === "boolean") {
-                    return typeof this.value == "boolean" ? this.value : this.value === "true";
-                } else {
-                    return this.value;
-                }
+                return this.value;
             },
             set(val) {
                 this.$emit("input", val);
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 28d18f9a2ef..1ae35da205c 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -191,7 +191,7 @@ export default {
         remapAllowed() {
             return this.job_id && this.formConfig.job_remap;
         },
-        remapLabel() {
+        remapTitle() {
             if (this.formConfig.job_remap === "job_produced_collection_elements") {
                 return "Replace elements in collection ?";
             } else {

From 31419e2ecb9669c8e30c6a723eeac00981af0267 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 6 Jul 2021 10:59:01 -0400
Subject: [PATCH 063/452] Replace custom error message with bootstrap vue alert

---
 client/src/components/Tool/ToolForm.vue | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 1ae35da205c..6e8c780682d 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -14,14 +14,14 @@
                         :tool-name="toolName"
                     />
                     <Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
-                    <div v-if="showError" class="errormessagelarge">
+                    <b-alert v-if="showError" show variant="danger">
+                        <h4>{{ errorTitle | l }}</h4>
                         <p>
                             The server could not complete the request. Please contact the Galaxy Team if this error
                             persists.
-                            {{ errorTitle | l }}
                         </p>
                         <pre>{{ errorContentPretty }}</pre>
-                    </div>
+                    </b-alert>
                     <ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
                     <ToolCard
                         v-if="showForm"
@@ -150,7 +150,7 @@ export default {
             error: null,
             formConfig: {},
             formData: {},
-            errorTitle: "",
+            errorTitle: null,
             errorContent: null,
             messageVariant: "",
             messageText: "",

From f3476a2d8119647e70efd616a77a844ba9e9bf3e Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Thu, 8 Jul 2021 10:10:07 -0400
Subject: [PATCH 064/452] Fix fugue pencil on old sharing not showing up
 correctly. Instead of tracking down and fixing the static I just swapped to
 the fontawesome pencil already used elsewhere.

---
 templates/webapps/galaxy/workflow/sharing.mako | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/templates/webapps/galaxy/workflow/sharing.mako b/templates/webapps/galaxy/workflow/sharing.mako
index 533031a5dd0..a6df627a173 100644
--- a/templates/webapps/galaxy/workflow/sharing.mako
+++ b/templates/webapps/galaxy/workflow/sharing.mako
@@ -145,7 +145,9 @@
                             ${"/".join( url_parts[:-1] )}/<span id='item-identifier'>${url_parts[-1]}</span>
                         </span>
 
-                        <a href="javascript:void(0)" role="button" id="edit-identifier"><img src="${h.url_for('/static/images/fugue/pencil.png')}"/></a>
+                        <a href="javascript:void(0)" role="button" id="edit-identifier">
+                            <span class="fa fa-pencil" style=""></span>
+                        </a>
                     </blockquote>
 
                     %if item.published:

From 8ba9c2416f54380769a6e0467b2c489ee202cd74 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Thu, 8 Jul 2021 13:13:51 -0400
Subject: [PATCH 065/452] Fix autodetect message as seen in edit attributes
 (and click autodetect metadata)

---
 lib/galaxy/webapps/galaxy/controllers/dataset.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py
index 39d1c39da65..2335b14c4b5 100644
--- a/lib/galaxy/webapps/galaxy/controllers/dataset.py
+++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py
@@ -386,6 +386,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
         elif operation == 'autodetect':
             # The user clicked the Auto-detect button on the 'Edit Attributes' form
             self.hda_manager.set_metadata(trans, data, overwrite=True)
+            message = "Auto-detect operation successfully submitted."
         elif operation == 'conversion':
             target_type = payload.get('target_type')
             if target_type:

From fb864c0eff92252a015138ec3e6234c638504b78 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 8 Jul 2021 16:03:07 -0400
Subject: [PATCH 066/452] Use v-model to access remap option, selectively add
 optional execution flags to inputs

---
 client/src/components/Tool/ToolForm.vue | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 6e8c780682d..087d528729e 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -64,7 +64,7 @@
                                 :help="remapHelp"
                                 type="boolean"
                                 :value="false"
-                                @input="onChangeRemap"
+                                v-model="remap"
                             />
                             <FormElement
                                 v-if="reuseAllowed(user)"
@@ -235,9 +235,6 @@ export default {
         onChangeVersion(newVersion) {
             this.requestTool(newVersion);
         },
-        onChangeRemap(remap) {
-            this.remap = remap ? this.job_id : null;
-        },
         onUpdateFavorites(user, newFavorites) {
             user.preferences["favorites"] = newFavorites;
         },
@@ -264,8 +261,19 @@ export default {
                 history_id: historyId,
                 tool_id: this.formConfig.id,
                 tool_version: this.formConfig.version,
-                inputs: this.formData,
+                inputs: {
+                    ...this.formData
+                },
             };
+            if (this.email) {
+                jobDef.inputs["send_email_notification"] = true;
+            }
+            if (this.remap) {
+                jobDef.inputs["rerun_remap_job_id"] = this.job_id;
+            }
+            if (this.reuse) {
+                jobDef.inputs["use_cached_job"] = true;
+            }
             console.debug("toolForm::onExecute()", jobDef);
             submitJob(jobDef).then(
                 (jobResponse) => {

From 90187dce9a6e78cc5d8e6a5272a31337d8d3ed00 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 9 Jul 2021 02:46:44 -0400
Subject: [PATCH 067/452] Fix lint

---
 client/src/components/Tool/ToolForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 087d528729e..282d702b1b0 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -262,7 +262,7 @@ export default {
                 tool_id: this.formConfig.id,
                 tool_version: this.formConfig.version,
                 inputs: {
-                    ...this.formData
+                    ...this.formData,
                 },
             };
             if (this.email) {

From 2c0a60d801c153d00e39c86102f0003714cb362f Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Fri, 9 Jul 2021 18:28:15 +0200
Subject: [PATCH 068/452] add <creator> to the schema template, use live links
 in xsd

---
 doc/schema_template.md              |  3 +++
 lib/galaxy/tool_util/xsd/galaxy.xsd | 38 ++++++++++++++---------------
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/doc/schema_template.md b/doc/schema_template.md
index 681ca56a5fe..2f98d31a22d 100644
--- a/doc/schema_template.md
+++ b/doc/schema_template.md
@@ -20,6 +20,9 @@ $tag:tool|edam_topics://complexType[@name='EdamTopics']
 $tag:tool|edam_operations://complexType[@name='EdamOperations']
 $tag:tool|xrefs://complexType[@name='xrefs']
 $tag:tool|xrefs|xref://complexType[@name='xref']
+$tag:tool|creator://complexType[@name='Creator']
+$tag:tool|creator|person://complexType[@name='Person']
+$tag:tool|creator|organization://complexType[@name='Organization']
 $tag:tool|requirements://complexType[@name='Requirements']
 $tag:tool|requirements|requirement://complexType[@name='Requirement']
 $tag:tool|requirements|container://complexType[@name='Container']
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index b39a34e8fe7..4d1ac59047b 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -228,11 +228,11 @@ for the list of behavior changes associated with profile versions.
       <xs:attribute name="license" type="xs:string">
         <xs:annotation>
           <xs:documentation xml:lang="en">This string specifies any full URI or a
-a short SPDX (https://spdx.org/licenses/) identifier for a license for this tool
+a short [SPDX](https://spdx.org/licenses/) identifier for a license for this tool
 wrapper. The tool wrapper version can be indepedent of the underlying tool. This
 license covers the tool XML and associated scripts shipped with the tool.
 
-This is interpreted as a schema.org license attribute (see also https://schema.org/license).
+This is interpreted as [schema.org/license](https://schema.org/license)).
 </xs:documentation>
         </xs:annotation>
       </xs:attribute>
@@ -418,47 +418,47 @@ This tag set is contained within the ``<entry_point>`` tag set. It contains the
   <xs:attributeGroup name="Thing">
     <xs:attribute name="name" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/name</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/name](https://schema.org/name)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="url" type="xs:string" use="optional">
       <xs:annotation>
-      <xs:documentation xml:lang="en">https://schema.org/url</xs:documentation>
+      <xs:documentation xml:lang="en">[schema.org/url](https://schema.org/url)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="identifier" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/identifier</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/identifier](https://schema.org/identifier)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="image" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/image</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/image](https://schema.org/image)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="address" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/address</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/address](https://schema.org/address)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="email" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/email</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/email](https://schema.org/email)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="telephone" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/telephone</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/telephone](https://schema.org/telephone)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="faxNumber" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/faxNumber</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/faxNumber](https://schema.org/faxNumber)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="alternateName" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/alternateName</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/alternateName](https://schema.org/alternateName)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
   </xs:attributeGroup>
@@ -466,33 +466,33 @@ This tag set is contained within the ``<entry_point>`` tag set. It contains the
   <xs:complexType name="Person">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
-Describes a person. Tries to stay close to schema.org data model - see https://schema.org/Person.
+Describes a person. Tries to stay close to [schema.org/Person](https://schema.org/Person).
 ]]></xs:documentation>
     </xs:annotation>
     <xs:attributeGroup ref="Thing"/>
     <xs:attribute name="givenName" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/givenName</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/givenName](https://schema.org/givenName)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="familyName" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/familyName</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/familyName](https://schema.org/familyName)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="honorificPrefix" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/honorificPrefix</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/honorificPrefix](https://schema.org/honorificPrefix)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="honorificSuffix" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/honorificSuffix</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/honorificSuffix](https://schema.org/honorificSuffix)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="jobTitle" type="xs:string" use="optional">
       <xs:annotation>
-        <xs:documentation xml:lang="en">https://schema.org/jobTitle</xs:documentation>
+        <xs:documentation xml:lang="en">[schema.org/jobTitle](https://schema.org/jobTitle)</xs:documentation>
       </xs:annotation>
     </xs:attribute>
   </xs:complexType>
@@ -500,7 +500,7 @@ Describes a person. Tries to stay close to schema.org data model - see https://s
   <xs:complexType name="Organization">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
-Describes an organization. Tries to stay close to schema.org data model - see https://schema.org/Organization.
+Describes an organization. Tries to stay close to [schema.org/Organisation](https://schema.org/Organization).
 ]]></xs:documentation>
     </xs:annotation>
     <xs:attributeGroup ref="Thing"/>
@@ -515,7 +515,7 @@ Describes an organization. Tries to stay close to schema.org data model - see ht
 
   <xs:complexType name="Creator">
     <xs:annotation>
-      <xs:documentation xml:lang="en">The creator(s) of this work. See https://schema.org/creator.</xs:documentation>
+      <xs:documentation xml:lang="en">The creator(s) of this work. See [schema.org/creator](https://schema.org/creator).</xs:documentation>
     </xs:annotation>
     <xs:sequence>
       <xs:group ref="PersonOrOrganization" minOccurs="0" maxOccurs="unbounded"/>

From 1be37a31e1053841e60492ea05d95f8fdd222952 Mon Sep 17 00:00:00 2001
From: Martin Cech <cech.marten@gmail.com>
Date: Fri, 9 Jul 2021 19:10:48 +0200
Subject: [PATCH 069/452] fix typo

Co-authored-by: Nicola Soranzo <nicola.soranzo@gmail.com>
---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 4d1ac59047b..5b2d19193e3 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -500,7 +500,7 @@ Describes a person. Tries to stay close to [schema.org/Person](https://schema.or
   <xs:complexType name="Organization">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
-Describes an organization. Tries to stay close to [schema.org/Organisation](https://schema.org/Organization).
+Describes an organization. Tries to stay close to [schema.org/Organization](https://schema.org/Organization).
 ]]></xs:documentation>
     </xs:annotation>
     <xs:attributeGroup ref="Thing"/>

From 6e4f08a7cc36a6205d2736383b0c9d00f72c2071 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Sat, 10 Jul 2021 06:31:19 +0530
Subject: [PATCH 070/452] Call metadata handling function explicitly instead of
 relying on legacy fallback

---
 lib/galaxy/jobs/runners/kubernetes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py
index a2d854d151d..9e0c089a2ba 100644
--- a/lib/galaxy/jobs/runners/kubernetes.py
+++ b/lib/galaxy/jobs/runners/kubernetes.py
@@ -842,6 +842,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             self.monitor_queue.put(ajs)
 
     def finish_job(self, job_state):
+        self._handle_metadata_externally(job_state.job_wrapper, resolve_requirements=True)
         super().finish_job(job_state)
         jobs = find_job_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace'])
         if len(jobs.response['items']) != 1:

From 5ce661e416f287f0d2c12c82c35a896a9618db50 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 11 Jul 2021 14:30:11 +0200
Subject: [PATCH 071/452] Load tool_version attribtue in extended metadata mode

This is needed for pulsar jobs that use metadata_strategy: extended and
remote metadata.
---
 lib/galaxy/jobs/runners/pulsar.py          | 2 +-
 lib/galaxy/metadata/set_metadata.py        | 8 ++++----
 lib/galaxy/model/__init__.py               | 1 +
 lib/galaxy/model/store/__init__.py         | 7 +++++--
 test/integration/test_extended_metadata.py | 1 +
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index eff37f03774..4124ec3479c 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -758,7 +758,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
         metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None)
         dynamic_outputs = None  # use default
         if metadata_strategy == "extended" and PulsarJobRunner.__remote_metadata(client):
-            # if Pulsar is doing remote metdata and the remote metadata is extended,
+            # if Pulsar is doing remote metadata and the remote metadata is extended,
             # we only need to recover the final model store.
             dynamic_outputs = EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN
         client_outputs = ClientOutputs(
diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index afaae7602cd..9de6af4fd08 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -136,7 +136,7 @@ def set_metadata_portable():
 
     object_store = None
     job_context = None
-    version_string = ""
+    version_string = None
 
     export_store = None
     final_job_state = Job.states.OK
@@ -195,9 +195,9 @@ def set_metadata_portable():
         else:
             final_job_state = Job.states.ERROR
 
-        version_string = ""
-        if os.path.exists(COMMAND_VERSION_FILENAME):
-            version_string = open(COMMAND_VERSION_FILENAME).read()
+        version_string_path = os.path.join('outputs', COMMAND_VERSION_FILENAME)
+        if os.path.exists(version_string_path):
+            version_string = open(version_string_path).read().strip()
 
         expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
 
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 2d728ae70bb..7a818541c37 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -3509,6 +3509,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
         rval["hid"] = self.hid
         rval["annotation"] = unicodify(getattr(self, 'annotation', ''))
         rval["tags"] = self.make_tag_string_list()
+        rval['tool_version'] = self.tool_version
         if self.history:
             rval["history_encoded_id"] = serialization_options.get_identifier(id_encoder, self.history)
 
diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py
index 9393e51efc2..edc04dd6fba 100644
--- a/lib/galaxy/model/store/__init__.py
+++ b/lib/galaxy/model/store/__init__.py
@@ -203,7 +203,7 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                         "object_store_id",
                         "total_size",
                         "created_from_basename",
-                        "uuid"
+                        "uuid",
                     ]
 
                     for attribute in dataset_attributes:
@@ -231,10 +231,11 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                     "designation",
                     "visible",
                     "metadata",
+                    "tool_version",
                 ]
                 for attribute in attributes:
                     if attribute in dataset_attrs:
-                        value = dataset_attrs[attribute]
+                        value = dataset_attrs.get(attribute)
                         if attribute == "metadata":
                             def remap_objects(p, k, obj):
                                 if isinstance(obj, dict) and "model_class" in obj and obj["model_class"] == "MetadataFile":
@@ -262,6 +263,7 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                                                                        visible=dataset_attrs['visible'],
                                                                        deleted=dataset_attrs.get('deleted', False),
                                                                        dbkey=metadata['dbkey'],
+                                                                       tool_version=metadata.get('tool_version'),
                                                                        metadata=metadata,
                                                                        history=history,
                                                                        create_dataset=True,
@@ -278,6 +280,7 @@ class ModelImportStore(metaclass=abc.ABCMeta):
                                                                               visible=dataset_attrs['visible'],
                                                                               deleted=dataset_attrs.get('deleted', False),
                                                                               dbkey=metadata['dbkey'],
+                                                                              tool_version=metadata.get('tool_version'),
                                                                               metadata=metadata,
                                                                               create_dataset=True,
                                                                               sa_session=self.sa_session)
diff --git a/test/integration/test_extended_metadata.py b/test/integration/test_extended_metadata.py
index 785b1340206..4bde5749361 100644
--- a/test/integration/test_extended_metadata.py
+++ b/test/integration/test_extended_metadata.py
@@ -3,6 +3,7 @@
 from galaxy_test.driver import integration_util
 
 TEST_TOOL_IDS = [
+    "job_properties",
     "multi_output",
     "multi_output_configured",
     "multi_output_assign_primary",

From 996e0a55c91ac6e7b324ec7eb5986fcfaaefcf4e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 11 Jul 2021 12:55:26 -0400
Subject: [PATCH 072/452] Add mock config provider, add test case for tool form

---
 client/src/components/Tool/ToolForm.test.js   | 61 +++++++++++++++++++
 client/src/components/Tool/ToolForm.vue       |  5 +-
 .../providers/MockConfigProvider.js           |  9 +++
 client/src/mvc/form/form-view.js              |  7 +--
 4 files changed, 73 insertions(+), 9 deletions(-)
 create mode 100644 client/src/components/Tool/ToolForm.test.js
 create mode 100644 client/src/components/providers/MockConfigProvider.js

diff --git a/client/src/components/Tool/ToolForm.test.js b/client/src/components/Tool/ToolForm.test.js
new file mode 100644
index 00000000000..4845fe1e9a8
--- /dev/null
+++ b/client/src/components/Tool/ToolForm.test.js
@@ -0,0 +1,61 @@
+import MockAdapter from "axios-mock-adapter";
+import axios from "axios";
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import ToolForm from "./ToolForm";
+import MockCurrentUser from "../providers/MockCurrentUser";
+import MockConfigProvider from "../providers/MockConfigProvider";
+import MockCurrentHistory from "../History/providers/UserHistories/MockCurrentHistory";
+import Vue from "vue";
+
+const localVue = getLocalVue();
+
+describe("ToolForm", () => {
+    let wrapper;
+    let axiosMock;
+
+    beforeEach(() => {
+        axiosMock = new MockAdapter(axios);
+
+        const toolData = {
+            id: "tool_id",
+            name: "tool_name",
+            version: "version",
+            inputs: [],
+            help: "help_text",
+        };
+        axiosMock.onGet(`/api/tools/tool_id/build?tool_version=version`).reply(200, toolData);
+
+        const citations = [];
+        axiosMock.onGet(`/api/tools/tool_id/citations`).reply(200, citations);
+
+        wrapper = mount(ToolForm, {
+            propsData: {
+                id: "tool_id",
+                version: "version",
+            },
+            localVue,
+            stubs: {
+                CurrentUser: MockCurrentUser({ id: "fakeuser" }),
+                UserHistories: MockCurrentHistory({ id: "fakehistory" }),
+                ConfigProvider: MockConfigProvider({ id: "fakeconfig" }),
+                FormDisplay: true,
+            }
+        });
+    });
+
+    afterEach(() => {
+        axiosMock.restore();
+        axiosMock.reset();
+    });
+
+    it("check props", async () => {
+        await Vue.nextTick();
+        const button = wrapper.find(".btn-primary");
+        expect(button.attributes("title")).toBe("Execute: tool_name (version)");
+        const dropdown = wrapper.findAll(".dropdown-item");
+        expect(dropdown.length).toBe(2);
+        const help = wrapper.find(".form-help");
+        expect(help.text()).toBe("help_text");
+    });
+});
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 282d702b1b0..29f5de66e28 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -54,7 +54,6 @@
                                 title="Email notification"
                                 help="Send an email notification when the job completes."
                                 type="boolean"
-                                :value="false"
                                 v-model="email"
                             />
                             <FormElement
@@ -63,7 +62,6 @@
                                 :title="remapTitle"
                                 :help="remapHelp"
                                 type="boolean"
-                                :value="false"
                                 v-model="remap"
                             />
                             <FormElement
@@ -72,7 +70,6 @@
                                 title="Attempt to re-use jobs with identical parameters?"
                                 help="This may skip executing jobs that you have already run."
                                 type="boolean"
-                                :value="false"
                                 v-model="reuse"
                             />
                         </template>
@@ -169,7 +166,7 @@ export default {
     created() {
         this.requestTool();
         const Galaxy = getGalaxyInstance();
-        if (Galaxy.currHistoryPanel) {
+        if (Galaxy && Galaxy.currHistoryPanel) {
             Galaxy.currHistoryPanel.collection.on("change", () => {
                 this.onUpdate();
             });
diff --git a/client/src/components/providers/MockConfigProvider.js b/client/src/components/providers/MockConfigProvider.js
new file mode 100644
index 00000000000..0126446102b
--- /dev/null
+++ b/client/src/components/providers/MockConfigProvider.js
@@ -0,0 +1,9 @@
+const MockConfigProvider = (fakeConfig) => ({
+    render() {
+        return this.$scopedSlots.default({
+            config: fakeConfig,
+        });
+    },
+});
+
+export default MockConfigProvider;
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index 78bd6dd1c6e..fe0bc26a869 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -8,7 +8,6 @@ import Portlet from "mvc/ui/ui-portlet";
 import Ui from "mvc/ui/ui-misc";
 import FormSection from "mvc/form/form-section";
 import FormData from "mvc/form/form-data";
-import { getGalaxyInstance } from "app";
 
 export default Backbone.View.extend({
     initialize: function (options) {
@@ -33,8 +32,7 @@ export default Backbone.View.extend({
             if (field.update) {
                 field.update(node);
                 field.trigger("change");
-                const Galaxy = getGalaxyInstance();
-                Galaxy.emit.debug("form-view::update()", `Updating input: ${input_id}`);
+                console.debug("form-view::update()", `Updating input: ${input_id}`);
             }
         });
     },
@@ -157,7 +155,6 @@ export default Backbone.View.extend({
                 message: options.message,
             });
         }
-        const Galaxy = getGalaxyInstance();
-        Galaxy.emit.debug("form-view::initialize()", "Completed");
+        console.debug("form-view::initialize()", "Completed");
     },
 });

From 58efe7110612ab7bde6d61e20ebdaa6014ad6dd0 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 11 Jul 2021 18:32:40 +0200
Subject: [PATCH 073/452] Unify getting version_string, drop legacy behavior

Also drops the explcit unlink of the version file.
This unlink had been added together with the version command,
but we can rely on the regular working directory cleanup.
---
 lib/galaxy/job_execution/output_collect.py |  8 ++++++++
 lib/galaxy/jobs/__init__.py                | 21 +++++++++------------
 lib/galaxy/metadata/set_metadata.py        |  4 ++--
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py
index 979636562ab..71e221ab783 100644
--- a/lib/galaxy/job_execution/output_collect.py
+++ b/lib/galaxy/job_execution/output_collect.py
@@ -646,3 +646,11 @@ def collect_extra_files(object_store, dataset, job_working_directory):
                 dataset.set_size()
         except Exception as e:
             log.warning('Unable to generate primary composite file automatically for %s: %s', dataset.dataset.id, unicodify(e))
+
+
+def collect_shrinked_content_from_path(path):
+    try:
+        with open(path, 'rb') as fh:
+            return galaxy.util.shrink_and_unicodify(fh.read().strip())
+    except FileNotFoundError:
+        return None
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index d10676d6518..7bcdddc26ca 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -24,7 +24,6 @@ import packaging.version
 import yaml
 from pulsar.client.staging import COMMAND_VERSION_FILENAME
 
-import galaxy
 from galaxy import (
     model,
     util,
@@ -40,7 +39,10 @@ from galaxy.job_execution.datasets import (
     OutputsToWorkingDirectoryPathRewriter,
     TaskPathRewriter
 )
-from galaxy.job_execution.output_collect import collect_extra_files
+from galaxy.job_execution.output_collect import (
+    collect_extra_files,
+    collect_shrinked_content_from_path,
+)
 from galaxy.job_execution.setup import (  # noqa: F401
     create_working_directory_for_job,
     ensure_configs_directory,
@@ -1648,16 +1650,6 @@ class JobWrapper(HasResourceParameters):
         else:
             final_job_state = job.states.ERROR
 
-        if self.tool.version_string_cmd:
-            version_filename = self.get_version_string_path()
-            # TODO: Remove in Galaxy 20.XX, for running jobs at GX upgrade
-            if not os.path.exists(version_filename):
-                version_filename = self.get_version_string_path_legacy()
-            if os.path.exists(version_filename):
-                with open(version_filename, 'rb') as fh:
-                    self.version_string = galaxy.util.shrink_and_unicodify(fh.read())
-                os.unlink(version_filename)
-
         outputs_to_working_directory = util.asbool(self.get_destination_configuration("outputs_to_working_directory", False))
         if not extended_metadata and outputs_to_working_directory and not self.__link_file_check():
             # output will be moved by job if metadata_strategy is extended_metadata, so skip moving here
@@ -1686,6 +1678,11 @@ class JobWrapper(HasResourceParameters):
             except Exception:
                 log.exception(f"problem importing job outputs. stdout [{job.stdout}] stderr [{job.stderr}]")
                 raise
+        else:
+            if self.tool.version_string_cmd:
+                version_filename = self.get_version_string_path()
+                self.version_string = collect_shrinked_content_from_path(version_filename)
+
         output_dataset_associations = job.output_datasets + job.output_library_datasets
         for dataset_assoc in output_dataset_associations:
             context = self.get_dataset_finish_context(job_context, dataset_assoc)
diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index 9de6af4fd08..3047645a73e 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -30,6 +30,7 @@ from galaxy.job_execution.output_collect import (
     collect_dynamic_outputs,
     collect_extra_files,
     collect_primary_datasets,
+    collect_shrinked_content_from_path,
     default_exit_code_file,
     read_exit_code_from,
     SessionlessJobContext,
@@ -196,8 +197,7 @@ def set_metadata_portable():
             final_job_state = Job.states.ERROR
 
         version_string_path = os.path.join('outputs', COMMAND_VERSION_FILENAME)
-        if os.path.exists(version_string_path):
-            version_string = open(version_string_path).read().strip()
+        version_string = collect_shrinked_content_from_path(version_string_path)
 
         expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
 

From 67341aa2b4004eb617490db9931a73eee7368b1a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 11 Jul 2021 21:06:14 -0400
Subject: [PATCH 074/452] Run prettier

---
 client/src/components/Tool/ToolForm.test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolForm.test.js b/client/src/components/Tool/ToolForm.test.js
index 4845fe1e9a8..15cf8df15f7 100644
--- a/client/src/components/Tool/ToolForm.test.js
+++ b/client/src/components/Tool/ToolForm.test.js
@@ -40,7 +40,7 @@ describe("ToolForm", () => {
                 UserHistories: MockCurrentHistory({ id: "fakehistory" }),
                 ConfigProvider: MockConfigProvider({ id: "fakeconfig" }),
                 FormDisplay: true,
-            }
+            },
         });
     });
 

From f0c47158392fe1c5199e0b86e564e40d6531780f Mon Sep 17 00:00:00 2001
From: Cameron Hyde <c.hyde@qcif.edu.au>
Date: Mon, 12 Jul 2021 15:41:09 +1000
Subject: [PATCH 075/452] Lint for whitespace in tool XML

---
 lib/galaxy/tool_util/linters/general.py  | 19 +++++++++-
 test/unit/tool_util/test_tool_linters.py | 47 ++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tool_util/linters/general.py b/lib/galaxy/tool_util/linters/general.py
index c2624daca81..7641caff6b0 100644
--- a/lib/galaxy/tool_util/linters/general.py
+++ b/lib/galaxy/tool_util/linters/general.py
@@ -1,4 +1,4 @@
-"""This module contains a linting functions for general aspects of the tool."""
+"""This module contains linting functions for general aspects of the tool."""
 import re
 
 import packaging.version
@@ -18,6 +18,10 @@ PROFILE_INFO_DEFAULT_MSG = "Tool targets 16.01 Galaxy profile."
 PROFILE_INFO_SPECIFIED_MSG = "Tool specifies profile version [%s]."
 PROFILE_INVALID_MSG = "Tool specifies an invalid profile version [%s]."
 
+WARN_WHITESPACE_MSG = "%s contains whitespace, this may cause errors: [%s]."
+WARN_ID_WHITESPACE_MSG = (
+    "Tool ID contains whitespace - this is discouraged: [%s].")
+
 lint_tool_types = ["*"]
 
 
@@ -29,12 +33,16 @@ def lint_general(tool_source, lint_ctx):
         lint_ctx.error(ERROR_VERSION_MSG)
     elif isinstance(parsed_version, packaging.version.LegacyVersion):
         lint_ctx.warn(WARN_VERSION_MSG % version)
+    elif version != version.strip():
+        lint_ctx.warn(WARN_WHITESPACE_MSG % ('Tool version', version))
     else:
         lint_ctx.valid(VALID_VERSION_MSG % version)
 
     name = tool_source.parse_name()
     if not name:
         lint_ctx.error(ERROR_NAME_MSG)
+    elif name != name.strip():
+        lint_ctx.warn(WARN_WHITESPACE_MSG % ('Tool name', name))
     else:
         lint_ctx.valid(VALID_NAME_MSG % name)
 
@@ -53,5 +61,12 @@ def lint_general(tool_source, lint_ctx):
     else:
         lint_ctx.valid(PROFILE_INFO_SPECIFIED_MSG % profile)
 
+    requirements, containers = tool_source.parse_requirements_and_containers()
+    for r in requirements:
+        # Warn requirement attributes with leading/trailing whitespace:
+        if r.version != r.version.strip():
+            lint_ctx.warn(
+                WARN_WHITESPACE_MSG % ('Requirement version', r.version))
+
     if re.search(r"\s", tool_id):
-        lint_ctx.warn("Tool id contains a space - this is discouraged.")
+        lint_ctx.warn(WARN_ID_WHITESPACE_MSG % tool_id)
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index eb978887269..f98d9a0ce38 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -1,10 +1,21 @@
 import pytest
 
 from galaxy.tool_util.lint import LintContext
-from galaxy.tool_util.linters import inputs
+from galaxy.tool_util.linters import inputs, general
+from galaxy.tool_util.parser.xml import XmlToolSource
 from galaxy.util import etree
 
 
+def lint_general(xml_tree, lint_ctx):
+    """Wrap calling of lint_general to provide XmlToolSource argument.
+
+    This allows general.lint_general to be called with the other linters which
+    take an XmlTree as an argument.
+    """
+    tool_source = XmlToolSource(xml_tree)
+    return general.lint_general(tool_source, lint_ctx)
+
+
 NO_SECTIONS_XML = """
 <tool name="BWA Mapper" id="bwa" version="1.0.1" is_multi_byte="true" display_interface="true" require_login="true" hidden="true">
     <description>The BWA Mapper</description>
@@ -93,6 +104,23 @@ SELECT_OPTION_DEFINITIONS = """
 </tool>
 """
 
+WHITESPACE_IN_VERSIONS_AND_NAMES = """
+<tool name=" BWA Mapper " id="bwa tool" version=" 1.0.1 " is_multi_byte="true" display_interface="true" require_login="true" hidden="true">
+    <description>The BWA Mapper</description>
+    <requirements>
+        <requirement type="package" version=" 1.2.5 "> bwa </requirement>
+    </requirements>
+    <version_command interpreter="python">bwa.py --version</version_command>
+    <inputs>
+        <param name="select_fd_op" type="select">
+            <options from_dataset="xyz"/>
+            <options from_data_table="xyz"/>
+            <option value="x">x</option>
+        </param>
+    </inputs>
+</tool>
+"""
+
 TESTS = [
     (
         NO_SECTIONS_XML, inputs.lint_inputs,
@@ -142,6 +170,15 @@ TESTS = [
             and "Select parameter [select_fd_fdt] options uses 'from_dataset' and 'from_data_table' attribute." in x.error_messages
             and len(x.warn_messages) == 0 and len(x.error_messages) == 5
     ),
+    (
+        WHITESPACE_IN_VERSIONS_AND_NAMES, lint_general,
+        lambda x:
+            "Tool version contains whitespace, this may cause errors: [ 1.0.1 ]." in x.warn_messages
+            and "Tool name contains whitespace, this may cause errors: [ BWA Mapper ]." in x.warn_messages
+            and "Requirement version contains whitespace, this may cause errors: [ 1.2.5 ]." in x.warn_messages
+            and "Tool ID contains whitespace - this is discouraged: [bwa tool]."
+            and len(x.warn_messages) == 4 and len(x.error_messages) == 0
+    ),
 ]
 
 TEST_IDS = [
@@ -150,7 +187,8 @@ TEST_IDS = [
     'radio select incompatibilities',
     'select duplicated options',
     'select deprecations',
-    'select option definitions'
+    'select option definitions',
+    'hazardous whitespace',
 ]
 
 
@@ -159,4 +197,7 @@ def test_tool_xml(tool_xml, lint_func, assert_func):
     lint_ctx = LintContext('all')
     tree = etree.ElementTree(element=etree.fromstring(tool_xml))
     lint_ctx.lint(name="test_lint", lint_func=lint_func, lint_target=tree)
-    assert assert_func(lint_ctx), f"Warnings: {lint_ctx.warn_messages}\nErrors: {lint_ctx.error_messages}"
+    assert assert_func(lint_ctx), (
+        f"Warnings: {lint_ctx.warn_messages}\n"
+        f"Errors: {lint_ctx.error_messages}"
+    )

From 4eda692eae36c658f67ea68c84bc0545e2feb2f2 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 12 Jul 2021 16:25:34 +0200
Subject: [PATCH 076/452] Restore Grep1 version 1.0.1

The select option value for inverting results changed from `true` to
`-v`, so if you don't update your workflow you'll run grep without
the `-v` flag.
I don't think there's a way around this change (that is now part of
21.05 for a while, so we can't just switch the option value back).

Addresses https://github.com/galaxyproject/galaxy/issues/12249.
We should also require an additional confirmation.
---
 lib/galaxy/config/sample/tool_conf.xml.sample |   1 +
 tools/filters/grep.py                         | 137 ++++++++++++++++++
 tools/filters/grep_1.0.1.xml                  |  85 +++++++++++
 3 files changed, 223 insertions(+)
 create mode 100644 tools/filters/grep.py
 create mode 100644 tools/filters/grep_1.0.1.xml

diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample
index 286a0c9f1c2..1d14e0240f1 100644
--- a/lib/galaxy/config/sample/tool_conf.xml.sample
+++ b/lib/galaxy/config/sample/tool_conf.xml.sample
@@ -78,6 +78,7 @@
     <tool file="stats/filtering.xml" />
     <tool file="filters/sorter.xml" />
     <tool file="filters/grep.xml" />
+    <tool file="filters/grep_1.0.1.xml"/>
     <label id="gff" text="GFF" />
     <tool file="filters/gff/extract_GFF_Features.xml" />
     <tool file="filters/gff/gff_filter_by_attribute.xml" />
diff --git a/tools/filters/grep.py b/tools/filters/grep.py
new file mode 100644
index 00000000000..2cd8e0c4d3d
--- /dev/null
+++ b/tools/filters/grep.py
@@ -0,0 +1,137 @@
+# Filename: grep.py
+# Author: Ian N. Schenck
+# Version: 8/23/2005
+#
+# This script accepts regular expressions, as well as an "invert"
+# option, and applies the regular expression using grep.  This wrapper
+# provides security and pipeline.
+#
+# Grep is launched based on these inputs:
+# -i Input file
+# -o Output file
+# -pattern RegEx pattern
+# -v true or false (output NON-matching lines)
+from __future__ import print_function
+
+import os
+import re
+import subprocess
+import sys
+from subprocess import PIPE, Popen
+from tempfile import NamedTemporaryFile
+
+
+# This function is exceedingly useful, perhaps package for reuse?
+def getopts(argv):
+    opts = {}
+    while argv:
+        if argv[0][0] == '-':
+            opts[argv[0]] = argv[1]
+            argv = argv[2:]
+        else:
+            argv = argv[1:]
+    return opts
+
+
+def main():
+    args = sys.argv[1:]
+
+    try:
+        opts = getopts(args)
+    except IndexError:
+        print("Usage:")
+        print(" -i Input file")
+        print(" -o Output file")
+        print(" -pattern RegEx pattern")
+        print(" -v true or false (Invert match)")
+        return 0
+
+    outputfile = opts.get("-o")
+    if outputfile is None:
+        print("No output file specified.")
+        return -1
+
+    inputfile = opts.get("-i")
+    if inputfile is None:
+        print("No input file specified.")
+        return -2
+
+    invert = opts.get("-v")
+    if invert is None:
+        print("Match style (Invert or normal) not specified.")
+        return -3
+
+    pattern = opts.get("-pattern")
+    if pattern is None:
+        print("RegEx pattern not specified.")
+        return -4
+
+    # All inputs have been specified at this point, now validate.
+
+    # replace if input has been escaped, remove sq
+    # characters that are allowed but need to be escaped
+    mapped_chars = {'>' : '__gt__',
+                    '<' : '__lt__',
+                    '\'': '__sq__',
+                    '"' : '__dq__',
+                    '[' : '__ob__',
+                    ']' : '__cb__',
+                    '{' : '__oc__',
+                    '}' : '__cc__'}
+
+    # with new sanitizing we only need to replace for single quote,
+    # but this needs to remain for backwards compatibility
+    for key, value in mapped_chars.items():
+        pattern = pattern.replace(value, key)
+
+    # match filename and invert flag
+    fileRegEx = re.compile(r"^[A-Za-z0-9./\-_]+$")
+    invertRegEx = re.compile(r"(true)|(false)")
+
+    # verify that filename and inversion flag are in the correct format
+    if not fileRegEx.match(outputfile):
+        print("Illegal output filename.")
+        return -5
+    if not fileRegEx.match(inputfile):
+        print("Illegal input filename.")
+        return -6
+    if not invertRegEx.match(invert):
+        print("Illegal invert option.")
+        return -7
+
+    # invert grep search?
+    if invert == "true":
+        invertflag = "-v"
+        print("Not matching pattern: %s" % pattern)
+    else:
+        invertflag = ""
+        print("Matching pattern: %s" % pattern)
+
+    # set version flag
+    versionflag = "-P"
+
+    # MacOS 10.8.2 does not support -P option for perl-regex anymore
+    versionmatch = Popen("grep -V | grep 'BSD'", shell=True, stdout=PIPE).communicate()[0]
+    if versionmatch:
+        versionflag = "-E"
+
+    # create temp file holding pattern
+    # by using a file to hold the pattern, we don't have worry about sanitizing grep commandline and can include single quotes in pattern
+    pattern_file_name = NamedTemporaryFile().name
+    open(pattern_file_name, 'w').write(pattern)
+
+    # generate grep command
+    commandline = "grep %s %s -f %s %s > %s" % (versionflag, invertflag, pattern_file_name, inputfile, outputfile)
+
+    # run grep
+    errorcode = subprocess.call(commandline, shell=True)
+
+    # remove temp pattern file
+    os.unlink(pattern_file_name)
+
+    # return error code
+    return errorcode
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/filters/grep_1.0.1.xml b/tools/filters/grep_1.0.1.xml
new file mode 100644
index 00000000000..c904bb4c769
--- /dev/null
+++ b/tools/filters/grep_1.0.1.xml
@@ -0,0 +1,85 @@
+<tool id="Grep1" name="Select" version="1.0.1">
+  <description>lines that match an expression</description>
+  <edam_operations>
+    <edam_operation>operation_3695</edam_operation>
+  </edam_operations>
+  <command interpreter="python">grep.py -i $input -o $out_file1 -pattern '$pattern' -v $invert</command>
+  <inputs>
+    <param format="txt" name="input" type="data" label="Select lines from"/>
+    <param name="invert" type="select" label="that">
+      <option value="false">Matching</option>
+      <option value="true">NOT Matching</option>
+    </param>
+    <param name="pattern" type="text" value="^chr([0-9A-Za-z])+" label="the pattern" help="here you can enter text or regular expression (for syntax check lower part of this frame)">
+      <sanitizer>
+        <valid initial="string.printable">
+         <remove value="&apos;"/>
+        </valid>
+        <mapping initial="none">
+          <add source="&apos;" target="__sq__"/>
+        </mapping>
+      </sanitizer>
+    </param>
+  </inputs>
+  <outputs>
+    <data format="input" name="out_file1" metadata_source="input"/>
+  </outputs>
+  <tests>
+    <test>
+      <param name="input" value="1.bed"/>
+      <param name="invert" value="false"/>
+      <param name="pattern" value="^chr[0-9]*"/>
+      <output name="out_file1" file="fs-grep.dat"/>
+    </test>
+  </tests>
+  <help>
+
+.. class:: infomark
+
+**TIP:** If your data is not TAB delimited, use *Text Manipulation-&gt;Convert*
+
+-----
+
+**Syntax**
+
+The select tool searches the data for lines containing or not containing a match to the given pattern. Regular Expression is introduced in this tool. A Regular Expression is a pattern describing a certain amount of text.
+
+- **( ) { } [ ] . * ? + \ ^ $** are all special characters. **\\** can be used to "escape" a special character, allowing that special character to be searched for.
+- **\\A** matches the beginning of a string(but not an internal line).
+- **\\d** matches a digit, same as [0-9].
+- **\\D** matches a non-digit.
+- **\\s** matches a whitespace character.
+- **\\S** matches anything BUT a whitespace.
+- **\\t** matches a tab.
+- **\\w** matches an alphanumeric character.
+- **\\W** matches anything but an alphanumeric character.
+- **(** .. **)** groups a particular pattern.
+- **\\Z** matches the end of a string(but not a internal line).
+- **{** n or n, or n,m **}** specifies an expected number of repetitions of the preceding pattern.
+
+  - **{n}** The preceding item is matched exactly n times.
+  - **{n,}** The preceding item is matched n or more times.
+  - **{n,m}** The preceding item is matched at least n times but not more than m times.
+
+- **[** ... **]** creates a character class. Within the brackets, single characters can be placed. A dash (-) may be used to indicate a range such as **a-z**.
+- **.** Matches any single character except a newline.
+- ***** The preceding item will be matched zero or more times.
+- **?** The preceding item is optional and matched at most once.
+- **+** The preceding item will be matched one or more times.
+- **^** has two meaning:
+  - matches the beginning of a line or string.
+  - indicates negation in a character class. For example, [^...] matches every character except the ones inside brackets.
+- **$** matches the end of a line or string.
+- **\|** Separates alternate possibilities.
+
+-----
+
+**Example**
+
+- **^chr([0-9A-Za-z])+** would match lines that begin with chromosomes, such as lines in a BED format file.
+- **(ACGT){1,5}** would match at least 1 "ACGT" and at most 5 "ACGT" consecutively.
+- **([^,][0-9]{1,3})(,[0-9]{3})\*** would match a large integer that is properly separated with commas such as 23,078,651.
+- **(abc)|(def)** would match either "abc" or "def".
+- **^\\W+#** would match any line that is a comment.
+</help>
+</tool>

From dcac61fd1dcdd941d0e5ddd36d0e64ebd7869d6b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 12 Jul 2021 18:29:19 +0200
Subject: [PATCH 077/452] Fix linting in grep.py script

---
 tools/filters/grep.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tools/filters/grep.py b/tools/filters/grep.py
index 2cd8e0c4d3d..fad39281aac 100644
--- a/tools/filters/grep.py
+++ b/tools/filters/grep.py
@@ -70,14 +70,14 @@ def main():
 
     # replace if input has been escaped, remove sq
     # characters that are allowed but need to be escaped
-    mapped_chars = {'>' : '__gt__',
-                    '<' : '__lt__',
+    mapped_chars = {'>': '__gt__',
+                    '<': '__lt__',
                     '\'': '__sq__',
-                    '"' : '__dq__',
-                    '[' : '__ob__',
-                    ']' : '__cb__',
-                    '{' : '__oc__',
-                    '}' : '__cc__'}
+                    '"': '__dq__',
+                    '[': '__ob__',
+                    ']': '__cb__',
+                    '{': '__oc__',
+                    '}': '__cc__'}
 
     # with new sanitizing we only need to replace for single quote,
     # but this needs to remain for backwards compatibility

From a8a993e32618165ee7cf98bd4458ea78ca9096c3 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 12 Jul 2021 22:00:49 +0200
Subject: [PATCH 078/452] Get tool from toolbox, which gets the latest version

---
 lib/galaxy/tools/search/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py
index ac695b5d114..430582d4cf7 100644
--- a/lib/galaxy/tools/search/__init__.py
+++ b/lib/galaxy/tools/search/__init__.py
@@ -101,7 +101,7 @@ class ToolBoxSearch:
             for tool_id in tool_ids_to_remove:
                 writer.delete_by_term('id', tool_id)
             for tool_id in tool_cache._new_tool_ids - indexed_tool_ids:
-                tool = tool_cache.get_tool_by_id(tool_id)
+                tool = self.toolbox.get_tool(tool_id)
                 if tool and tool.is_latest_version:
                     if tool.hidden:
                         # we check if there is an older tool we can return

From 499507877f4a5c2e3507d07b5ff38ed21a36c559 Mon Sep 17 00:00:00 2001
From: Cameron Hyde <c.hyde@qcif.edu.au>
Date: Tue, 13 Jul 2021 07:38:27 +1000
Subject: [PATCH 079/452] Fix import order

---
 test/unit/tool_util/test_tool_linters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index f98d9a0ce38..31b2e082ba5 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -1,7 +1,7 @@
 import pytest
 
 from galaxy.tool_util.lint import LintContext
-from galaxy.tool_util.linters import inputs, general
+from galaxy.tool_util.linters import general, inputs
 from galaxy.tool_util.parser.xml import XmlToolSource
 from galaxy.util import etree
 

From ac29c883a04334e5a69ec3f482ddf1ae09b21d3f Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 5 Jul 2021 21:16:51 +0200
Subject: [PATCH 080/452] Use relative path in recursive discover_datasets tag

I've added a new option for this. While this might be the intuitive
default I don't want to break existing tools, so the new option is
`match_relative_path`.
---
 lib/galaxy/job_execution/output_collect.py    | 14 +++-
 .../tool_util/parser/output_collection_def.py |  2 +
 lib/galaxy/tool_util/xsd/galaxy.xsd           | 10 +++
 .../functional/tools/multi_output_recurse.xml | 78 +++++++++++++------
 4 files changed, 75 insertions(+), 29 deletions(-)

diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py
index 1c9b56b0f78..e95d13832b9 100644
--- a/lib/galaxy/job_execution/output_collect.py
+++ b/lib/galaxy/job_execution/output_collect.py
@@ -489,24 +489,27 @@ def walk_over_file_collectors(extra_file_collectors, job_working_directory, matc
             yield match, extra_file_collector
 
 
-def walk_over_extra_files(target_dir, extra_file_collector, job_working_directory, matchable):
+def walk_over_extra_files(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths=None):
     """
     Walks through all files in a given directory, and returns all files that
     match the given collector's match criteria. If the collector has the
     recurse flag enabled, will also recursively descend into child folders.
     """
     matches = []
+    parent_paths = parent_paths or []
     directory = discover_target_directory(target_dir, job_working_directory)
     if os.path.isdir(directory):
         for filename in os.listdir(directory):
             path = os.path.join(directory, filename)
             if os.path.isdir(path):
                 if extra_file_collector.recurse:
+                    new_parent_paths = parent_paths[:]
+                    new_parent_paths.append(filename)
                     # The current directory is already validated, so use that as the next job_working_directory when recursing
-                    for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable):
+                    for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths):
                         yield match
             else:
-                match = extra_file_collector.match(matchable, filename, path=path)
+                match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths)
                 if match:
                     matches.append(match)
 
@@ -553,6 +556,7 @@ class DatasetCollector:
         self.directory = dataset_collection_description.directory
         self.assign_primary_output = dataset_collection_description.assign_primary_output
         self.recurse = dataset_collection_description.recurse
+        self.match_relative_path = dataset_collection_description.match_relative_path
 
     def _pattern_for_dataset(self, dataset_instance=None):
         token_replacement = r'\d+'
@@ -560,8 +564,10 @@ class DatasetCollector:
             token_replacement = str(dataset_instance.id)
         return self.pattern.replace(DATASET_ID_TOKEN, token_replacement)
 
-    def match(self, dataset_instance, filename, path=None):
+    def match(self, dataset_instance, filename, path=None, parent_paths=None):
         pattern = self._pattern_for_dataset(dataset_instance)
+        if self.match_relative_path and parent_paths:
+            filename = os.path.join(*parent_paths, filename)
         re_match = re.match(pattern, filename)
         match_object = None
         if re_match:
diff --git a/lib/galaxy/tool_util/parser/output_collection_def.py b/lib/galaxy/tool_util/parser/output_collection_def.py
index 17d5f7130b9..f7a9e7a40e9 100644
--- a/lib/galaxy/tool_util/parser/output_collection_def.py
+++ b/lib/galaxy/tool_util/parser/output_collection_def.py
@@ -80,6 +80,7 @@ class DatasetCollectionDescription:
         self.assign_primary_output = asbool(kwargs.get('assign_primary_output', False))
         self.directory = kwargs.get("directory", None)
         self.recurse = False
+        self.match_relative_path = kwargs.get("match_relative_path", False)
 
     def to_dict(self):
         return {
@@ -106,6 +107,7 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription):
         super().__init__(**kwargs)
         pattern = kwargs.get("pattern", "__default__")
         self.recurse = asbool(kwargs.get("recurse", False))
+        self.match_relative_path = asbool(kwargs.get("match_relative_path", False))
         if pattern in NAMED_PATTERNS:
             pattern = NAMED_PATTERNS.get(pattern)
         self.pattern = pattern
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index b39a34e8fe7..216c9c61c82 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4646,6 +4646,11 @@ More information can be found on Planemo's documentation for
         <xs:documentation xml:lang="en">Indicates that the specified directory should be searched recursively for matching files.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="match_relative_path" type="xs:boolean" use="optional">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Indicates that the entire path of the discovered dataset relative to the specified directory should be available for matching patterns.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="format" type="xs:string" use="optional">
       <xs:annotation>
         <xs:documentation xml:lang="en">Format (or datatype) of discovered datasets (an alias with ``ext``).</xs:documentation>
@@ -4709,6 +4714,11 @@ Galaxy, including:
         <xs:documentation xml:lang="en">Indicates that the specified directory should be searched recursively for matching files.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="match_relative_path" type="xs:boolean" use="optional">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Indicates that the entire path of the discovered dataset relative to the specified directory should be available for matching patterns.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="format" type="xs:string" use="optional">
       <xs:annotation>
         <xs:documentation xml:lang="en">Format (or datatype) of discovered datasets (an alias with ``ext``).</xs:documentation>
diff --git a/test/functional/tools/multi_output_recurse.xml b/test/functional/tools/multi_output_recurse.xml
index 7798bc359d4..98e9c4ff12d 100644
--- a/test/functional/tools/multi_output_recurse.xml
+++ b/test/functional/tools/multi_output_recurse.xml
@@ -1,66 +1,94 @@
 <tool id="multi_output_recurse" name="multi_output_recurse" version="0.1.0" profile="18.01">
   <command>
-    echo "Hello" > $report;
+    echo "Hello" &gt; $report;
     mkdir subdir1;
-    echo "This" > subdir1/this.txt;
-    echo "That" > subdir1/that.txt;
+    echo "This" &gt; subdir1/this.txt;
+    echo "That" &gt; subdir1/that.txt;
     mkdir subdir2;
-    echo "1" > subdir2/CUSTOM_1.txt;
-    echo "2" > subdir2/CUSTOM_2.txt;
+    echo "1" &gt; subdir2/CUSTOM_1.txt;
+    echo "2" &gt; subdir2/CUSTOM_2.txt;
     mkdir subdir3;
-    echo "Foo" > subdir3/Foo;
+    echo "Foo" &gt; subdir3/Foo;
     mkdir subdir3/nested1;
-    echo "Bar" > subdir3/nested1/bar.txt;
-    echo "Hello" > subdir3/nested1/hello;
-    echo "1" > sample1.report.txt;
-    echo "2" > sample2.report.txt;
+    echo "Bar" &gt; subdir3/nested1/bar.txt;
+    echo "Hello" &gt; subdir3/nested1/hello;
+    echo "1" &gt; sample1.report.txt;
+    echo "2" &gt; sample2.report.txt;
   </command>
   <inputs>
-    <param name="num_param" type="integer" value="7" />
-    <param name="input" type="data" />
+    <param name="num_param" type="integer" value="7"/>
+    <param name="input" type="data"/>
   </inputs>
   <outputs>
     <data auto_format="true" name="report">
-      <discover_datasets pattern="__designation__" directory="." visible="true" recurse="true" />
+      <discover_datasets pattern="__designation__" directory="." visible="true" recurse="true"/>
     </data>
     <data auto_format="true" name="report_optional">
-      <discover_datasets pattern="__designation__" directory="i_do_not_exist" visible="true" recurse="true" />
+      <discover_datasets pattern="__designation__" directory="i_do_not_exist" visible="true" recurse="true"/>
+    </data>
+    <data auto_format="true" name="report_match_relative_path">
+      <discover_datasets pattern="__designation__" directory="." visible="true" recurse="true" match_relative_path="true"/>
     </data>
   </outputs>
   <tests>
     <test>
-      <param name="num_param" value="7" />
+      <param name="num_param" value="7"/>
       <param name="input" ftype="txt" value="simple_line.txt"/>
       <output name="report">
         <assert_contents>
-          <has_line line="Hello" />
+          <has_line line="Hello"/>
         </assert_contents>
         <discovered_dataset designation="this.txt">
-          <assert_contents><has_line line="This" /></assert_contents>
+          <assert_contents>
+            <has_line line="This"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="that.txt">
-          <assert_contents><has_line line="That" /></assert_contents>
+          <assert_contents>
+            <has_line line="That"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="CUSTOM_1.txt">
-          <assert_contents><has_line line="1" /></assert_contents>
+          <assert_contents>
+            <has_line line="1"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="CUSTOM_2.txt">
-          <assert_contents><has_line line="2" /></assert_contents>
+          <assert_contents>
+            <has_line line="2"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="Foo">
-          <assert_contents><has_line line="Foo" /></assert_contents>
+          <assert_contents>
+            <has_line line="Foo"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="bar.txt">
-          <assert_contents><has_line line="Bar" /></assert_contents>
+          <assert_contents>
+            <has_line line="Bar"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="hello">
-          <assert_contents><has_line line="Hello" /></assert_contents>
+          <assert_contents>
+            <has_line line="Hello"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="sample1.report.txt">
-          <assert_contents><has_line line="1" /></assert_contents>
+          <assert_contents>
+            <has_line line="1"/>
+          </assert_contents>
         </discovered_dataset>
         <discovered_dataset designation="sample2.report.txt">
-          <assert_contents><has_line line="2" /></assert_contents>
+          <assert_contents>
+            <has_line line="2"/>
+          </assert_contents>
+        </discovered_dataset>
+      </output>
+      <output name="report_match_relative_path">
+        <discovered_dataset designation="subdir3/nested1/bar.txt">
+          <assert_contents>
+            <has_line line="Bar"/>
+          </assert_contents>
         </discovered_dataset>
       </output>
     </test>

From 73f3c442fef9de441023bd2c8ed1156bd2fe1a84 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 13 Jul 2021 17:30:56 +0200
Subject: [PATCH 081/452] Separate test cases

---
 .../functional/tools/multi_output_recurse.xml | 35 ++++++++++---------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/test/functional/tools/multi_output_recurse.xml b/test/functional/tools/multi_output_recurse.xml
index 98e9c4ff12d..e8be0f3e8e8 100644
--- a/test/functional/tools/multi_output_recurse.xml
+++ b/test/functional/tools/multi_output_recurse.xml
@@ -1,39 +1,39 @@
 <tool id="multi_output_recurse" name="multi_output_recurse" version="0.1.0" profile="18.01">
-  <command>
-    echo "Hello" &gt; $report;
+  <command><![CDATA[
+    echo "Hello" > '$report';
     mkdir subdir1;
-    echo "This" &gt; subdir1/this.txt;
-    echo "That" &gt; subdir1/that.txt;
+    echo "This" > subdir1/this.txt;
+    echo "That" > subdir1/that.txt;
     mkdir subdir2;
-    echo "1" &gt; subdir2/CUSTOM_1.txt;
-    echo "2" &gt; subdir2/CUSTOM_2.txt;
+    echo "1" > subdir2/CUSTOM_1.txt;
+    echo "2" > subdir2/CUSTOM_2.txt;
     mkdir subdir3;
-    echo "Foo" &gt; subdir3/Foo;
+    echo "Foo" > subdir3/Foo;
     mkdir subdir3/nested1;
-    echo "Bar" &gt; subdir3/nested1/bar.txt;
-    echo "Hello" &gt; subdir3/nested1/hello;
-    echo "1" &gt; sample1.report.txt;
-    echo "2" &gt; sample2.report.txt;
-  </command>
+    echo "Bar" > subdir3/nested1/bar.txt;
+    echo "Hello" > subdir3/nested1/hello;
+    echo "1" > sample1.report.txt;
+    echo "2" > sample2.report.txt;
+    ]]></command>
   <inputs>
-    <param name="num_param" type="integer" value="7"/>
-    <param name="input" type="data"/>
+    <param type="boolean" name="match_relative"/>
   </inputs>
   <outputs>
     <data auto_format="true" name="report">
+      <filter>not bool(match_relative)</filter>
       <discover_datasets pattern="__designation__" directory="." visible="true" recurse="true"/>
     </data>
     <data auto_format="true" name="report_optional">
+      <filter>not bool(match_relative)</filter>
       <discover_datasets pattern="__designation__" directory="i_do_not_exist" visible="true" recurse="true"/>
     </data>
     <data auto_format="true" name="report_match_relative_path">
+      <filter>bool(match_relative)</filter>
       <discover_datasets pattern="__designation__" directory="." visible="true" recurse="true" match_relative_path="true"/>
     </data>
   </outputs>
   <tests>
     <test>
-      <param name="num_param" value="7"/>
-      <param name="input" ftype="txt" value="simple_line.txt"/>
       <output name="report">
         <assert_contents>
           <has_line line="Hello"/>
@@ -84,6 +84,9 @@
           </assert_contents>
         </discovered_dataset>
       </output>
+    </test>
+    <test>
+      <param name="match_relative" value="true"/>
       <output name="report_match_relative_path">
         <discovered_dataset designation="subdir3/nested1/bar.txt">
           <assert_contents>

From 3ab2aa4b2e8516f92cf3c8cf574d8afc338ee1ce Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 2 Jul 2021 21:45:21 +0200
Subject: [PATCH 082/452] Add OrderParamField and HistoryFilterQueryParams

This allows to use different default ordering values for each kind of item that uses the FilterQueryParams.
---
 lib/galaxy/schema/__init__.py              |  4 ++--
 lib/galaxy/schema/fields.py                | 14 ++++++++++++++
 lib/galaxy/webapps/galaxy/api/histories.py | 12 +++++++++---
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/schema/__init__.py b/lib/galaxy/schema/__init__.py
index 06125eee4a8..1159a5a9ddd 100644
--- a/lib/galaxy/schema/__init__.py
+++ b/lib/galaxy/schema/__init__.py
@@ -47,12 +47,12 @@ class FilterQueryParams(BaseModel):
         description="The maximum number of items to return.",
     )
     order: Optional[str] = Field(
-        default="create_time-dsc",
+        default=None,
         title="Order",
         description=(
             "String containing one of the valid ordering attributes followed (optionally) "
             "by '-asc' or '-dsc' for ascending and descending order respectively. "
-                "Orders can be stacked as a comma-separated list of values."
+            "Orders can be stacked as a comma-separated list of values."
         ),
         example="name-dsc,create_time",
     )
diff --git a/lib/galaxy/schema/fields.py b/lib/galaxy/schema/fields.py
index 04a1a4fa4fa..f9d2d365667 100644
--- a/lib/galaxy/schema/fields.py
+++ b/lib/galaxy/schema/fields.py
@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from pydantic import (
     Field,
@@ -62,3 +63,16 @@ def ModelClassField(class_name: str) -> str:
         description="The name of the database model class.",
         const=True,  # Make this field constant
     )
+
+
+def OrderParamField(default_order: str) -> Optional[str]:
+    return Field(
+        default=default_order,
+        title="Order",
+        description=(
+            "String containing one of the valid ordering attributes followed (optionally) "
+            "by '-asc' or '-dsc' for ascending and descending order respectively. "
+            "Orders can be stacked as a comma-separated list of values."
+        ),
+        example="name-dsc,create_time",
+    )
diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py
index 35e557f1d56..caf333b783e 100644
--- a/lib/galaxy/webapps/galaxy/api/histories.py
+++ b/lib/galaxy/webapps/galaxy/api/histories.py
@@ -4,6 +4,7 @@ API operations on a history.
 .. seealso:: :class:`galaxy.model.History`
 """
 import logging
+from typing import Optional
 
 from galaxy import (
     util
@@ -13,6 +14,7 @@ from galaxy.managers import (
     sharable,
 )
 from galaxy.schema import FilterQueryParams
+from galaxy.schema.fields import OrderParamField
 from galaxy.schema.schema import (
     CreateHistoryPayload,
     ExportHistoryArchivePayload,
@@ -32,6 +34,10 @@ from . import BaseGalaxyAPIController, depends
 log = logging.getLogger(__name__)
 
 
+class HistoryFilterQueryParams(FilterQueryParams):
+    order: Optional[str] = OrderParamField(default_order="create_time-dsc")
+
+
 class HistoriesController(BaseGalaxyAPIController):
     service: histories.HistoriesService = depends(histories.HistoriesService)
 
@@ -130,7 +136,7 @@ class HistoriesController(BaseGalaxyAPIController):
         deleted_only = util.string_as_bool(deleted)
         all_histories = util.string_as_bool(kwd.get('all', False))
         serialization_params = parse_serialization_params(**kwd)
-        filter_parameters = FilterQueryParams(**kwd)
+        filter_parameters = HistoryFilterQueryParams(**kwd)
         return self.service.index(trans, serialization_params, filter_parameters, deleted_only, all_histories)
 
     @expose_api_anonymous
@@ -183,7 +189,7 @@ class HistoriesController(BaseGalaxyAPIController):
         Follows the same filtering logic as the index() method above.
         """
         serialization_params = parse_serialization_params(**kwd)
-        filter_parameters = FilterQueryParams(**kwd)
+        filter_parameters = HistoryFilterQueryParams(**kwd)
         return self.service.published(trans, serialization_params, filter_parameters)
 
     @expose_api
@@ -199,7 +205,7 @@ class HistoriesController(BaseGalaxyAPIController):
         Follows the same filtering logic as the index() method above.
         """
         serialization_params = parse_serialization_params(**kwd)
-        filter_parameters = FilterQueryParams(**kwd)
+        filter_parameters = HistoryFilterQueryParams(**kwd)
         return self.service.shared_with_me(trans, serialization_params, filter_parameters)
 
     @expose_api_anonymous

From 88f477bc3b39afedf6db61e9d626b3d171e4001f Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 2 Jul 2021 21:46:20 +0200
Subject: [PATCH 083/452] Add encode function to ServiceBase

---
 lib/galaxy/managers/base.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index 969616ac464..a22c37862df 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -114,6 +114,10 @@ def decode_with_security(security: IdEncodingHelper, id: Any):
     return security.decode_id(str(id))
 
 
+def encode_with_security(security: IdEncodingHelper, id: Any):
+    return security.encode_id(id)
+
+
 def get_object(trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None):
     """
     Convenience method to get a model object with the specified checks. This is
@@ -1211,6 +1215,10 @@ class ServiceBase:
         """Decodes a previously encoded database ID."""
         return decode_with_security(self.security, id)
 
+    def encode_id(self, id):
+        """Decodes a previously encoded database ID."""
+        return encode_with_security(self.security, id)
+
     def build_order_by(self, manager: SortableManager, order_by_query: Optional[str] = None):
         """Returns an ORM compatible order_by clause using the order attribute and the given manager.
 

From 0bbce36548ff3e3d62ef213278b2365542e6649d Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 2 Jul 2021 21:47:15 +0200
Subject: [PATCH 084/452] Add SortableManager interface to
 HistoryContentsManager

---
 lib/galaxy/managers/history_contents.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py
index fb87bf0a156..e149b85f429 100644
--- a/lib/galaxy/managers/history_contents.py
+++ b/lib/galaxy/managers/history_contents.py
@@ -39,7 +39,7 @@ log = logging.getLogger(__name__)
 
 # into its own class to have it's own filters, etc.
 # TODO: but can't inherit from model manager (which assumes only one model)
-class HistoryContentsManager(containers.ContainerManagerMixin):
+class HistoryContentsManager(containers.ContainerManagerMixin, base.SortableManager):
 
     root_container_class = model.History
 

From 31d6347ef9e522c20d854e43c5ec7ac8936dad17 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 2 Jul 2021 22:04:48 +0200
Subject: [PATCH 085/452] Add HistoriesContentsService and some pydantic models

- This time, the HistoriesContentsService is living directly in the `history_contents.py` API module instead of the manager module because it would cause a circular dependency with HistoryManager otherwise. In any case, the `Service` classes are more in context with the API than with the managers so maybe its a good idea to move them all?
- The added pydantic models will be used by the index action in an attempt to combine both versions (legacy and dev) of the parameters. I will try to add the *legacy* keyword to the name of those parameters and methods that may be deprecated in the future to identify them more easily.
---
 .../webapps/galaxy/api/history_contents.py    | 120 ++++++++++++++++++
 1 file changed, 120 insertions(+)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 115c86a9578..b7845d33564 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -6,6 +6,14 @@ import json
 import logging
 import os
 import re
+from typing import (
+    Any,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Union,
+)
 
 import dateutil.parser
 
@@ -20,11 +28,26 @@ from galaxy.managers import (
     histories,
     history_contents
 )
+from galaxy.managers.base import (
+    ModelSerializer,
+    ServiceBase,
+)
 from galaxy.managers.collections_util import (
     api_payload_to_create_params,
     dictify_dataset_collection_instance,
 )
 from galaxy.managers.jobs import fetch_job_states, summarize_jobs_to_dict
+from galaxy.schema import FilterQueryParams
+from galaxy.schema.fields import (
+    EncodedDatabaseIdField,
+    Field,
+    OrderParamField,
+)
+from galaxy.schema.schema import (
+    HistoryContentType,
+    Model,
+)
+from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.util.json import safe_dumps
 from galaxy.util.zipstream import ZipstreamWrapper
 from galaxy.web import (
@@ -41,6 +64,103 @@ from . import BaseGalaxyAPIController, depends
 
 log = logging.getLogger(__name__)
 
+DatasetDetailsType = Union[Set[EncodedDatabaseIdField], Literal['all']]
+
+
+class HistoryContentsFilterQueryParams(FilterQueryParams):
+    order: Optional[str] = OrderParamField(default_order="hid-asc")
+
+
+class HistoryContentsIndexLegacyParamsBase(Model):
+    deleted: Optional[bool] = Field(
+        default=None,
+        title="Deleted",
+        description="Whether to return deleted or undeleted datasets only. Leave unset for both.",
+    )
+    visible: Optional[bool] = Field(
+        default=None,
+        title="Visible",
+        description="Whether to return visible or hidden datasets only. Leave unset for both.",
+    )
+
+
+class HistoryContentsIndexLegacyParams(HistoryContentsIndexLegacyParamsBase):
+    v: Optional[str] = Field(  # Should this be deprecated at some point and directly use the latest version by default?
+        default=None,
+        title="Version",
+        description="Only `dev` value is allowed. Set it to use the latest version of this endpoint.",
+    )
+    ids: Optional[str] = Field(
+        default=None,
+        title="IDs",
+        description=(
+            "A comma separated list of encoded `HDA` IDs. If this list is provided, only information about the "
+            "specific datasets will be provided. Also, setting this value will set `dataset_details` to `all`."
+        ),
+    )
+    types: Optional[str] = Field(
+        default=None,
+        alias="type",  # Legacy alias
+        title="Types",
+        description=(
+            "A comma separated list of kinds of contents to return "
+            "(currently just `dataset` and `dataset_collection` are available)."
+        ),
+    )
+    dataset_details: Optional[str] = Field(
+        default=None,
+        alias="details",  # Legacy alias
+        title="Dataset Details",
+        description=(
+            "A comma separated list of encoded dataset IDs that will return additional (full) details "
+            "instead of the *summary* default information."
+        ),
+    )
+
+
+class ParsedHistoryContentsLegacyParams(HistoryContentsIndexLegacyParamsBase):
+    ids: Optional[List[int]] = Field(
+        default=None,
+        title="IDs",
+        description="A list of (decoded) `HDA` IDs to return detailed information. Only these items will be returned.",
+    )
+    types: List[HistoryContentType] = Field(
+        default=[],
+        title="Types",
+        description="A list with the types of contents to return.",
+    )
+    dataset_details: Optional[DatasetDetailsType] = Field(
+        default=None,
+        title="Dataset Details",
+        description=(
+            "A set of encoded IDs for the datasets that will be returned with detailed "
+            "information or the value `all` to return (full) details for all datasets."
+        ),
+    )
+
+
+class HistoriesContentsService(ServiceBase):
+    """Common interface/service logic for interactions with histories contents in the context of the API.
+
+    Provides the logic of the actions invoked by API controllers and uses type definitions
+    and pydantic models to declare its parameters and return types.
+    """
+
+    def __init__(
+        self,
+        security: IdEncodingHelper,
+        history_manager: histories.HistoryManager,
+        history_contents_manager: history_contents.HistoryContentsManager,
+        hda_serializer: hdas.HDASerializer,
+        hdca_serializer: hdcas.HDCASerializer,
+        history_contents_filters: history_contents.HistoryContentsFilters,
+    ):
+        super().__init__(security)
+        self.history_manager = history_manager
+        self.history_contents_manager = history_contents_manager
+        self.hda_serializer = hda_serializer
+        self.hdca_serializer = hdca_serializer
+        self.history_contents_filters = history_contents_filters
 
 class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin):
     hda_manager: hdas.HDAManager = depends(hdas.HDAManager)

From 75ea86ee588896c1f5e8bbd16edd753a9cd17bd8 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 5 Jul 2021 17:31:49 +0200
Subject: [PATCH 086/452] Refactor `index` action

---
 .../webapps/galaxy/api/history_contents.py    | 346 +++++++++---------
 1 file changed, 176 insertions(+), 170 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index b7845d33564..b21b36bcc71 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -37,6 +37,11 @@ from galaxy.managers.collections_util import (
     dictify_dataset_collection_instance,
 )
 from galaxy.managers.jobs import fetch_job_states, summarize_jobs_to_dict
+from galaxy.model import (
+    History,
+    HistoryDatasetAssociation,
+    HistoryDatasetCollectionAssociation,
+)
 from galaxy.schema import FilterQueryParams
 from galaxy.schema.fields import (
     EncodedDatabaseIdField,
@@ -44,9 +49,16 @@ from galaxy.schema.fields import (
     OrderParamField,
 )
 from galaxy.schema.schema import (
+    HDABeta,
+    HDADetailed,
+    HDASummary,
+    HDCABeta,
+    HDCADetailed,
+    HDCASummary,
     HistoryContentType,
     Model,
 )
+from galaxy.schema.types import SerializationParams
 from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.util.json import safe_dumps
 from galaxy.util.zipstream import ZipstreamWrapper
@@ -60,11 +72,16 @@ from galaxy.webapps.base.controller import (
     UsesLibraryMixinItems,
     UsesTagsMixin
 )
+from galaxy.webapps.galaxy.api.common import parse_serialization_params
 from . import BaseGalaxyAPIController, depends
 
 log = logging.getLogger(__name__)
 
 DatasetDetailsType = Union[Set[EncodedDatabaseIdField], Literal['all']]
+AnyHistoryContentItem = Union[
+    HDASummary, HDADetailed, HDABeta,
+    HDCASummary, HDCADetailed, HDCABeta
+]
 
 
 class HistoryContentsFilterQueryParams(FilterQueryParams):
@@ -162,6 +179,154 @@ class HistoriesContentsService(ServiceBase):
         self.hdca_serializer = hdca_serializer
         self.history_contents_filters = history_contents_filters
 
+    def index(
+        self,
+        trans,
+        history_id: EncodedDatabaseIdField,
+        legacy_params: HistoryContentsIndexLegacyParams,
+        serialization_params: SerializationParams,
+        filter_query_params: HistoryContentsFilterQueryParams,
+    ) -> List[AnyHistoryContentItem]:
+        """
+        Return a list of contents (HDAs and HDCAs) for the history with the given ``id``
+
+        .. note:: Anonymous users are allowed to get their current history contents
+        """
+        if legacy_params.v == 'dev':
+            return self.__index_v2(
+                trans, history_id, legacy_params, serialization_params, filter_query_params
+            )
+        return self.__index_legacy(trans, history_id, legacy_params)
+
+    def __index_legacy(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        legacy_params: HistoryContentsIndexLegacyParams,
+    ) -> List[AnyHistoryContentItem]:
+        """Legacy implementation of the `index` action."""
+        history = self._get_history(trans, history_id)
+        parsed_legacy_params = self._parse_legacy_contents_params(legacy_params)
+        contents = history.contents_iter(**parsed_legacy_params)
+        return [
+            self._serialize_legacy_content_item(trans, content, parsed_legacy_params.get("dataset_details"))
+            for content in contents
+        ]
+
+    def __index_v2(
+        self,
+        trans,
+        history_id: EncodedDatabaseIdField,
+        legacy_params: HistoryContentsIndexLegacyParams,
+        serialization_params: SerializationParams,
+        filter_query_params: HistoryContentsFilterQueryParams,
+    ) -> List[AnyHistoryContentItem]:
+        """
+        Latests implementation of the `index` action.
+        Allows additional filtering of contents and custom serialization.
+        """
+        history = self._get_history(trans, history_id)
+        filters = self.history_contents_filters.parse_query_filters(filter_query_params)
+        order_by = self.build_order_by(self.history_contents_manager, filter_query_params.order)
+
+        # TODO: > 16.04: remove these
+        # TODO: remove 'dataset_details' and the following section when the UI doesn't need it
+        parsed_legacy_params = self._parse_legacy_contents_params(legacy_params)
+
+        contents = self.history_contents_manager.contents(
+            history,
+            filters=filters,
+            limit=filter_query_params.limit,
+            offset=filter_query_params.offset,
+            order_by=order_by,
+            serialization_params=serialization_params
+        )
+        return [
+            self._serialize_content_item(
+                trans, content,
+                dataset_details=parsed_legacy_params.get("dataset_details"),
+                serialization_params=serialization_params,
+            )
+            for content in contents
+        ]
+
+    def _serialize_legacy_content_item(
+        self,
+        trans,
+        content,
+        dataset_details: Optional[DatasetDetailsType] = None,
+    ):
+        encoded_content_id = self.encode_id(content.id)
+        detailed = dataset_details and (dataset_details == 'all' or (encoded_content_id in dataset_details))
+        if isinstance(content, HistoryDatasetAssociation):
+            view = 'detailed' if detailed else 'summary'
+            return self.hda_serializer.serialize_to_view(content, view=view, user=trans.user, trans=trans)
+        elif isinstance(content, HistoryDatasetCollectionAssociation):
+            view = 'element' if detailed else 'collection'
+            return self.__collection_dict(trans, content, view=view)
+
+    def _serialize_content_item(
+            self,
+            trans,
+            content,
+            dataset_details: Optional[DatasetDetailsType],
+            serialization_params: SerializationParams,
+            default_view: str = "summary",
+    ):
+        """
+        Returns a dictionary with the appropriate values depending on the
+        serialization parameters provided.
+        """
+        view = serialization_params.pop("view", default_view) or default_view
+
+        serializer: Optional[ModelSerializer] = None
+        if isinstance(content, HistoryDatasetAssociation):
+            serializer = self.hda_serializer
+            if dataset_details and (dataset_details == 'all' or self.encode_id(content.id) in dataset_details):
+                view = "detailed"
+        elif isinstance(content, HistoryDatasetCollectionAssociation):
+            serializer = self.hdca_serializer
+
+        if serializer is None:
+            raise exceptions.UnknownContentsType(f'Unknown contents type: {content.content_type}')
+
+        return serializer.serialize_to_view(
+            content, user=trans.user, trans=trans, view=view, **serialization_params
+        )
+
+    def _parse_legacy_contents_params(self, params: HistoryContentsIndexLegacyParams):
+        if params.types:
+            types = util.listify(params.types)
+        else:
+            types = [e.value for e in HistoryContentType]
+
+        details: Any = params.dataset_details
+        ids = None
+        if params.ids:
+            ids = [self.decode_id(id) for id in params.ids.split(',')]
+            # If explicit ids given, always used detailed result.
+            details = 'all'
+        else:
+            if details and details != 'all':
+                details = set(util.listify(details))
+
+        return ParsedHistoryContentsLegacyParams(
+            types=types,
+            ids=ids,
+            deleted=params.deleted,
+            visible=params.visible,
+            dataset_details=details,
+        ).dict(exclude_defaults=True)
+
+    def __collection_dict(self, trans, dataset_collection_instance, **kwds):
+        return dictify_dataset_collection_instance(dataset_collection_instance,
+            security=trans.security, parent=dataset_collection_instance.history, **kwds)
+
+    def _get_history(self, trans, history_id: EncodedDatabaseIdField) -> History:
+        """Retrieves the History with the given ID or raises an error if the current user cannot access it."""
+        history = self.history_manager.get_accessible(self.decode_id(history_id), trans.user, current_history=trans.history)
+        return history
+
+
 class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin):
     hda_manager: hdas.HDAManager = depends(hdas.HDAManager)
     history_manager: histories.HistoryManager = depends(histories.HistoryManager)
@@ -172,8 +337,10 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
     hdca_serializer: hdcas.HDCASerializer = depends(hdcas.HDCASerializer)
     history_contents_filters: history_contents.HistoryContentsFilters = depends(history_contents.HistoryContentsFilters)
 
+    service: HistoriesContentsService = depends(HistoriesContentsService)
+
     @expose_api_anonymous
-    def index(self, trans, history_id, ids=None, v=None, **kwd):
+    def index(self, trans, history_id, **kwd):
         """
         GET /api/histories/{history_id}/contents
 
@@ -198,54 +365,14 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :rtype:     list
         :returns:   dictionaries containing summary or detailed HDA information
         """
-        if v == 'dev':
-            return self.__index_v2(trans, history_id, **kwd)
-
-        rval = []
-
-        history = self.history_manager.get_accessible(self.decode_id(history_id), trans.user, current_history=trans.history)
-
-        # Allow passing in type or types - for continuity rest of methods
-        # take in type - but this one can be passed multiple types and
-        # type=dataset,dataset_collection is a bit silly.
-        types = kwd.get('type', kwd.get('types', None)) or []
-        if types:
-            types = util.listify(types)
-        else:
-            types = ['dataset', "dataset_collection"]
-
-        contents_kwds = {'types': types}
-        if ids:
-            ids = [self.decode_id(id) for id in ids.split(',')]
-            contents_kwds['ids'] = ids
-            # If explicit ids given, always used detailed result.
-            details = 'all'
-        else:
-            contents_kwds['deleted'] = kwd.get('deleted', None)
-            contents_kwds['visible'] = kwd.get('visible', None)
-            # details param allows a mixed set of summary and detailed hdas
-            # Ever more convoluted due to backwards compat..., details
-            # should be considered deprecated in favor of more specific
-            # dataset_details (and to be implemented dataset_collection_details).
-            details = kwd.get('details', None) or kwd.get('dataset_details', None) or []
-            if details and details != 'all':
-                details = util.listify(details)
-
-        for content in history.contents_iter(**contents_kwds):
-            encoded_content_id = trans.security.encode_id(content.id)
-            detailed = details == 'all' or (encoded_content_id in details)
-
-            if isinstance(content, trans.app.model.HistoryDatasetAssociation):
-                view = 'detailed' if detailed else 'summary'
-                hda_dict = self.hda_serializer.serialize_to_view(content, view=view, user=trans.user, trans=trans)
-                rval.append(hda_dict)
-
-            elif isinstance(content, trans.app.model.HistoryDatasetCollectionAssociation):
-                view = 'element' if detailed else 'collection'
-                collection_dict = self.__collection_dict(trans, content, view=view)
-                rval.append(collection_dict)
-
-        return rval
+        legacy_params = HistoryContentsIndexLegacyParams(**kwd)
+        serialization_params = parse_serialization_params(**kwd)
+        filter_parameters = HistoryContentsFilterQueryParams(**kwd)
+        return self.service.index(
+            trans, history_id,
+            legacy_params,
+            serialization_params, filter_parameters
+        )
 
     def __collection_dict(self, trans, dataset_collection_instance, **kwds):
         return dictify_dataset_collection_instance(dataset_collection_instance,
@@ -942,127 +1069,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
     def __handle_unknown_contents_type(self, trans, contents_type):
         raise exceptions.UnknownContentsType(f'Unknown contents type: {type}')
 
-    def __index_v2(self, trans, history_id, **kwd):
-        """
-        index( self, trans, history_id, **kwd )
-        * GET /api/histories/{history_id}/contents
-            return a list of HDA data for the history with the given ``id``
-        .. note:: Anonymous users are allowed to get their current history contents
-
-        If ids is given, index returns a *more complete* json object for each
-        HDA in the ids list.
-
-        :type   history_id: str
-        :param  history_id: encoded id string of the HDA's History
-
-        :rtype:     list
-        :returns:   dictionaries containing summary or detailed HDA information
-
-        The following are optional parameters:
-            view:   string, one of ('summary','detailed'), defaults to 'summary'
-                    controls which set of properties to return
-            keys:   comma separated strings, unused by default
-                    keys/names of individual properties to return
-
-        If neither keys or views are sent, the default view (set of keys) is returned.
-        If both a view and keys are sent, the key list and the view's keys are
-        combined.
-        If keys are sent and no view, only those properties in keys are returned.
-
-        For which properties are available see:
-            galaxy/managers/hdas/HDASerializer
-        and:
-            galaxy/managers/collection_util
-
-        The list returned can be filtered by using two optional parameters:
-            q:      string, generally a property name to filter by followed
-                    by an (often optional) hyphen and operator string.
-            qv:     string, the value to filter by
-
-        ..example:
-            To filter the list to only those created after 2015-01-29,
-            the query string would look like:
-                '?q=create_time-gt&qv=2015-01-29'
-
-            Multiple filters can be sent in using multiple q/qv pairs:
-                '?q=create_time-gt&qv=2015-01-29&q=name-contains&qv=experiment-1'
-
-        The list returned can be paginated using two optional parameters:
-            limit:  integer, defaults to no value and no limit (return all)
-                    how many items to return
-            offset: integer, defaults to 0 and starts at the beginning
-                    skip the first ( offset - 1 ) items and begin returning
-                    at the Nth item
-
-        ..example:
-            limit and offset can be combined. Skip the first two and return five:
-                '?limit=5&offset=3'
-
-        The list returned can be ordered using the optional parameter:
-            order:  string containing one of the valid ordering attributes followed
-                    (optionally) by '-asc' or '-dsc' for ascending and descending
-                    order respectively. Orders can be stacked as a comma-
-                    separated list of values.
-
-        ..example:
-            To sort by name descending then create time descending:
-                '?order=name-dsc,create_time'
-
-        The ordering attributes and their default orders are:
-            hid defaults to 'hid-asc'
-            create_time defaults to 'create_time-dsc'
-            update_time defaults to 'update_time-dsc'
-            name    defaults to 'name-asc'
-
-        'order' defaults to 'hid-asc'
-        """
-        rval = []
-
-        history = self.history_manager.get_accessible(self.decode_id(history_id), trans.user,
-            current_history=trans.history)
-
-        filter_params = self.parse_filter_params(kwd)
-        filters = self.history_contents_filters.parse_filters(filter_params)
-        limit, offset = self.parse_limit_offset(kwd)
-        order_by = self._parse_order_by(manager=self.history_contents_manager, order_by_string=kwd.get('order', 'hid-asc'))
-        serialization_params = self._parse_serialization_params(kwd, 'summary')
-        # TODO: > 16.04: remove these
-        # TODO: remove 'dataset_details' and the following section when the UI doesn't need it
-        # details param allows a mixed set of summary and detailed hdas
-        # Ever more convoluted due to backwards compat..., details
-        # should be considered deprecated in favor of more specific
-        # dataset_details (and to be implemented dataset_collection_details).
-        details = kwd.get('details', [])
-        if details and details != 'all':
-            details = util.listify(details)
-        view = serialization_params.pop('view')
-
-        contents = self.history_contents_manager.contents(history,
-            filters=filters,
-            limit=limit,
-            offset=offset,
-            order_by=order_by,
-            serialization_params=serialization_params)
-
-        for content in contents:
-
-            # TODO: remove split
-            if isinstance(content, trans.app.model.HistoryDatasetAssociation):
-                # TODO: remove split
-                if details == 'all' or trans.security.encode_id(content.id) in details:
-                    rval.append(self.hda_serializer.serialize_to_view(content,
-                        user=trans.user, trans=trans, view='detailed', **serialization_params))
-                else:
-                    rval.append(self.hda_serializer.serialize_to_view(content,
-                        user=trans.user, trans=trans, view=view, **serialization_params))
-
-            elif isinstance(content, trans.app.model.HistoryDatasetCollectionAssociation):
-                collection = self.hdca_serializer.serialize_to_view(content,
-                    user=trans.user, trans=trans, view=view, **serialization_params)
-                rval.append(collection)
-
-        return rval
-
     def encode_type_id(self, type_id):
         TYPE_ID_SEP = '-'
         split = type_id.split(TYPE_ID_SEP, 1)

From 446cf52616bc6ae2245b88cc56f7edcb59999f02 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 5 Jul 2021 20:23:01 +0200
Subject: [PATCH 087/452] Refactor `show` action

---
 .../webapps/galaxy/api/history_contents.py    | 87 ++++++++++++++++++-
 1 file changed, 83 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index b21b36bcc71..bba2e201498 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -32,6 +32,7 @@ from galaxy.managers.base import (
     ModelSerializer,
     ServiceBase,
 )
+from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.collections_util import (
     api_payload_to_create_params,
     dictify_dataset_collection_instance,
@@ -168,6 +169,8 @@ class HistoriesContentsService(ServiceBase):
         security: IdEncodingHelper,
         history_manager: histories.HistoryManager,
         history_contents_manager: history_contents.HistoryContentsManager,
+        hda_manager: hdas.HDAManager,
+        dataset_collection_manager: DatasetCollectionManager,
         hda_serializer: hdas.HDASerializer,
         hdca_serializer: hdcas.HDCASerializer,
         history_contents_filters: history_contents.HistoryContentsFilters,
@@ -175,6 +178,8 @@ class HistoriesContentsService(ServiceBase):
         super().__init__(security)
         self.history_manager = history_manager
         self.history_contents_manager = history_contents_manager
+        self.hda_manager = hda_manager
+        self.dataset_collection_manager = dataset_collection_manager
         self.hda_serializer = hda_serializer
         self.hdca_serializer = hdca_serializer
         self.history_contents_filters = history_contents_filters
@@ -198,6 +203,55 @@ class HistoriesContentsService(ServiceBase):
             )
         return self.__index_legacy(trans, history_id, legacy_params)
 
+    def show(
+        self,
+        trans,
+        id: EncodedDatabaseIdField,
+        serialization_params: SerializationParams,
+        contents_type: HistoryContentType = HistoryContentType.dataset,
+        fuzzy_count: Optional[int] = None,
+    ) -> AnyHistoryContentItem:
+        """
+        Return detailed information about an HDA or HDCA within a history
+
+        .. note:: Anonymous users are allowed to get their current history contents
+
+        :type   id:                         str
+        :param  id:                         the encoded id of the HDA or HDCA to return
+        :type   contents_type:              str
+        :param  id:                         'dataset' or 'dataset_collection'
+        :type   serialization_params.view:  str
+        :param  serialization_params.view:  if fetching a dataset collection - the view style of
+                                            the dataset collection to produce.
+                                            'collection' returns no element information, 'element'
+                                            returns detailed element information for all datasets,
+                                            'element-reference' returns a minimal set of information
+                                            about datasets (for instance id, type, and state but not
+                                            metadata, peek, info, or name). The default is 'element'.
+        :type  fuzzy_count: int
+        :param fuzzy_count: this value can be used to broadly restrict the magnitude
+                            of the number of elements returned via the API for large
+                            collections. The number of actual elements returned may
+                            be "a bit" more than this number or "a lot" less - varying
+                            on the depth of nesting, balance of nesting at each level,
+                            and size of target collection. The consumer of this API should
+                            not expect a stable number or pre-calculable number of
+                            elements to be produced given this parameter - the only
+                            promise is that this API will not respond with an order
+                            of magnitude more elements estimated with this value.
+                            The UI uses this parameter to fetch a "balanced" concept of
+                            the "start" of large collections at every depth of the
+                            collection.
+
+        :rtype:     dict
+        :returns:   dictionary containing detailed HDA or HDCA information
+        """
+        if contents_type == HistoryContentType.dataset:
+            return self.__show_dataset(trans, id, serialization_params)
+        elif contents_type == HistoryContentType.dataset_collection:
+            return self.__show_dataset_collection(trans, id, serialization_params, fuzzy_count)
+        raise exceptions.UnknownContentsType(f'Unknown contents type: {contents_type}')
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -326,6 +380,32 @@ class HistoriesContentsService(ServiceBase):
         history = self.history_manager.get_accessible(self.decode_id(history_id), trans.user, current_history=trans.history)
         return history
 
+    def __show_dataset(
+        self, trans,
+        id: EncodedDatabaseIdField,
+        serialization_params: SerializationParams,
+    ):
+        serialization_params["default_view"] = "detailed"
+        hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user)
+        return self.hda_serializer.serialize_to_view(
+            hda, user=trans.user, trans=trans, **serialization_params
+        )
+
+    def __show_dataset_collection(
+        self, trans,
+        id: EncodedDatabaseIdField,
+        serialization_params: SerializationParams,
+        fuzzy_count: Optional[int] = None,
+    ):
+        dataset_collection_instance = self.__get_accessible_collection(trans, id)
+        view = serialization_params.get("view") or "element"
+        return self.__collection_dict(trans, dataset_collection_instance, view=view, fuzzy_count=fuzzy_count)
+
+    def __get_accessible_collection(self, trans, id: EncodedDatabaseIdField):
+        return self.dataset_collection_manager.get_dataset_collection_instance(
+            trans=trans, instance_type="history", id=id,
+        )
+
 
 class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin):
     hda_manager: hdas.HDAManager = depends(hdas.HDAManager)
@@ -420,11 +500,10 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :rtype:     dict
         :returns:   dictionary containing detailed HDA or HDCA information
         """
+        serialization_params = parse_serialization_params(**kwd)
         contents_type = self.__get_contents_type(trans, kwd)
-        if contents_type == 'dataset':
-            return self.__show_dataset(trans, id, **kwd)
-        elif contents_type == 'dataset_collection':
-            return self.__show_dataset_collection(trans, id, history_id, **kwd)
+        fuzzy_count = kwd.get("fuzzy_count")
+        return self.service.show(trans, id, serialization_params, contents_type, fuzzy_count)
 
     @expose_api_anonymous
     def index_jobs_summary(self, trans, history_id, **kwd):

From ef1d41ec48c532ddf6200651acddaab4caed0585 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 5 Jul 2021 21:19:55 +0200
Subject: [PATCH 088/452] Use `Literal` from `typing_extensions` instead of
 `typing`

To support older versions of Python.
---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index bba2e201498..d01da9dfe2f 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -9,13 +9,13 @@ import re
 from typing import (
     Any,
     List,
-    Literal,
     Optional,
     Set,
     Union,
 )
 
 import dateutil.parser
+from typing_extensions import Literal
 
 from galaxy import (
     exceptions,

From c3c75c051186f5be9a3530f792c494429cea4722 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Tue, 6 Jul 2021 18:04:13 +0200
Subject: [PATCH 089/452] Fix fuzzy_count parameter parsing

---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index d01da9dfe2f..dc44c5322dd 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -502,7 +502,9 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         """
         serialization_params = parse_serialization_params(**kwd)
         contents_type = self.__get_contents_type(trans, kwd)
-        fuzzy_count = kwd.get("fuzzy_count")
+        fuzzy_count = kwd.get("fuzzy_count", None)
+        if fuzzy_count:
+            fuzzy_count = int(fuzzy_count)
         return self.service.show(trans, id, serialization_params, contents_type, fuzzy_count)
 
     @expose_api_anonymous

From 762b125873eede18fe7b915994b3a19419daec14 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 7 Jul 2021 20:50:05 +0200
Subject: [PATCH 090/452] Refactor `index_jobs_summary` action

- Add pydantic models for job state summaries
- Move controller logic to service
- Add type annotations
- Include `encode_all_ids` in ServiceBase
---
 lib/galaxy/managers/base.py                   |  8 ++++
 lib/galaxy/schema/schema.py                   | 21 +++++++++
 .../webapps/galaxy/api/history_contents.py    | 45 +++++++++++++++----
 3 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index a22c37862df..44e9134b9ec 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -1219,6 +1219,14 @@ class ServiceBase:
         """Decodes a previously encoded database ID."""
         return encode_with_security(self.security, id)
 
+    def encode_all_ids(self, rval, recursive=False):
+        """
+        Encodes all integer values in the dict rval whose keys are 'id' or end with '_id'
+
+        It might be useful to turn this in to a decorator
+        """
+        return self.security.encode_all_ids(rval, recursive=recursive)
+
     def build_order_by(self, manager: SortableManager, order_by_query: Optional[str] = None):
         """Returns an ORM compatible order_by clause using the order attribute and the given manager.
 
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index da34ba115f2..524da5a7d09 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -1028,6 +1028,27 @@ class JobImportHistoryResponse(JobBaseModel):
     )
 
 
+class JobStateSummary(Model):
+    id: EncodedDatabaseIdField = EncodedEntityIdField
+    model: str = ModelClassField("Job")
+    populated_state: DatasetCollection.populated_states = PopulatedStateField
+    states: Dict[Job.states, int] = Field(
+        {},
+        title="States",
+        description=(
+            "A dictionary of job states and the number of jobs in that state."
+        )
+    )
+
+
+class ImplicitCollectionJobsStateSummary(JobStateSummary):
+    model: str = ModelClassField("ImplicitCollectionJobs")
+
+
+class WorkflowInvocationStateSummary(JobStateSummary):
+    model: str = ModelClassField("WorkflowInvocation")
+
+
 class JobSummary(JobBaseModel):
     """Basic information about a job."""
     external_id: Optional[str] = Field(
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index dc44c5322dd..0301731d1e5 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -57,7 +57,11 @@ from galaxy.schema.schema import (
     HDCADetailed,
     HDCASummary,
     HistoryContentType,
+    ImplicitCollectionJobsStateSummary,
+    JobSourceType,
+    JobStateSummary,
     Model,
+    WorkflowInvocationStateSummary,
 )
 from galaxy.schema.types import SerializationParams
 from galaxy.security.idencoding import IdEncodingHelper
@@ -79,11 +83,18 @@ from . import BaseGalaxyAPIController, depends
 log = logging.getLogger(__name__)
 
 DatasetDetailsType = Union[Set[EncodedDatabaseIdField], Literal['all']]
+
 AnyHistoryContentItem = Union[
     HDASummary, HDADetailed, HDABeta,
     HDCASummary, HDCADetailed, HDCABeta
 ]
 
+AnyJobStateSummary = Union[
+    JobStateSummary,
+    ImplicitCollectionJobsStateSummary,
+    WorkflowInvocationStateSummary,
+]
+
 
 class HistoryContentsFilterQueryParams(FilterQueryParams):
     order: Optional[str] = OrderParamField(default_order="hid-asc")
@@ -252,6 +263,28 @@ class HistoriesContentsService(ServiceBase):
             return self.__show_dataset_collection(trans, id, serialization_params, fuzzy_count)
         raise exceptions.UnknownContentsType(f'Unknown contents type: {contents_type}')
 
+    def index_jobs_summary(
+        self, trans,
+        ids: List[EncodedDatabaseIdField],
+        types: List[JobSourceType],
+    ) -> List[AnyJobStateSummary]:
+        """
+        Return job state summary info for jobs, implicit groups jobs for collections or workflow invocations
+
+        Warning: We allow anyone to fetch job state information about any object they
+        can guess an encoded ID for - it isn't considered protected data. This keeps
+        polling IDs as part of state calculation for large histories and collections as
+        efficient as possible.
+
+        :param  ids:    the encoded ids of job summary objects to return - if ids
+                        is specified types must also be specified and have same length.
+        :param  types:  type of object represented by elements in the ids array - any of
+                        Job, ImplicitCollectionJob, or WorkflowInvocation.
+
+        :returns:   an array of job summary object dictionaries.
+        """
+        return [self.encode_all_ids(trans, job_state) for job_state in fetch_job_states(trans.sa_session, ids, types)]
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -530,15 +563,9 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :rtype:     dict[]
         :returns:   an array of job summary object dictionaries.
         """
-        ids = kwd.get("ids", None)
-        types = kwd.get("types", None)
-        if ids is None:
-            assert types is None
-            # TODO: ...
-        else:
-            ids = [self.app.security.decode_id(i) for i in util.listify(ids)]
-            types = util.listify(types)
-        return [self.encode_all_ids(trans, s) for s in fetch_job_states(trans.sa_session, ids, types)]
+        ids = util.listify(kwd.get("ids", None))
+        types = util.listify(kwd.get("types", None))
+        return self.service.index_jobs_summary(trans, ids, types)
 
     @expose_api_anonymous
     def show_jobs_summary(self, trans, id, history_id, **kwd):

From 3ed390c7b160bd0c80bb6b36368e69f3c5d1427e Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 8 Jul 2021 14:32:09 +0200
Subject: [PATCH 091/452] Refactor `show_jobs_summary` action

---
 .../webapps/galaxy/api/history_contents.py    | 59 ++++++++++++-------
 1 file changed, 37 insertions(+), 22 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 0301731d1e5..1b9ea8b081e 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -227,11 +227,8 @@ class HistoriesContentsService(ServiceBase):
 
         .. note:: Anonymous users are allowed to get their current history contents
 
-        :type   id:                         str
         :param  id:                         the encoded id of the HDA or HDCA to return
-        :type   contents_type:              str
-        :param  id:                         'dataset' or 'dataset_collection'
-        :type   serialization_params.view:  str
+        :param  contents_type:              'dataset' or 'dataset_collection'
         :param  serialization_params.view:  if fetching a dataset collection - the view style of
                                             the dataset collection to produce.
                                             'collection' returns no element information, 'element'
@@ -239,7 +236,6 @@ class HistoriesContentsService(ServiceBase):
                                             'element-reference' returns a minimal set of information
                                             about datasets (for instance id, type, and state but not
                                             metadata, peek, info, or name). The default is 'element'.
-        :type  fuzzy_count: int
         :param fuzzy_count: this value can be used to broadly restrict the magnitude
                             of the number of elements returned via the API for large
                             collections. The number of actual elements returned may
@@ -254,7 +250,6 @@ class HistoriesContentsService(ServiceBase):
                             the "start" of large collections at every depth of the
                             collection.
 
-        :rtype:     dict
         :returns:   dictionary containing detailed HDA or HDCA information
         """
         if contents_type == HistoryContentType.dataset:
@@ -285,6 +280,41 @@ class HistoriesContentsService(ServiceBase):
         """
         return [self.encode_all_ids(trans, job_state) for job_state in fetch_job_states(trans.sa_session, ids, types)]
 
+    def show_jobs_summary(
+        self, trans,
+        id: EncodedDatabaseIdField,
+        contents_type: HistoryContentType = HistoryContentType.dataset,
+    ) -> AnyJobStateSummary:
+        """
+        Return detailed information about an HDA or HDCAs jobs
+
+        Warning: We allow anyone to fetch job state information about any object they
+        can guess an encoded ID for - it isn't considered protected data. This keeps
+        polling IDs as part of state calculation for large histories and collections as
+        efficient as possible.
+
+        :param  id:             the encoded id of the HDA or HDCA to return
+        :param  contents_type:  'dataset' or 'dataset_collection'
+
+        :returns:   dictionary containing jobs summary object
+        """
+        # At most one of job or implicit_collection_jobs should be found.
+        job = None
+        implicit_collection_jobs = None
+        if contents_type == HistoryContentType.dataset:
+            hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user)
+            job = hda.creating_job
+        elif contents_type == HistoryContentType.dataset_collection:
+            dataset_collection_instance = self.__get_accessible_collection(trans, id)
+            job_source_type = dataset_collection_instance.job_source_type
+            if job_source_type == JobSourceType.Job:
+                job = dataset_collection_instance.job
+            elif job_source_type == JobSourceType.ImplicitCollectionJobs:
+                implicit_collection_jobs = dataset_collection_instance.implicit_collection_jobs
+
+        assert job is None or implicit_collection_jobs is None
+        return self.encode_all_ids(summarize_jobs_to_dict(trans.sa_session, job or implicit_collection_jobs))
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -587,22 +617,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :returns:   dictionary containing jobs summary object
         """
         contents_type = self.__get_contents_type(trans, kwd)
-        # At most one of job or implicit_collection_jobs should be found.
-        job = None
-        implicit_collection_jobs = None
-        if contents_type == 'dataset':
-            hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user)
-            job = hda.creating_job
-        elif contents_type == 'dataset_collection':
-            dataset_collection_instance = self.__get_accessible_collection(trans, id, history_id)
-            job_source_type = dataset_collection_instance.job_source_type
-            if job_source_type == "Job":
-                job = dataset_collection_instance.job
-            elif job_source_type == "ImplicitCollectionJobs":
-                implicit_collection_jobs = dataset_collection_instance.implicit_collection_jobs
-
-        assert job is None or implicit_collection_jobs is None
-        return self.encode_all_ids(trans, summarize_jobs_to_dict(trans.sa_session, job or implicit_collection_jobs))
+        return self.service.show_jobs_summary(trans, id, contents_type)
 
     def __get_contents_type(self, trans, kwd):
         contents_type = kwd.get('type', 'dataset')

From f03129af5234101ffe6ada11acfd31d061e7c865 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 8 Jul 2021 15:15:53 +0200
Subject: [PATCH 092/452] Refactor `download_dataset_collection`

- Move the controller logic to the service
---
 .../webapps/galaxy/api/history_contents.py    | 48 ++++++++-----------
 1 file changed, 21 insertions(+), 27 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 1b9ea8b081e..59a61f6713b 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -315,6 +315,26 @@ class HistoriesContentsService(ServiceBase):
         assert job is None or implicit_collection_jobs is None
         return self.encode_all_ids(summarize_jobs_to_dict(trans.sa_session, job or implicit_collection_jobs))
 
+    def download_dataset_collection(self, trans, id: EncodedDatabaseIdField):
+        """
+        Download the content of a HistoryDatasetCollection as a tgz archive
+        while maintaining approximate collection structure.
+
+        :param id: encoded HistoryDatasetCollectionAssociation (HDCA) id
+        """
+        try:
+            dataset_collection_instance = self.__get_accessible_collection(trans, id)
+            return self.__stream_dataset_collection(trans, dataset_collection_instance)
+        except Exception as e:
+            log.exception("Error in API while creating dataset collection archive")
+            trans.response.status = 500
+            return {'error': util.unicodify(e)}
+
+    def __stream_dataset_collection(self, trans, dataset_collection_instance):
+        archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_mod_zip=trans.app.config.upstream_mod_zip)
+        trans.response.headers.update(archive.get_headers())
+        return archive.response()
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -626,21 +646,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
 
         return contents_type
 
-    def __show_dataset(self, trans, id, **kwd):
-        hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user)
-        return self.hda_serializer.serialize_to_view(hda,
-                                                     user=trans.user,
-                                                     trans=trans,
-                                                     **self._parse_serialization_params(kwd, 'detailed'))
-
-    def __show_dataset_collection(self, trans, id, history_id, **kwd):
-        dataset_collection_instance = self.__get_accessible_collection(trans, id, history_id)
-        view = kwd.get("view", "element")
-        fuzzy_count = kwd.get("fuzzy_count", None)
-        if fuzzy_count:
-            fuzzy_count = int(fuzzy_count)
-        return self.__collection_dict(trans, dataset_collection_instance, view=view, fuzzy_count=fuzzy_count)
-
     def __get_accessible_collection(self, trans, id, history_id):
         return trans.app.dataset_collection_manager.get_dataset_collection_instance(
             trans=trans,
@@ -660,18 +665,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :param id: encoded HistoryDatasetCollectionAssociation (HDCA) id
         :param history_id: encoded id string of the HDCA's History
         """
-        try:
-            dataset_collection_instance = self.__get_accessible_collection(trans, id, history_id)
-            return self.__stream_dataset_collection(trans, dataset_collection_instance)
-        except Exception as e:
-            log.exception("Error in API while creating dataset collection archive")
-            trans.response.status = 500
-            return {'error': util.unicodify(e)}
-
-    def __stream_dataset_collection(self, trans, dataset_collection_instance):
-        archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_mod_zip=trans.app.config.upstream_mod_zip)
-        trans.response.headers.update(archive.get_headers())
-        return archive.response()
+        return self.service.download_dataset_collection(trans, id)
 
     @expose_api_anonymous
     def create(self, trans, history_id, payload, **kwd):

From 5f59e99f4d0950b0cf8ed1df92d461c34882a4f2 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 9 Jul 2021 16:21:27 +0200
Subject: [PATCH 093/452] Fix `index_jobs_summary` action

- The ids are now decoded as they should
- Added explicit check for parameter length matching
---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 59a61f6713b..57f5a289b01 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -278,7 +278,15 @@ class HistoriesContentsService(ServiceBase):
 
         :returns:   an array of job summary object dictionaries.
         """
-        return [self.encode_all_ids(trans, job_state) for job_state in fetch_job_states(trans.sa_session, ids, types)]
+        if len(ids) != len(types):
+            raise exceptions.RequestParameterInvalidException(
+                f"The number of ids ({len(ids)}) and types ({len(types)}) must match."
+            )
+        decoded_ids = self.decode_ids(ids)
+        return [
+            self.encode_all_ids(job_state)
+            for job_state in fetch_job_states(trans.sa_session, decoded_ids, types)
+        ]
 
     def show_jobs_summary(
         self, trans,

From c1ecedd43ab5cdc55a2e3ccd895cd859239df6b5 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 9 Jul 2021 16:22:11 +0200
Subject: [PATCH 094/452] Add type hints for encode/decode functions in service

---
 lib/galaxy/managers/base.py                       | 15 +++++++++++----
 lib/galaxy/webapps/galaxy/api/history_contents.py |  2 +-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index 44e9134b9ec..c95d0171ab4 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -47,6 +47,7 @@ from galaxy import exceptions
 from galaxy import model
 from galaxy.model import tool_shed_install
 from galaxy.schema import FilterQueryParams
+from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.structured_app import BasicApp, MinimalManagerApp
 from galaxy.util import namedtuple
@@ -1211,15 +1212,21 @@ class ServiceBase:
     def __init__(self, security: IdEncodingHelper):
         self.security = security
 
-    def decode_id(self, id):
+    def decode_id(self, id: EncodedDatabaseIdField) -> int:
         """Decodes a previously encoded database ID."""
         return decode_with_security(self.security, id)
 
-    def encode_id(self, id):
-        """Decodes a previously encoded database ID."""
+    def encode_id(self, id: int) -> EncodedDatabaseIdField:
+        """Encodes a raw database ID."""
         return encode_with_security(self.security, id)
 
-    def encode_all_ids(self, rval, recursive=False):
+    def decode_ids(self, ids: List[EncodedDatabaseIdField]) -> List[int]:
+        """
+        Decodes all encoded IDs in the given list.
+        """
+        return [self.decode_id(id) for id in ids]
+
+    def encode_all_ids(self, rval, recursive: bool = False):
         """
         Encodes all integer values in the dict rval whose keys are 'id' or end with '_id'
 
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 57f5a289b01..55bff9b9172 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -447,7 +447,7 @@ class HistoriesContentsService(ServiceBase):
         details: Any = params.dataset_details
         ids = None
         if params.ids:
-            ids = [self.decode_id(id) for id in params.ids.split(',')]
+            ids = [self.decode_id(EncodedDatabaseIdField(id)) for id in params.ids.split(',')]
             # If explicit ids given, always used detailed result.
             details = 'all'
         else:

From e2ecd88e5e1d8d4e6b054eb3beaa1586774518f6 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 12 Jul 2021 16:32:30 +0200
Subject: [PATCH 095/452] Register LibraryDatasetsManager in the dependency
 container

---
 lib/galaxy/app.py                       | 3 +++
 lib/galaxy/managers/library_datasets.py | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index a7526531c65..bb6e982f65b 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -26,6 +26,7 @@ from galaxy.managers.histories import HistoryManager
 from galaxy.managers.interactivetool import InteractiveToolManager
 from galaxy.managers.jobs import JobSearch
 from galaxy.managers.libraries import LibraryManager
+from galaxy.managers.library_datasets import LibraryDatasetsManager
 from galaxy.managers.roles import RoleManager
 from galaxy.managers.session import GalaxySessionManager
 from galaxy.managers.tools import DynamicToolManager
@@ -149,6 +150,7 @@ class MinimalGalaxyApplication(BasicApp, config.ConfiguresGalaxyMixin, HaltableC
 
 class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
     """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context."""
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._register_singleton(MinimalManagerApp, self)
@@ -173,6 +175,7 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
         self.workflow_contents_manager = self._register_singleton(WorkflowContentsManager)
         self.library_folder_manager = self._register_singleton(FolderManager)
         self.library_manager = self._register_singleton(LibraryManager)
+        self.library_datasets_manager = self._register_singleton(LibraryDatasetsManager)
         self.role_manager = self._register_singleton(RoleManager)
         from galaxy.jobs.manager import JobManager
         self.job_manager = self._register_singleton(JobManager)
diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py
index 7bb2f55ecf2..b26314c0d7e 100644
--- a/lib/galaxy/managers/library_datasets.py
+++ b/lib/galaxy/managers/library_datasets.py
@@ -15,6 +15,7 @@ from galaxy.managers import (
     datasets,
 )
 from galaxy.model import tags
+from galaxy.structured_app import MinimalManagerApp
 from galaxy.util import validation
 
 log = logging.getLogger(__name__)
@@ -24,7 +25,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager):
     """Interface/service object for interacting with library datasets."""
     model_class = model.LibraryDatasetDatasetAssociation
 
-    def __init__(self, app):
+    def __init__(self, app: MinimalManagerApp):
         self.app = app
         self.tag_handler = tags.GalaxyTagHandler(app.model.context)
 

From 512d36c5ace1d0da582f64f46b6486aa4615afce Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 12 Jul 2021 17:56:59 +0200
Subject: [PATCH 096/452] Refactor `create` action

---
 lib/galaxy/schema/schema.py                   |  15 +
 .../webapps/galaxy/api/history_contents.py    | 492 +++++++++++-------
 2 files changed, 331 insertions(+), 176 deletions(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 524da5a7d09..91efc9d56cd 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -208,6 +208,21 @@ class DatasetSourceType(str, Enum):
     ldda = "ldda"
 
 
+class ColletionSourceType(str, Enum):
+    hda = "hda"
+    ldda = "ldda"
+    hdca = "hdca"
+    new_collection = "new_collection"
+
+
+class HistoryContentSource(str, Enum):
+    hda = "hda"
+    hdca = "hdca"
+    library = "library"
+    library_folder = "library_folder"
+    new_collection = "new_collection"
+
+
 class TagCollection(Model):
     """Represents the collection of tags associated with an item."""
     __root__: List[str] = Field(
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 55bff9b9172..5771e08c587 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -15,6 +15,7 @@ from typing import (
 )
 
 import dateutil.parser
+from pydantic import Extra
 from typing_extensions import Literal
 
 from galaxy import (
@@ -38,11 +39,14 @@ from galaxy.managers.collections_util import (
     dictify_dataset_collection_instance,
 )
 from galaxy.managers.jobs import fetch_job_states, summarize_jobs_to_dict
+from galaxy.managers.library_datasets import LibraryDatasetsManager
 from galaxy.model import (
     History,
     HistoryDatasetAssociation,
     HistoryDatasetCollectionAssociation,
+    LibraryDataset,
 )
+from galaxy.model.security import GalaxyRBACAgent
 from galaxy.schema import FilterQueryParams
 from galaxy.schema.fields import (
     EncodedDatabaseIdField,
@@ -50,12 +54,14 @@ from galaxy.schema.fields import (
     OrderParamField,
 )
 from galaxy.schema.schema import (
+    ColletionSourceType,
     HDABeta,
     HDADetailed,
     HDASummary,
     HDCABeta,
     HDCADetailed,
     HDCASummary,
+    HistoryContentSource,
     HistoryContentType,
     ImplicitCollectionJobsStateSummary,
     JobSourceType,
@@ -168,6 +174,101 @@ class ParsedHistoryContentsLegacyParams(HistoryContentsIndexLegacyParamsBase):
     )
 
 
+class CreateHistoryContentPayloadBase(Model):
+    type: Optional[HistoryContentType] = Field(
+        HistoryContentType.dataset,
+        title="Type",
+        description="The type of content to be created in the history.",
+    )
+
+
+class CreateHistoryContentPayloadFromCopy(CreateHistoryContentPayloadBase):
+    source: Optional[HistoryContentSource] = Field(
+        None,
+        title="Source",
+        description="The source of the content. Can be other history element to be copied or library elements.",
+    )
+    content: Optional[EncodedDatabaseIdField] = Field(
+        None,
+        title="Content",
+        description=(
+            "Depending on the `source` it can be:\n"
+            "- The encoded id from the library dataset\n"
+            "- The encoded id from the library folder\n"
+            "- The encoded id from the HDA\n"
+            "- The encoded id from the HDCA\n"
+        ),
+    )
+
+
+class CollectionElementIdentifier(Model):
+    name: Optional[str] = Field(
+        None,
+        title="Name",
+        description="The name of the element.",
+    )
+    src: ColletionSourceType = Field(
+        ...,
+        title="Source",
+        description="The source of the element.",
+    )
+    id: EncodedDatabaseIdField = Field(
+        ...,
+        title="ID",
+        description="The encoded ID of the element.",
+    )
+    tags: List[str] = Field(
+        default=[],
+        title="Tags",
+        description="The list of tags associated with the element.",
+    )
+
+
+class CreateNewCollectionPayload(Model):
+    collection_type: str = Field(
+        ...,
+        title="Collection Type",
+        description="The type of the collection. For example, `list`, `paired`, `list:paired`.",
+    )
+    element_identifiers: List[CollectionElementIdentifier] = Field(
+        ...,
+        title="Element Identifiers",
+        description="List of elements that should be in the new collection.",
+    )
+    name: Optional[str] = Field(
+        default=None,
+        title="Name",
+        description="The name of the new collection.",
+    )
+    hide_source_items: Optional[bool] = Field(
+        default=False,
+        title="Hide Source Items",
+        description="Whether to mark the original HDAs as hidden.",
+    )
+
+
+class CreateHistoryContentPayloadFromCollection(CreateHistoryContentPayloadFromCopy):
+    dbkey: Optional[str] = Field(
+        default=None,
+        title="DBKey",
+        description="TODO",
+    )
+    copy_elements: Optional[bool] = Field(
+        default=False,
+        title="Copy Elements",
+        description=(
+            "If the source is a collection, whether to copy child HDAs into the target "
+            "history as well, defaults to False but this is less than ideal and may "
+            "be changed in future releases."
+        ),
+    )
+
+
+class CreateHistoryContentPayload(CreateHistoryContentPayloadFromCollection):
+    class Config:
+        extra = Extra.allow
+
+
 class HistoriesContentsService(ServiceBase):
     """Common interface/service logic for interactions with histories contents in the context of the API.
 
@@ -182,6 +283,8 @@ class HistoriesContentsService(ServiceBase):
         history_contents_manager: history_contents.HistoryContentsManager,
         hda_manager: hdas.HDAManager,
         dataset_collection_manager: DatasetCollectionManager,
+        ldda_manager: LibraryDatasetsManager,
+        folder_manager: folders.FolderManager,
         hda_serializer: hdas.HDASerializer,
         hdca_serializer: hdcas.HDCASerializer,
         history_contents_filters: history_contents.HistoryContentsFilters,
@@ -191,6 +294,8 @@ class HistoriesContentsService(ServiceBase):
         self.history_contents_manager = history_contents_manager
         self.hda_manager = hda_manager
         self.dataset_collection_manager = dataset_collection_manager
+        self.ldda_manager = ldda_manager
+        self.folder_manager = folder_manager
         self.hda_serializer = hda_serializer
         self.hdca_serializer = hdca_serializer
         self.history_contents_filters = history_contents_filters
@@ -343,6 +448,33 @@ class HistoriesContentsService(ServiceBase):
         trans.response.headers.update(archive.get_headers())
         return archive.response()
 
+    def create(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        payload: CreateHistoryContentPayload,
+        serialization_params: SerializationParams,
+    ) -> AnyHistoryContentItem:
+        """
+        Create a new HDA or HDCA.
+
+        ..note:
+            Currently, a user can only copy an HDA from a history that the user owns.
+        """
+        history = self.history_manager.get_owned(
+            self.decode_id(history_id), trans.user, current_history=trans.history
+        )
+
+        type = payload.type
+        if type == HistoryContentType.dataset:
+            source = payload.source
+            if source == HistoryContentSource.library_folder:
+                return self.__create_datasets_from_library_folder(trans, history, payload, serialization_params)
+            else:
+                return self.__create_dataset(trans, history, payload, serialization_params)
+        elif type == HistoryContentType.dataset_collection:
+            return self.__create_dataset_collection(trans, history, payload, serialization_params)
+        raise exceptions.UnknownContentsType(f'Unknown contents type: {payload.type}')
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -497,6 +629,187 @@ class HistoriesContentsService(ServiceBase):
             trans=trans, instance_type="history", id=id,
         )
 
+    def __create_datasets_from_library_folder(
+        self, trans,
+        history: History,
+        payload: CreateHistoryContentPayloadFromCopy,
+        serialization_params: SerializationParams,
+    ):
+        rval = []
+        source = payload.source
+        if source == HistoryContentSource.library_folder:
+            content = payload.content
+            if content is None:
+                raise exceptions.RequestParameterMissingException("'content' id of lda or hda is missing")
+
+            folder_id = self.folder_manager.cut_and_decode(trans, content)
+            folder = self.folder_manager.get(trans, folder_id)
+
+            current_user_roles = trans.get_current_user_roles()
+            security_agent: GalaxyRBACAgent = trans.app.security_agent
+
+            def traverse(folder):
+                admin = trans.user_is_admin
+                rval = []
+                for subfolder in folder.active_folders:
+                    if not admin:
+                        can_access, folder_ids = security_agent.check_folder_contents(trans.user, current_user_roles, subfolder)
+                    if (admin or can_access) and not subfolder.deleted:
+                        rval.extend(traverse(subfolder))
+                for ld in folder.datasets:
+                    if not admin:
+                        can_access = security_agent.can_access_dataset(
+                            current_user_roles,
+                            ld.library_dataset_dataset_association.dataset
+                        )
+                    if (admin or can_access) and not ld.deleted:
+                        rval.append(ld)
+                return rval
+
+            for ld in traverse(folder):
+                hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True)
+                hda_dict = self.hda_serializer.serialize_to_view(
+                    hda, user=trans.user, trans=trans, default_view='detailed', **serialization_params
+                )
+                rval.append(hda_dict)
+        else:
+            message = f"Invalid 'source' parameter in request: {source}"
+            raise exceptions.RequestParameterInvalidException(message)
+
+        trans.sa_session.flush()
+        return rval
+
+    def __create_dataset(
+        self, trans,
+        history: History,
+        payload: CreateHistoryContentPayloadFromCopy,
+        serialization_params: SerializationParams,
+    ):
+        source = payload.source
+        if source not in (HistoryContentSource.library, HistoryContentSource.hda):
+            raise exceptions.RequestParameterInvalidException(
+                f"'source' must be either 'library' or 'hda': {source}")
+        content = payload.content
+        if content is None:
+            raise exceptions.RequestParameterMissingException("'content' id of lda or hda is missing")
+
+        hda = None
+        if source == HistoryContentSource.library:
+            hda = self.__create_hda_from_ldda(trans, history, content)
+        elif source == HistoryContentSource.hda:
+            hda = self.__create_hda_from_copy(trans, history, content)
+
+        if hda is None:
+            return None
+
+        trans.sa_session.flush()
+        serialization_params["default_view"] = 'detailed'
+        return self.hda_serializer.serialize_to_view(
+            hda, user=trans.user, trans=trans, **serialization_params
+        )
+
+    def __create_hda_from_ldda(self, trans, history: History, ldda_id: EncodedDatabaseIdField):
+        decoded_ldda_id = self.decode_id(ldda_id)
+        ld = self.ldda_manager.get(trans, decoded_ldda_id)
+        if type(ld) is not LibraryDataset:
+            raise exceptions.RequestParameterInvalidException(
+                f"Library content id ( {ldda_id} ) is not a dataset")
+        hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True)
+        return hda
+
+    def __create_hda_from_copy(self, trans, history: History, original_hda_id: EncodedDatabaseIdField):
+        unencoded_hda_id = self.decode_id(original_hda_id)
+        original = self.hda_manager.get_accessible(unencoded_hda_id, trans.user)
+        # check for access on history that contains the original hda as well
+        self.history_manager.error_unless_accessible(original.history, trans.user, current_history=trans.history)
+        hda = self.hda_manager.copy(original, history=history)
+        return hda
+
+    def __create_dataset_collection(
+        self, trans,
+        history: History,
+        payload: CreateHistoryContentPayloadFromCollection,
+        serialization_params: SerializationParams,
+    ):
+        """Create hdca in a history from the list of element identifiers
+
+        :param history: history the new hdca should be added to
+        :param source: whether to create a new collection or copy existing one
+        :type  source: str
+        :param payload: dictionary structure containing:
+            :param collection_type: type (and depth) of the new collection
+            :type name: str
+            :param element_identifiers: list of elements that should be in the new collection
+                :param element: one member of the collection
+                    :param name: name of the element
+                    :type name: str
+                    :param src: source of the element (hda/ldda)
+                    :type src: str
+                    :param id: identifier
+                    :type id: str
+                    :param id: tags
+                    :type id: list
+                :type element: dict
+            :type name: list
+            :param name: name of the collection
+            :type name: str
+            :param hide_source_items: whether to mark the original hdas as hidden
+            :type name: bool
+            :param copy_elements: whether to copy HDAs when creating collection
+            :type name: bool
+        :type  payload: dict
+
+       .. note:: Elements may be nested depending on the collection_type
+
+        :returns:   dataset collection information
+        :rtype:     dict
+
+        :raises: RequestParameterInvalidException, RequestParameterMissingException
+        """
+        source = payload.source or HistoryContentSource.new_collection
+
+        dataset_collection_manager = self.dataset_collection_manager
+        if source == HistoryContentSource.new_collection:
+            create_params = api_payload_to_create_params(payload.dict())
+            dataset_collection_instance = dataset_collection_manager.create(
+                trans,
+                parent=history,
+                history=history,
+                **create_params
+            )
+        elif source == HistoryContentSource.hdca:
+            content = payload.content
+            if content is None:
+                raise exceptions.RequestParameterMissingException("'content' id of target to copy is missing")
+            dbkey = payload.dbkey
+            copy_required = dbkey is not None
+            copy_elements = payload.copy_elements or copy_required
+            if copy_required and not copy_elements:
+                raise exceptions.RequestParameterMissingException("copy_elements passed as 'false' but it is required to change specified attributes")
+            dataset_instance_attributes = {}
+            if dbkey is not None:
+                dataset_instance_attributes["dbkey"] = dbkey
+            dataset_collection_instance = dataset_collection_manager.copy(
+                trans=trans,
+                parent=history,
+                source=source,
+                encoded_source_id=content,
+                copy_elements=copy_elements,
+                dataset_instance_attributes=dataset_instance_attributes,
+            )
+        else:
+            message = f"Invalid 'source' parameter in request: {source}"
+            raise exceptions.RequestParameterInvalidException(message)
+
+        # if the consumer specified keys or view, use the secondary serializer
+        if serialization_params.get('view') or serialization_params.get('keys'):
+            serialization_params["default_view"] = 'detailed'
+            return self.hdca_serializer.serialize_to_view(
+                dataset_collection_instance, user=trans.user, trans=trans, **serialization_params
+            )
+
+        return self.__collection_dict(trans, dataset_collection_instance, view="element")
+
 
 class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin):
     hda_manager: hdas.HDAManager = depends(hdas.HDAManager)
@@ -755,182 +1068,9 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :returns:   dictionary containing detailed information for the new HDA
 
         """
-        # TODO: Flush out create new collection documentation above, need some
-        # examples. See also bioblend and API tests for specific examples.
-
-        history = self.history_manager.get_owned(self.decode_id(history_id), trans.user,
-                                                 current_history=trans.history)
-
-        type = payload.get('type', 'dataset')
-        if type == 'dataset':
-            source = payload.get('source', None)
-            if source == 'library_folder':
-                return self.__create_datasets_from_library_folder(trans, history, payload, **kwd)
-            else:
-                return self.__create_dataset(trans, history, payload, **kwd)
-        elif type == 'dataset_collection':
-            return self.__create_dataset_collection(trans, history, payload, **kwd)
-        else:
-            return self.__handle_unknown_contents_type(trans, type)
-
-    def __create_dataset(self, trans, history, payload, **kwd):
-        source = payload.get('source', None)
-        if source not in ('library', 'hda'):
-            raise exceptions.RequestParameterInvalidException(
-                f"'source' must be either 'library' or 'hda': {source}")
-        content = payload.get('content', None)
-        if content is None:
-            raise exceptions.RequestParameterMissingException("'content' id of lda or hda is missing")
-
-        # copy from library dataset
-        hda = None
-        if source == 'library':
-            hda = self.__create_hda_from_ldda(trans, content, history)
-
-        # copy an existing, accessible hda
-        elif source == 'hda':
-            unencoded_hda_id = self.decode_id(content)
-            original = self.hda_manager.get_accessible(unencoded_hda_id, trans.user)
-            # check for access on history that contains the original hda as well
-            self.history_manager.error_unless_accessible(original.history, trans.user, current_history=trans.history)
-            hda = self.hda_manager.copy(original, history=history)
-
-        trans.sa_session.flush()
-        if not hda:
-            return None
-        return self.hda_serializer.serialize_to_view(hda,
-            user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed'))
-
-    def __create_hda_from_ldda(self, trans, content, history):
-        ld = self.get_library_dataset(trans, content)
-        if type(ld) is not trans.app.model.LibraryDataset:
-            raise exceptions.RequestParameterInvalidException(
-                f"Library content id ( {content} ) is not a dataset")
-        hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True)
-        return hda
-
-    def __create_datasets_from_library_folder(self, trans, history, payload, **kwd):
-        rval = []
-
-        source = payload.get('source', None)
-        if source == 'library_folder':
-            content = payload.get('content', None)
-            if content is None:
-                raise exceptions.RequestParameterMissingException("'content' id of lda or hda is missing")
-
-            folder_id = self.folder_manager.cut_and_decode(trans, content)
-            folder = self.folder_manager.get(trans, folder_id)
-
-            current_user_roles = trans.get_current_user_roles()
-
-            def traverse(folder):
-                admin = trans.user_is_admin
-                rval = []
-                for subfolder in folder.active_folders:
-                    if not admin:
-                        can_access, folder_ids = trans.app.security_agent.check_folder_contents(trans.user, current_user_roles, subfolder)
-                    if (admin or can_access) and not subfolder.deleted:
-                        rval.extend(traverse(subfolder))
-                for ld in folder.datasets:
-                    if not admin:
-                        can_access = trans.app.security_agent.can_access_dataset(
-                            current_user_roles,
-                            ld.library_dataset_dataset_association.dataset
-                        )
-                    if (admin or can_access) and not ld.deleted:
-                        rval.append(ld)
-                return rval
-
-            for ld in traverse(folder):
-                hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True)
-                hda_dict = self.hda_serializer.serialize_to_view(hda,
-                    user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed'))
-                rval.append(hda_dict)
-        else:
-            message = f"Invalid 'source' parameter in request {source}"
-            raise exceptions.RequestParameterInvalidException(message)
-
-        trans.sa_session.flush()
-        return rval
-
-    def __create_dataset_collection(self, trans, history, payload, **kwd):
-        """Create hdca in a history from the list of element identifiers
-
-        :param history: history the new hdca should be added to
-        :type  history: History
-        :param source: whether to create a new collection or copy existing one
-        :type  source: str
-        :param payload: dictionary structure containing:
-            :param collection_type: type (and depth) of the new collection
-            :type name: str
-            :param element_identifiers: list of elements that should be in the new collection
-                :param element: one member of the collection
-                    :param name: name of the element
-                    :type name: str
-                    :param src: source of the element (hda/ldda)
-                    :type src: str
-                    :param id: identifier
-                    :type id: str
-                    :param id: tags
-                    :type id: list
-                :type element: dict
-            :type name: list
-            :param name: name of the collection
-            :type name: str
-            :param hide_source_items: whether to mark the original hdas as hidden
-            :type name: bool
-            :param copy_elements: whether to copy HDAs when creating collection
-            :type name: bool
-        :type  payload: dict
-
-       .. note:: Elements may be nested depending on the collection_type
-
-        :returns:   dataset collection information
-        :rtype:     dict
-
-        :raises: RequestParameterInvalidException, RequestParameterMissingException
-        """
-        source = kwd.get("source", payload.get("source", "new_collection"))
-
-        service = trans.app.dataset_collection_manager
-        if source == "new_collection":
-            create_params = api_payload_to_create_params(payload)
-            dataset_collection_instance = service.create(
-                trans,
-                parent=history,
-                history=history,
-                **create_params
-            )
-        elif source == "hdca":
-            content = payload.get('content', None)
-            if content is None:
-                raise exceptions.RequestParameterMissingException("'content' id of target to copy is missing")
-            dbkey = payload.get("dbkey")
-            copy_required = dbkey is not None
-            copy_elements = payload.get('copy_elements', copy_required)
-            if copy_required and not copy_elements:
-                raise exceptions.RequestParameterMissingException("copy_elements passed as 'false' but it is required to change specified attributes")
-            dataset_instance_attributes = {}
-            if dbkey is not None:
-                dataset_instance_attributes["dbkey"] = dbkey
-            dataset_collection_instance = service.copy(
-                trans=trans,
-                parent=history,
-                source="hdca",
-                encoded_source_id=content,
-                copy_elements=copy_elements,
-                dataset_instance_attributes=dataset_instance_attributes,
-            )
-        else:
-            message = f"Invalid 'source' parameter in request {source}"
-            raise exceptions.RequestParameterInvalidException(message)
-
-        # if the consumer specified keys or view, use the secondary serializer
-        if 'view' in kwd or 'keys' in kwd:
-            return self.hdca_serializer.serialize_to_view(dataset_collection_instance,
-                user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed'))
-
-        return self.__collection_dict(trans, dataset_collection_instance, view="element")
+        serialization_params = parse_serialization_params(**kwd)
+        create_payload = CreateHistoryContentPayload(**payload)
+        return self.service.create(trans, history_id, create_payload, serialization_params)
 
     @expose_api
     def show_roles(self, trans, encoded_dataset_id, **kwd):

From 83fd85dc893ef07a42b9be1da33e2bde5071c203 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 12 Jul 2021 19:16:20 +0200
Subject: [PATCH 097/452] Mark `show_roles` as not implemented

This action doesn't seem to be used or mapped in `buildapp.py`. Temporarily raising NotImplementedError util confirmation.
---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 5771e08c587..cf480dd8b36 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -1088,8 +1088,9 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
 
         :raises: InsufficientPermissionsException
         """
-        hda = self.hda_manager.get_owned(self.decode_id(encoded_dataset_id), trans.user, current_history=trans.history, trans=trans)
-        return self.hda_manager.serialize_dataset_association_roles(trans, hda)
+        raise NotImplementedError() # TODO: This endpoint doesn't seem to be in use, remove?
+        # hda = self.hda_manager.get_owned(self.decode_id(encoded_dataset_id), trans.user, current_history=trans.history, trans=trans)
+        # return self.hda_manager.serialize_dataset_association_roles(trans, hda)
 
     @expose_api
     def update_permissions(self, trans, history_id, history_content_id, payload=None, **kwd):

From a1388108ef266c69084f0d7349a4b1c56c1ef7d2 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Tue, 13 Jul 2021 16:42:27 +0200
Subject: [PATCH 098/452] Refactor `update_permissions` action

---
 lib/galaxy/schema/schema.py                   | 66 +++++++++++++++++++
 .../webapps/galaxy/api/history_contents.py    | 58 ++++++++++++++--
 2 files changed, 118 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 91efc9d56cd..da10d745bdf 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -2054,6 +2054,12 @@ class LibraryPermissionAction(str, Enum):
     remove_restrictions = "remove_restrictions"  # name inconsistency? should be `make_public`?
 
 
+class DatasetPermissionAction(str, Enum):
+    set_permissions = "set_permissions"
+    make_private = "make_private"
+    remove_restrictions = "remove_restrictions"
+
+
 class LibraryPermissionsPayloadBase(BaseModel):
     class Config:
         use_enum_values = True  # When using .dict()
@@ -2214,3 +2220,63 @@ class LibraryFolderCurrentPermissions(BaseModel):
         title="Add Role List",
         description="A list containing pairs of role names and corresponding encoded IDs which can add items to the Library folder.",
     )
+
+
+class DatasetAssociationRoles(Model):
+    access_dataset_roles: List[RoleNameIdTuple] = Field(
+        default=[],
+        title="Access Roles",
+        description=(
+            "A list of roles that can access the dataset. "
+            "The user has to **have all these roles** in order to access this dataset. "
+            "Users without access permission **cannot** have other permissions on this dataset. "
+            "If there are no access roles set on the dataset it is considered **unrestricted**."
+        ),
+    )
+    manage_dataset_roles: List[RoleNameIdTuple] = Field(
+        default=[],
+        title="Manage Roles",
+        description=(
+            "A list of roles that can manage permissions on the dataset. "
+            "Users with **any** of these roles can manage permissions of this dataset. "
+            "If you remove yourself you will lose the ability to manage this dataset unless you are an admin."
+        ),
+    )
+    modify_item_roles: List[RoleNameIdTuple] = Field(
+        default=[],
+        title="Modify Roles",
+        description=(
+            "A list of roles that can modify the library item. This is a library related permission. "
+            "User with **any** of these roles can modify name, metadata, and other information about this library item."
+        ),
+    )
+
+
+class UpdateDatasetPermissionsPayload(BaseModel):
+    class Config:
+        use_enum_values = True  # When using .dict()
+        allow_population_by_alias = True
+
+    action: Optional[DatasetPermissionAction] = Field(
+        ...,
+        title="Action",
+        description="Indicates what action should be performed on the dataset.",
+    )
+    access_ids: Optional[RoleIdList] = Field(
+        [],
+        alias="access_ids[]",  # Added for backward compatibility but it looks really ugly...
+        title="Access IDs",
+        description="A list of role encoded IDs defining roles that should have access permission on the dataset.",
+    )
+    manage_ids: Optional[RoleIdList] = Field(
+        [],
+        alias="manage_ids[]",
+        title="Manage IDs",
+        description="A list of role encoded IDs defining roles that should have manage permission on the dataset.",
+    )
+    modify_ids: Optional[RoleIdList] = Field(
+        [],
+        alias="modify_ids[]",
+        title="Modify IDs",
+        description="A list of role encoded IDs defining roles that should have modify permission on the dataset.",
+    )
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index cf480dd8b36..7f64719bfe0 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -8,6 +8,7 @@ import os
 import re
 from typing import (
     Any,
+    Dict,
     List,
     Optional,
     Set,
@@ -55,6 +56,8 @@ from galaxy.schema.fields import (
 )
 from galaxy.schema.schema import (
     ColletionSourceType,
+    DatasetAssociationRoles,
+    DatasetPermissionAction,
     HDABeta,
     HDADetailed,
     HDASummary,
@@ -67,6 +70,7 @@ from galaxy.schema.schema import (
     JobSourceType,
     JobStateSummary,
     Model,
+    UpdateDatasetPermissionsPayload,
     WorkflowInvocationStateSummary,
 )
 from galaxy.schema.types import SerializationParams
@@ -475,6 +479,35 @@ class HistoriesContentsService(ServiceBase):
             return self.__create_dataset_collection(trans, history, payload, serialization_params)
         raise exceptions.UnknownContentsType(f'Unknown contents type: {payload.type}')
 
+    def update_permissions(
+        self, trans,
+        history_content_id: EncodedDatabaseIdField,
+        payload: UpdateDatasetPermissionsPayload,
+    ) -> DatasetAssociationRoles:
+        """
+        Set permissions of the given dataset to the given role ids.
+
+        :param   payload: dictionary structure containing:
+            :param  action:     (required) describes what action should be performed
+                                available actions: make_private, remove_restrictions, set_permissions
+            :type   action:     string
+            :param  access_ids[]:      list of Role.id defining roles that should have access permission on the dataset
+            :type   access_ids[]:      string or list
+            :param  manage_ids[]:      list of Role.id defining roles that should have manage permission on the dataset
+            :type   manage_ids[]:      string or list
+            :param  modify_ids[]:      list of Role.id defining roles that should have modify permission on the library dataset item
+            :type   modify_ids[]:      string or list
+
+        :raises: RequestParameterInvalidException, ObjectNotFound, InsufficientPermissionsException, InternalServerError
+                    RequestParameterMissingException
+        """
+        payload_dict = payload.dict(by_alias=True)
+        hda = self.hda_manager.get_owned(self.decode_id(history_content_id), trans.user, current_history=trans.history, trans=trans)
+        assert hda is not None
+        self.hda_manager.update_permissions(trans, hda, **payload_dict)
+        roles = self.hda_manager.serialize_dataset_association_roles(trans, hda)
+        return DatasetAssociationRoles.parse_obj(roles)
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -1088,12 +1121,12 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
 
         :raises: InsufficientPermissionsException
         """
-        raise NotImplementedError() # TODO: This endpoint doesn't seem to be in use, remove?
+        raise NotImplementedError()  # TODO: This endpoint doesn't seem to be in use, remove?
         # hda = self.hda_manager.get_owned(self.decode_id(encoded_dataset_id), trans.user, current_history=trans.history, trans=trans)
         # return self.hda_manager.serialize_dataset_association_roles(trans, hda)
 
     @expose_api
-    def update_permissions(self, trans, history_id, history_content_id, payload=None, **kwd):
+    def update_permissions(self, trans, history_id, history_content_id, payload: Dict[str, Any] = None, **kwd):
         """
         Set permissions of the given library dataset to the given role ids.
 
@@ -1123,10 +1156,23 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         """
         if payload:
             kwd.update(payload)
-        hda = self.hda_manager.get_owned(self.decode_id(history_content_id), trans.user, current_history=trans.history, trans=trans)
-        assert hda is not None
-        self.hda_manager.update_permissions(trans, hda, **kwd)
-        return self.hda_manager.serialize_dataset_association_roles(trans, hda)
+        update_payload = self._get_update_permission_payload(kwd)
+        return self.service.update_permissions(trans, history_content_id, update_payload)
+
+    def _get_update_permission_payload(self, payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload:
+        """Coverts the payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing.
+
+        This is an attempt on supporting multiple aliases for the permissions params."""
+        # There are several allowed names for the same role list parameter, i.e.: `access`, `access_ids`, `access_ids[]`
+        # The `access_ids[]` name is not pydantic friendly, so this will be modelled as an alias but we can only set one alias
+        # TODO: Maybe we should choose only one way and deprecate the others?
+        payload["access_ids[]"] = payload.get("access_ids[]") or payload.get("access")
+        payload["manage_ids[]"] = payload.get("manage_ids[]") or payload.get("manage")
+        payload["modify_ids[]"] = payload.get("modify_ids[]") or payload.get("modify")
+        # The action is required, so the default will be used if none is specified
+        payload["action"] = payload.get("action", DatasetPermissionAction.set_permissions)
+        update_payload = UpdateDatasetPermissionsPayload(**payload)
+        return update_payload
 
     @expose_api_anonymous
     def update_batch(self, trans, history_id, payload, **kwd):

From 10beef884f2c510df8f7a9125714027454d1ce20 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 14 Jul 2021 15:40:13 +0200
Subject: [PATCH 099/452] Refactor `update` and `update_batch` actions

- Add a new decorator to make all fields optional in a pydantic model. This will help reusing models for update actions.
- Move controller logic to service
- Add type annotations
---
 lib/galaxy/schema/fields.py                   |  21 ++
 lib/galaxy/schema/schema.py                   |  29 +++
 .../webapps/galaxy/api/history_contents.py    | 230 +++++++++++-------
 3 files changed, 194 insertions(+), 86 deletions(-)

diff --git a/lib/galaxy/schema/fields.py b/lib/galaxy/schema/fields.py
index f9d2d365667..cc7e81dfd8f 100644
--- a/lib/galaxy/schema/fields.py
+++ b/lib/galaxy/schema/fields.py
@@ -1,7 +1,9 @@
+import inspect
 import re
 from typing import Optional
 
 from pydantic import (
+    BaseModel,
     Field,
 )
 
@@ -76,3 +78,22 @@ def OrderParamField(default_order: str) -> Optional[str]:
         ),
         example="name-dsc,create_time",
     )
+
+
+def optional(*fields):
+    """Decorator function used to modify a pydantic model's fields to all be optional.
+    Alternatively, you can  also pass the field names that should be made optional as arguments
+    to the decorator.
+    Taken from https://github.com/samuelcolvin/pydantic/issues/1223#issuecomment-775363074
+    """
+    def dec(_cls):
+        for field in fields:
+            _cls.__fields__[field].required = False
+        return _cls
+
+    if fields and inspect.isclass(fields[0]) and issubclass(fields[0], BaseModel):
+        cls = fields[0]
+        fields = cls.__fields__
+        return dec(cls)
+
+    return dec
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index da10d745bdf..7035cec9653 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -31,6 +31,7 @@ from galaxy.model import (
 from galaxy.schema.fields import (
     EncodedDatabaseIdField,
     ModelClassField,
+    optional,
 )
 from galaxy.schema.types import RelativeUrl
 
@@ -524,6 +525,12 @@ class HDABeta(HDADetailed):  # TODO: change HDABeta name to a more appropriate o
     pass
 
 
+@optional
+class UpdateHDAPayload(HDABeta):
+    """Used for updating a particular HDA. All fields are optional."""
+    pass
+
+
 class DCSummary(Model):
     """Dataset Collection summary information."""
     model_class: str = ModelClassField(DC_MODEL_CLASS_NAME)
@@ -604,6 +611,28 @@ class HDCADetailed(HDCASummary):
     elements: List[DCESummary] = ElementsField
 
 
+@optional
+class UpdateHDCAPayload(HDCADetailed):
+    """Used for updating a particular HDCA. All fields are optional."""
+    pass
+
+
+class UpdateHistoryContentsBatchPayload(Model):
+    items: List[Union[UpdateHDAPayload, UpdateHDCAPayload]] = Field(
+        ...,
+        title="Items",
+        description="A list of content items to update with the changes.",
+    )
+    deleted: Optional[bool] = Field(
+        default=False,
+        title="Deleted",
+        description=(
+            "This will check the uploading state if not deleting (i.e: deleted=False), "
+            "otherwise cannot delete uploading files, so it will raise an error."
+        ),
+    )
+
+
 class HistoryBase(BaseModel):
     """Provides basic configuration for all the History models."""
     class Config:
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 7f64719bfe0..e0ade4387e9 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -71,6 +71,8 @@ from galaxy.schema.schema import (
     JobStateSummary,
     Model,
     UpdateDatasetPermissionsPayload,
+    UpdateHDAPayload,
+    UpdateHistoryContentsBatchPayload,
     WorkflowInvocationStateSummary,
 )
 from galaxy.schema.types import SerializationParams
@@ -290,6 +292,7 @@ class HistoriesContentsService(ServiceBase):
         ldda_manager: LibraryDatasetsManager,
         folder_manager: folders.FolderManager,
         hda_serializer: hdas.HDASerializer,
+        hda_deserializer: hdas.HDADeserializer,
         hdca_serializer: hdcas.HDCASerializer,
         history_contents_filters: history_contents.HistoryContentsFilters,
     ):
@@ -301,6 +304,7 @@ class HistoriesContentsService(ServiceBase):
         self.ldda_manager = ldda_manager
         self.folder_manager = folder_manager
         self.hda_serializer = hda_serializer
+        self.hda_deserializer = hda_deserializer
         self.hdca_serializer = hdca_serializer
         self.history_contents_filters = history_contents_filters
 
@@ -508,6 +512,138 @@ class HistoriesContentsService(ServiceBase):
         roles = self.hda_manager.serialize_dataset_association_roles(trans, hda)
         return DatasetAssociationRoles.parse_obj(roles)
 
+    def update(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        id: EncodedDatabaseIdField,
+        payload: Dict[str, Any],
+        serialization_params: SerializationParams,
+        contents_type: HistoryContentType = HistoryContentType.dataset,
+    ):
+        """
+        Updates the values for the history content item with the given ``id``
+
+        :param  history_id: encoded id string of the items's History
+        :param  id:         the encoded id of the history item to update
+        :param  payload:    a dictionary containing any or all the fields for
+                            HDA or HDCA with values different than the defaults
+
+        :returns:   an error object if an error occurred or a dictionary containing
+                    any values that were different from the original and, therefore, updated
+        """
+        if contents_type == HistoryContentType.dataset:
+            update_hda_payload = UpdateHDAPayload.parse_obj(payload)
+            return self.__update_dataset(trans, history_id, id, update_hda_payload, serialization_params)
+        elif contents_type == HistoryContentType.dataset_collection:
+            return self.__update_dataset_collection(trans, id, payload)
+        else:
+            raise exceptions.UnknownContentsType(f'Unknown contents type: {contents_type}')
+
+    def update_batch(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        payload: UpdateHistoryContentsBatchPayload,
+        serialization_params: SerializationParams,
+    ):
+        """
+        PUT /api/histories/{history_id}/contents
+
+        :type   history_id: str
+        :param  history_id: encoded id string of the history containing supplied items
+        :type   id:         str
+        :param  id:         the encoded id of the history to update
+        :type   payload:    dict
+        :param  payload:    a dictionary containing any or all the
+
+        :rtype:     dict
+        :returns:   an error object if an error occurred or a dictionary containing
+                    any values that were different from the original and, therefore, updated
+        """
+        history = self.history_manager.get_owned(
+            self.decode_id(history_id), trans.user, current_history=trans.history
+        )
+        items = payload.items
+        hda_ids = []
+        hdca_ids = []
+        for item in items:
+            contents_type = item.history_content_type
+            if contents_type == HistoryContentType.dataset:
+                decoded_id = self.decode_id(item.id)
+                hda_ids.append(decoded_id)
+            else:
+                hdca_ids.append(item.id)
+        payload_dict = payload.dict(exclude_defaults=True, exclude_unset=True)
+        hdas = self.__datasets_for_update(trans, history, hda_ids, payload_dict)
+        rval = []
+        for hda in hdas:
+            self.__deserialize_dataset(hda, payload_dict, trans)
+            serialization_params["default_view"] = "summary"
+            rval.append(self.hda_serializer.serialize_to_view(
+                hda, user=trans.user, trans=trans, **serialization_params
+            ))
+        for hdca_id in hdca_ids:
+            self.__update_dataset_collection(trans, hdca_id, payload)
+            dataset_collection_instance = self.__get_accessible_collection(trans, hdca_id)
+            rval.append(self.__collection_dict(trans, dataset_collection_instance, view="summary"))
+        return rval
+
+    def __update_dataset_collection(self, trans, id: EncodedDatabaseIdField, payload: Dict[str, Any]):
+        return self.dataset_collection_manager.update(trans, "history", id, payload)
+
+    def __update_dataset(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        id: EncodedDatabaseIdField,
+        payload: UpdateHDAPayload,
+        serialization_params: SerializationParams
+    ):
+        # anon user: ensure that history ids match up and the history is the current,
+        #   check for uploading, and use only the subset of attribute keys manipulatable by anon users
+        payload_dict = payload.dict(exclude_defaults=True, exclude_unset=True)
+        decoded_id = self.decode_id(id)
+        history = self.history_manager.get_owned(
+            self.decode_id(history_id), trans.user, current_history=trans.history
+        )
+        hda = self.__datasets_for_update(trans, history, [decoded_id], payload_dict)[0]
+        if hda:
+            self.__deserialize_dataset(hda, payload_dict, trans)
+            serialization_params["default_view"] = 'detailed'
+            return self.hda_serializer.serialize_to_view(
+                hda, user=trans.user, trans=trans, **serialization_params
+            )
+        return {}
+
+    def __datasets_for_update(
+        self, trans,
+        history: History,
+        hda_ids: List[int],
+        payload: Dict[str, Any]
+    ):
+        anonymous_user = not trans.user_is_admin and trans.user is None
+        if anonymous_user:
+            anon_allowed_payload = {}
+            if 'deleted' in payload:
+                anon_allowed_payload['deleted'] = payload['deleted']
+            if 'visible' in payload:
+                anon_allowed_payload['visible'] = payload['visible']
+            payload = anon_allowed_payload
+
+        hdas = self.hda_manager.get_owned_ids(hda_ids, history=history)
+
+        # only check_state if not deleting, otherwise cannot delete uploading files
+        check_state = not payload.get('deleted', False)
+        if check_state:
+            for hda in hdas:
+                hda = self.hda_manager.error_if_uploading(hda)
+
+        return hdas
+
+    def __deserialize_dataset(self, hda, payload, trans):
+        self.hda_deserializer.deserialize(hda, payload, user=trans.user, trans=trans)
+        # TODO: this should be an effect of deleting the hda
+        if payload.get('deleted', False):
+            self.hda_manager.stop_creating_job(hda)
+
     def __index_legacy(
         self, trans,
         history_id: EncodedDatabaseIdField,
@@ -891,10 +1027,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             serialization_params, filter_parameters
         )
 
-    def __collection_dict(self, trans, dataset_collection_instance, **kwds):
-        return dictify_dataset_collection_instance(dataset_collection_instance,
-            security=trans.security, parent=dataset_collection_instance.history, **kwds)
-
     @expose_api_anonymous
     def show(self, trans, id, history_id, **kwd):
         """
@@ -1000,13 +1132,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
 
         return contents_type
 
-    def __get_accessible_collection(self, trans, id, history_id):
-        return trans.app.dataset_collection_manager.get_dataset_collection_instance(
-            trans=trans,
-            instance_type="history",
-            id=id
-        )
-
     @expose_api_raw_anonymous
     def download_dataset_collection(self, trans, id, history_id=None, **kwd):
         """
@@ -1190,30 +1315,9 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :returns:   an error object if an error occurred or a dictionary containing
                     any values that were different from the original and, therefore, updated
         """
-        items = payload.get("items")
-        hda_ids = []
-        hdca_ids = []
-        for item in items:
-            contents_type = item["history_content_type"]
-            if contents_type == "dataset":
-                decoded_id = self.decode_id(item["id"])
-                hda_ids.append(decoded_id)
-            else:
-                hdca_ids.append(item["id"])
-
-        history = self.history_manager.get_owned(self.decode_id(history_id), trans.user,
-                                                 current_history=trans.history)
-        hdas = self.__datasets_for_update(trans, history, hda_ids, payload)
-        rval = []
-        for hda in hdas:
-            self.__deserialize_dataset(hda, payload, trans)
-            rval.append(self.hda_serializer.serialize_to_view(hda,
-                                                              user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'summary')))
-        for hdca_id in hdca_ids:
-            self.__update_dataset_collection(trans, history_id, hdca_id, payload, **kwd)
-            dataset_collection_instance = self.__get_accessible_collection(trans, hdca_id, history_id)
-            rval.append(self.__collection_dict(trans, dataset_collection_instance, view="summary"))
-        return rval
+        serialization_params = parse_serialization_params(**kwd)
+        update_payload = UpdateHistoryContentsBatchPayload.parse_obj(payload)
+        return self.service.update_batch(trans, history_id, update_payload, serialization_params)
 
     @expose_api_anonymous
     def update(self, trans, history_id, id, payload, **kwd):
@@ -1237,14 +1341,11 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :returns:   an error object if an error occurred or a dictionary containing
             any values that were different from the original and, therefore, updated
         """
-        # TODO: PUT /api/histories/{encoded_history_id} payload = { rating: rating } (w/ no security checks)
-        contents_type = kwd.get('type', 'dataset')
-        if contents_type == "dataset":
-            return self.__update_dataset(trans, history_id, id, payload, **kwd)
-        elif contents_type == "dataset_collection":
-            return self.__update_dataset_collection(trans, history_id, id, payload, **kwd)
-        else:
-            return self.__handle_unknown_contents_type(trans, contents_type)
+        serialization_params = parse_serialization_params(**kwd)
+        contents_type = self.__get_contents_type(trans, kwd)
+        return self.service.update(
+            trans, history_id, id, payload, serialization_params, contents_type
+        )
 
     @expose_api_anonymous
     def validate(self, trans, history_id, history_content_id, payload=None, **kwd):
@@ -1269,49 +1370,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             self.hda_manager.set_metadata(trans, hda, overwrite=True, validate=True)
         return {}
 
-    def __update_dataset(self, trans, history_id, id, payload, **kwd):
-        # anon user: ensure that history ids match up and the history is the current,
-        #   check for uploading, and use only the subset of attribute keys manipulatable by anon users
-        decoded_id = self.decode_id(id)
-        history = self.history_manager.get_owned(self.decode_id(history_id), trans.user,
-                                                 current_history=trans.history)
-        hda = self.__datasets_for_update(trans, history, [decoded_id], payload)[0]
-        if hda:
-            self.__deserialize_dataset(hda, payload, trans)
-            return self.hda_serializer.serialize_to_view(hda,
-                                                         user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed'))
-
-        return {}
-
-    def __datasets_for_update(self, trans, history, hda_ids, payload):
-        anonymous_user = not trans.user_is_admin and trans.user is None
-        if anonymous_user:
-            anon_allowed_payload = {}
-            if 'deleted' in payload:
-                anon_allowed_payload['deleted'] = payload['deleted']
-            if 'visible' in payload:
-                anon_allowed_payload['visible'] = payload['visible']
-            payload = anon_allowed_payload
-
-        hdas = self.hda_manager.get_owned_ids(hda_ids, history=history)
-
-        # only check_state if not deleting, otherwise cannot delete uploading files
-        check_state = not payload.get('deleted', False)
-        if check_state:
-            for hda in hdas:
-                hda = self.hda_manager.error_if_uploading(hda)
-
-        return hdas
-
-    def __deserialize_dataset(self, hda, payload, trans):
-        self.hda_deserializer.deserialize(hda, payload, user=trans.user, trans=trans)
-        # TODO: this should be an effect of deleting the hda
-        if payload.get('deleted', False):
-            self.hda_manager.stop_creating_job(hda)
-
-    def __update_dataset_collection(self, trans, history_id, id, payload, **kwd):
-        return trans.app.dataset_collection_manager.update(trans, "history", id, payload)
-
     # TODO: allow anonymous del/purge and test security on this
     @expose_api
     def delete(self, trans, history_id, id, purge=False, recursive=False, **kwd):

From 8559b678d038daa2748c5c83b5ec706b93bd7daf Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 14 Jul 2021 16:28:45 +0200
Subject: [PATCH 100/452] Refactor `validate` action

---
 .../webapps/galaxy/api/history_contents.py    | 33 +++++++++++++++----
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index e0ade4387e9..d47b777d028 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -587,6 +587,31 @@ class HistoriesContentsService(ServiceBase):
             rval.append(self.__collection_dict(trans, dataset_collection_instance, view="summary"))
         return rval
 
+    def validate(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        history_content_id: EncodedDatabaseIdField
+    ):
+        """
+        Updates the values for the history content item with the given ``id``
+
+        :type   history_id: str
+        :param  history_id: encoded id string of the items's History
+        :type   id:         str
+        :param  id:         the encoded id of the history item to validate
+
+        :rtype:     dict
+        :returns:   TODO
+        """
+        decoded_id = self.decode_id(history_content_id)
+        history = self.history_manager.get_owned(
+            self.decode_id(history_id), trans.user, current_history=trans.history
+        )
+        hda = self.hda_manager.get_owned_ids([decoded_id], history=history)[0]
+        if hda:
+            self.hda_manager.set_metadata(trans, hda, overwrite=True, validate=True)
+        return {}
+
     def __update_dataset_collection(self, trans, id: EncodedDatabaseIdField, payload: Dict[str, Any]):
         return self.dataset_collection_manager.update(trans, "history", id, payload)
 
@@ -1362,13 +1387,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :rtype:     dict
         :returns:   TODO
         """
-        decoded_id = self.decode_id(history_content_id)
-        history = self.history_manager.get_owned(self.decode_id(history_id), trans.user,
-                                                 current_history=trans.history)
-        hda = self.hda_manager.get_owned_ids([decoded_id], history=history)[0]
-        if hda:
-            self.hda_manager.set_metadata(trans, hda, overwrite=True, validate=True)
-        return {}
+        return self.service.validate(trans, history_id, history_content_id)
 
     # TODO: allow anonymous del/purge and test security on this
     @expose_api

From 1f94f3c4c9f73f97266e705aa1a887e3020f0a0d Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 14 Jul 2021 18:10:03 +0200
Subject: [PATCH 101/452] Make `download_url` field of type `RelativeUrl`

Sometimes the URL is not absolute and the model validation fail because there is no valid scheme in the URL.
---
 lib/galaxy/schema/schema.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 7035cec9653..3f31160c473 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -54,7 +54,7 @@ RelativeUrlField: RelativeUrl = Field(
     deprecated=False  # TODO Should this field be deprecated in FastAPI?
 )
 
-DownloadUrlField: AnyUrl = Field(
+DownloadUrlField: RelativeUrl = Field(
     ...,
     title="Download URL",
     description="The URL to download this item from the server.",
@@ -227,7 +227,7 @@ class HistoryContentSource(str, Enum):
 class TagCollection(Model):
     """Represents the collection of tags associated with an item."""
     __root__: List[str] = Field(
-        [],
+        default=...,
         title="Tags",
         description="The collection of tags associated with an item.",
     )
@@ -240,7 +240,7 @@ class MetadataFile(Model):
         title="File Type",
         description="TODO",
     )
-    download_url: AnyUrl = DownloadUrlField
+    download_url: RelativeUrl = DownloadUrlField
 
 
 class DatasetPermissions(Model):
@@ -479,7 +479,7 @@ class HDADetailed(HDASummary):
         description="The message with details about the datatype validation result for this dataset.",
     )
     annotation: Optional[str] = AnnotationField
-    download_url: AnyUrl = DownloadUrlField
+    download_url: RelativeUrl = DownloadUrlField
     type: str = Field(
         "file",
         const=True,
@@ -888,7 +888,7 @@ class JobExportHistoryArchive(Model):
         title="Up to Date",
         description="False, if a new export archive should be generated for the corresponding history.",
     )
-    download_url: str = Field(
+    download_url: RelativeUrl = Field(
         ...,
         title="Download URL",
         description="Relative API URL to download the exported history archive.",

From 08f08d908c580fe73e244064cddfcca121774102 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 14 Jul 2021 19:21:47 +0200
Subject: [PATCH 102/452] Refactor `delete` action

---
 lib/galaxy/schema/schema.py                   | 13 +++
 .../webapps/galaxy/api/history_contents.py    | 91 ++++++++++++-------
 2 files changed, 72 insertions(+), 32 deletions(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 3f31160c473..b0c6f4ae6ea 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -2309,3 +2309,16 @@ class UpdateDatasetPermissionsPayload(BaseModel):
         title="Modify IDs",
         description="A list of role encoded IDs defining roles that should have modify permission on the dataset.",
     )
+
+
+class DeleteHDCAResult(Model):
+    id: EncodedDatabaseIdField = Field(
+        ...,
+        title="ID",
+        description="The encoded ID of the collection.",
+    )
+    deleted: bool = Field(
+        ...,
+        title="Deleted",
+        description="True if the collection was successfully deleted.",
+    )
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index d47b777d028..b1d4b57a284 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -58,6 +58,7 @@ from galaxy.schema.schema import (
     ColletionSourceType,
     DatasetAssociationRoles,
     DatasetPermissionAction,
+    DeleteHDCAResult,
     HDABeta,
     HDADetailed,
     HDASummary,
@@ -96,10 +97,8 @@ log = logging.getLogger(__name__)
 
 DatasetDetailsType = Union[Set[EncodedDatabaseIdField], Literal['all']]
 
-AnyHistoryContentItem = Union[
-    HDASummary, HDADetailed, HDABeta,
-    HDCASummary, HDCADetailed, HDCABeta
-]
+AnyHDA = Union[HDASummary, HDADetailed, HDABeta]
+AnyHistoryContentItem = Union[AnyHDA, HDCASummary, HDCADetailed, HDCABeta]
 
 AnyJobStateSummary = Union[
     JobStateSummary,
@@ -612,6 +611,57 @@ class HistoriesContentsService(ServiceBase):
             self.hda_manager.set_metadata(trans, hda, overwrite=True, validate=True)
         return {}
 
+    def delete(
+        self, trans,
+        id,
+        serialization_params: SerializationParams,
+        contents_type: HistoryContentType = HistoryContentType.dataset,
+        purge: bool = False,
+        recursive: bool = False,
+    ) -> Union[AnyHDA, DeleteHDCAResult]:
+        """
+        Delete the history content with the given ``id`` and specified type (defaults to dataset)
+
+        .. note:: Currently does not stop any active jobs for which this dataset is an output.
+
+        :param  id:     the encoded id of the history item to delete
+        :type   recursive:  bool
+        :param  recursive:  if True, and deleted an HDCA also delete containing HDAs
+        :type   purge:  bool
+        :param  purge:  if True, purge the target HDA or child HDAs of the target HDCA
+
+        :rtype:     dict
+        :returns:   an error object if an error occurred or a dictionary containing:
+            * id:         the encoded id of the history,
+            * deleted:    if the history content was marked as deleted,
+            * purged:     if the history content was purged
+        """
+        if contents_type == HistoryContentType.dataset:
+            return self.__delete_dataset(trans, id, purge, serialization_params)
+        elif contents_type == HistoryContentType.dataset_collection:
+            self.dataset_collection_manager.delete(trans, "history", id, recursive=recursive, purge=purge)
+            return DeleteHDCAResult(id=id, deleted=True)
+        else:
+            raise exceptions.UnknownContentsType(f'Unknown contents type: {contents_type}')
+
+    def __delete_dataset(
+        self, trans,
+        id: EncodedDatabaseIdField,
+        purge: bool,
+        serialization_params: SerializationParams
+    ):
+        hda = self.hda_manager.get_owned(self.decode_id(id), trans.user, current_history=trans.history)
+        self.hda_manager.error_if_uploading(hda)
+
+        if purge:
+            self.hda_manager.purge(hda)
+        else:
+            self.hda_manager.delete(hda)
+        serialization_params["default_view"] = 'detailed'
+        return self.hda_serializer.serialize_to_view(
+            hda, user=trans.user, trans=trans, **serialization_params
+        )
+
     def __update_dataset_collection(self, trans, id: EncodedDatabaseIdField, payload: Dict[str, Any]):
         return self.dataset_collection_manager.update(trans, "history", id, payload)
 
@@ -1422,38 +1472,15 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             * deleted:    if the history content was marked as deleted,
             * purged:     if the history content was purged
         """
-        contents_type = kwd.get('type', 'dataset')
-        if contents_type == "dataset":
-            return self.__delete_dataset(trans, history_id, id, purge=purge, **kwd)
-        elif contents_type == "dataset_collection":
-            purge = util.string_as_bool(purge)
-            recursive = util.string_as_bool(recursive)
-            if kwd.get('payload', None):
-                # payload takes priority
-                purge = util.string_as_bool(kwd['payload'].get('purge', purge))
-                recursive = util.string_as_bool(kwd['payload'].get('recursive', recursive))
-
-            trans.app.dataset_collection_manager.delete(trans, "history", id, recursive=recursive, purge=purge)
-            return {'id': id, "deleted": True}
-        else:
-            return self.__handle_unknown_contents_type(trans, contents_type)
-
-    def __delete_dataset(self, trans, history_id, id, purge, **kwd):
-        # get purge from the query or from the request body payload (a request body is optional here)
+        serialization_params = parse_serialization_params(**kwd)
+        contents_type = self.__get_contents_type(trans, kwd)
         purge = util.string_as_bool(purge)
+        recursive = util.string_as_bool(recursive)
         if kwd.get('payload', None):
             # payload takes priority
             purge = util.string_as_bool(kwd['payload'].get('purge', purge))
-
-        hda = self.hda_manager.get_owned(self.decode_id(id), trans.user, current_history=trans.history)
-        self.hda_manager.error_if_uploading(hda)
-
-        if purge:
-            self.hda_manager.purge(hda)
-        else:
-            self.hda_manager.delete(hda)
-        return self.hda_serializer.serialize_to_view(hda,
-                                                     user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed'))
+            recursive = util.string_as_bool(kwd['payload'].get('recursive', recursive))
+        return self.service.delete(trans, id, serialization_params, contents_type, purge, recursive)
 
     def __handle_unknown_contents_type(self, trans, contents_type):
         raise exceptions.UnknownContentsType(f'Unknown contents type: {type}')

From 52cbaba160099a7e43f40a598f703260abef8976 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 14 Jul 2021 12:42:23 -0400
Subject: [PATCH 103/452] Allow specifying useful classes of tools for mapping
 in job conf.

---
 lib/galaxy/jobs/__init__.py                   | 45 ++++++++++++++-----
 lib/galaxy/tools/__init__.py                  | 14 +++++-
 .../webapps/galaxy/job_config_schema.yml      |  2 +
 test/integration/delay_job_conf.yml           |  2 +-
 test/integration/embedded_pulsar_job_conf.xml |  2 +-
 ...dded_pulsar_metadata_extended_job_conf.yml |  2 +-
 .../embedded_pulsar_metadata_job_conf.yml     |  2 +-
 .../embedded_pulsar_singularity_job_conf.yml  |  2 +-
 .../resubmission_default_job_conf.yml         |  2 +-
 .../resubmission_dynamic_job_conf.xml         |  2 +-
 test/integration/resubmission_job_conf.yml    |  2 +-
 .../resubmission_pulsar_job_conf.xml          |  2 +-
 test/integration/singularity_job_conf.xml     |  2 +-
 test/unit/jobs/job_conf.sample_advanced.yml   |  8 ++++
 test/unit/tools_support.py                    |  2 +-
 15 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 7a2ff427e33..01d2d1a1105 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -82,6 +82,7 @@ DEFAULT_JOB_SHELL = '/bin/bash'
 DEFAULT_LOCAL_WORKERS = 4
 
 DEFAULT_CLEANUP_JOB = "always"
+VALID_TOOL_CLASSES = ["local", "requires_galaxy"]
 
 
 class JobDestination(Bunch):
@@ -244,10 +245,10 @@ def job_config_xml_to_dict(config, root):
     tools = root.find('tools')
     config_dict['tools'] = []
     if tools is not None:
-        for tool in ConfiguresHandlers._findall_with_required(tools, 'tool'):
+        for tool in tools.findall('tool'):
             # There can be multiple definitions with identical ids, but different params
             tool_mapping_conf = {}
-            for key in ['handler', 'destination', 'id', 'resources']:
+            for key in ['handler', 'destination', 'id', 'resources', 'class']:
                 value = tool.get(key)
                 if value:
                     if key == "destination":
@@ -283,6 +284,7 @@ class JobConfiguration(ConfiguresHandlers):
     handlers: dict
     handler_runner_plugins: Dict[str, str]
     tools: Dict[str, list]
+    tool_classes: Dict[str, list]
     resource_groups: Dict[str, list]
     destinations: Dict[str, tuple]
     resource_parameters: Dict[str, Any]
@@ -317,6 +319,7 @@ class JobConfiguration(ConfiguresHandlers):
         self.destinations = {}
         self.default_destination_id = None
         self.tools = {}
+        self.tool_classes = {}
         self.resource_groups = {}
         self.default_resource_group = None
         self.resource_parameters = {}
@@ -473,9 +476,18 @@ class JobConfiguration(ConfiguresHandlers):
 
         tools = job_config_dict.get('tools', [])
         for tool in tools:
-            tool_id = tool.get('id').lower().rstrip('/')
-            if tool_id not in self.tools:
-                self.tools[tool_id] = list()
+            raw_tool_id = tool.get('id')
+            tool_class = tool.get('class')
+            if raw_tool_id is not None:
+                assert tool_class is None
+                tool_id = raw_tool_id.lower().rstrip('/')
+                if tool_id not in self.tools:
+                    self.tools[tool_id] = list()
+            else:
+                assert tool_class in VALID_TOOL_CLASSES, tool_class
+                if tool_class not in self.tool_classes:
+                    self.tool_classes[tool_class] = list()
+
             params = tool.get("params")
             if params is None:
                 params = {}
@@ -486,7 +498,12 @@ class JobConfiguration(ConfiguresHandlers):
                 tool["params"] = params
             if "environment" in tool:
                 tool["destination"] = tool.pop("environment")
-            self.tools[tool_id].append(JobToolConfiguration(**dict(tool.items())))
+
+            jtc = JobToolConfiguration(**dict(tool.items()))
+            if raw_tool_id:
+                self.tools[tool_id].append(jtc)
+            else:
+                self.tool_classes[tool_class].append(jtc)
 
         types = dict(registered_user_concurrent_jobs=int,
                      anonymous_user_concurrent_jobs=int,
@@ -693,7 +710,7 @@ class JobConfiguration(ConfiguresHandlers):
         return JobToolConfiguration(id='_default_', handler=self.default_handler_id, destination=self.default_destination_id)
 
     # Called upon instantiation of a Tool object
-    def get_job_tool_configurations(self, ids):
+    def get_job_tool_configurations(self, ids, tool_classes):
         """
         Get all configured JobToolConfigurations for a tool ID, or, if given
         a list of IDs, the JobToolConfigurations for the first id in ``ids``
@@ -713,6 +730,7 @@ class JobConfiguration(ConfiguresHandlers):
         * Tool config tool id: ``filter_tool``
         """
         rval = []
+        match_found = False
         # listify if ids is a single (string) id
         ids = util.listify(ids)
         for id in ids:
@@ -723,11 +741,16 @@ class JobConfiguration(ConfiguresHandlers):
                 for job_tool_configuration in self.tools[id]:
                     if not job_tool_configuration.params:
                         break
-                else:
-                    rval.append(self.default_job_tool_configuration)
                 rval.extend(self.tools[id])
-                break
-        else:
+                match_found = True
+
+        if not match_found:
+            for tool_class in tool_classes:
+                if tool_class in self.tool_classes:
+                    rval.extend(self.tool_classes[tool_class])
+                    match_found = True
+                    break
+        if not match_found:
             rval.append(self.default_job_tool_configuration)
         return rval
 
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 8f8eed52b6a..662ce439517 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -490,6 +490,7 @@ class Tool(Dictifiable):
     requires_setting_metadata = True
     produces_entry_points = False
     default_tool_action = DefaultToolAction
+    tool_type_local = False
     dict_collection_visible_keys = ['id', 'name', 'version', 'description', 'labels']
 
     def __init__(self, config_file, tool_source, app, guid=None, repository_id=None, tool_shed_repository=None, allow_code_files=True, dynamic=False):
@@ -897,7 +898,16 @@ class Tool(Dictifiable):
 
         # In the toolshed context, there is no job config.
         if hasattr(self.app, 'job_config'):
-            self.job_tool_configurations = self.app.job_config.get_job_tool_configurations(self_ids)
+            # Order of this list must match documentation in job_conf.sample_advanced.yml
+            tool_classes = []
+            if self.tool_type_local:
+                tool_classes.append("local")
+            elif self.old_id in ['upload1', '__DATA_FETCH__']:
+                tool_classes.append("local")
+            if self.requires_galaxy_python_environment:
+                tool_classes.append("requires_galaxy")
+
+            self.job_tool_configurations = self.app.job_config.get_job_tool_configurations(self_ids, tool_classes)
 
         # Is this a 'hidden' tool (hidden in tool menu)
         self.hidden = tool_source.parse_hidden()
@@ -2431,6 +2441,7 @@ class OutputParameterJSONTool(Tool):
 class ExpressionTool(Tool):
     requires_js_runtime = True
     tool_type = 'expression'
+    tool_type_local = True
     EXPRESSION_INPUTS_NAME = "_expression_inputs_.json"
 
     def parse_command(self, tool_source):
@@ -2772,6 +2783,7 @@ class DataManagerTool(OutputParameterJSONTool):
 class DatabaseOperationTool(Tool):
     default_tool_action = ModelOperationToolAction
     require_dataset_ok = True
+    tool_type_local = True
 
     @property
     def valid_input_states(self):
diff --git a/lib/galaxy/webapps/galaxy/job_config_schema.yml b/lib/galaxy/webapps/galaxy/job_config_schema.yml
index f025bc85be1..63b5532a7af 100644
--- a/lib/galaxy/webapps/galaxy/job_config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/job_config_schema.yml
@@ -81,6 +81,8 @@ mapping:
         mapping:
           id:
             type: str
+          class:
+            type: str
           handler:
             type: str
           environment:
diff --git a/test/integration/delay_job_conf.yml b/test/integration/delay_job_conf.yml
index 61021c1fe5e..b2d5781f6ec 100644
--- a/test/integration/delay_job_conf.yml
+++ b/test/integration/delay_job_conf.yml
@@ -14,5 +14,5 @@ execution:
     upload_dest:
       runner: local
 tools:
-  - id: upload1
+  - class: local
     destination: upload_dest
diff --git a/test/integration/embedded_pulsar_job_conf.xml b/test/integration/embedded_pulsar_job_conf.xml
index f7d08cb3347..26c1f158189 100644
--- a/test/integration/embedded_pulsar_job_conf.xml
+++ b/test/integration/embedded_pulsar_job_conf.xml
@@ -13,6 +13,6 @@
         </destination>
     </destinations>
     <tools>
-        <tool id="upload1" destination="local" />
+        <tool class="local" destination="local" />
     </tools>
 </job_conf>
diff --git a/test/integration/embedded_pulsar_metadata_extended_job_conf.yml b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml
index 76c7b8b5d22..aeab99c5454 100644
--- a/test/integration/embedded_pulsar_metadata_extended_job_conf.yml
+++ b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml
@@ -20,5 +20,5 @@ execution:
             action: none
 
 tools:
-- id: upload1
+- class: local
   environment: local
diff --git a/test/integration/embedded_pulsar_metadata_job_conf.yml b/test/integration/embedded_pulsar_metadata_job_conf.yml
index 76520bf4ed5..62e9ddcee98 100644
--- a/test/integration/embedded_pulsar_metadata_job_conf.yml
+++ b/test/integration/embedded_pulsar_metadata_job_conf.yml
@@ -15,5 +15,5 @@ execution:
       default_file_action: copy
 
 tools:
-- id: upload1
+- class: local
   environment: local
diff --git a/test/integration/embedded_pulsar_singularity_job_conf.yml b/test/integration/embedded_pulsar_singularity_job_conf.yml
index cdcdedc19ff..d224519c951 100644
--- a/test/integration/embedded_pulsar_singularity_job_conf.yml
+++ b/test/integration/embedded_pulsar_singularity_job_conf.yml
@@ -15,5 +15,5 @@ execution:
       singularity_required: true
 
 tools:
-- id: upload1
+- class: local
   environment: local
diff --git a/test/integration/resubmission_default_job_conf.yml b/test/integration/resubmission_default_job_conf.yml
index accb17eabd3..2f0b2321694 100644
--- a/test/integration/resubmission_default_job_conf.yml
+++ b/test/integration/resubmission_default_job_conf.yml
@@ -29,6 +29,6 @@ resources:
     test: [test_name,failure_state,initial_target_environment,run_for]
 
 tools:
-  - id: upload1
+  - class: local
     environment: local
     resources: upload
diff --git a/test/integration/resubmission_dynamic_job_conf.xml b/test/integration/resubmission_dynamic_job_conf.xml
index 6221974fdc4..cf139722b83 100644
--- a/test/integration/resubmission_dynamic_job_conf.xml
+++ b/test/integration/resubmission_dynamic_job_conf.xml
@@ -26,7 +26,7 @@
     </destinations>
 
     <tools>
-        <tool id="upload1" destination="local" />
+        <tool class="local" destination="local" />
     </tools>
 
 </job_conf>
diff --git a/test/integration/resubmission_job_conf.yml b/test/integration/resubmission_job_conf.yml
index e55c236c718..cc2cb9e8998 100644
--- a/test/integration/resubmission_job_conf.yml
+++ b/test/integration/resubmission_job_conf.yml
@@ -109,6 +109,6 @@ resources:
     test: [test_name,failure_state,initial_target_environment,run_for]
 
 tools:
-  - id: upload1
+  - class: local
     environment: local
     resources: upload
diff --git a/test/integration/resubmission_pulsar_job_conf.xml b/test/integration/resubmission_pulsar_job_conf.xml
index 9ee1c867ae2..da6a2cd6559 100644
--- a/test/integration/resubmission_pulsar_job_conf.xml
+++ b/test/integration/resubmission_pulsar_job_conf.xml
@@ -22,6 +22,6 @@
     </destinations>
 
     <tools>
-        <tool id="upload1" destination="local" resources="upload" />
+        <tool class="local" destination="local" resources="upload" />
     </tools>
 </job_conf>
diff --git a/test/integration/singularity_job_conf.xml b/test/integration/singularity_job_conf.xml
index ccdc921727e..1a500dcab3f 100644
--- a/test/integration/singularity_job_conf.xml
+++ b/test/integration/singularity_job_conf.xml
@@ -15,7 +15,7 @@
     </destinations>
 
     <tools>
-        <tool id="upload1" destination="local_upload" />
+        <tool class="local" destination="local_upload" />
     </tools>
 
 </job_conf>
diff --git a/test/unit/jobs/job_conf.sample_advanced.yml b/test/unit/jobs/job_conf.sample_advanced.yml
index c0edf199626..89ae458a310 100644
--- a/test/unit/jobs/job_conf.sample_advanced.yml
+++ b/test/unit/jobs/job_conf.sample_advanced.yml
@@ -1016,6 +1016,14 @@ tools:
   id: foo
   handler: handler0
   source: trackster
+- 
+  # Classes can be used to map groups of tools - the current classes include
+  # - local (these special tools that aren't parameterized for remote execution - expression tools, upload, etc..)
+  # - requires_galaxy (these special tools require Galaxy's Python environment during execution)
+  # If a tool matches multiple classes, it will match the first class according to the order listed
+  # above in this document (local, requires_galaxy).
+  class: local
+  environment: local
 
 resources:
   default: default
diff --git a/test/unit/tools_support.py b/test/unit/tools_support.py
index 5c86cd64dbd..b3f5326b519 100644
--- a/test/unit/tools_support.py
+++ b/test/unit/tools_support.py
@@ -94,7 +94,7 @@ class UsesTools:
         self.app.config.drmaa_external_runjob_script = ""
         self.app.config.tool_secret = "testsecret"
         self.app.config.track_jobs_in_database = False
-        self.app.job_config["get_job_tool_configurations"] = lambda ids: [Bunch(handler=Bunch())]
+        self.app.job_config["get_job_tool_configurations"] = lambda ids, tool_classes: [Bunch(handler=Bunch())]
 
     def __setup_tool(self):
         tool_source = get_tool_source(self.tool_file)

From e21047dc36be7dc4b77ea7787017a9e420d39bbe Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 14 Jul 2021 10:42:59 -0400
Subject: [PATCH 104/452] Test composite.xml under Pulsar.

---
 test/integration/test_pulsar_embedded.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py
index 411daad976e..cb99e732fe3 100644
--- a/test/integration/test_pulsar_embedded.py
+++ b/test/integration/test_pulsar_embedded.py
@@ -21,6 +21,7 @@ class EmbeddedPulsarIntegrationInstance(integration_util.IntegrationInstance):
 instance = integration_util.integration_module_instance(EmbeddedPulsarIntegrationInstance)
 
 test_tools = integration_util.integration_tool_runner([
+    "composite",
     "simple_constructs",
     "multi_data_param",
     "output_filter",

From c33cf17eb2db57357ed0f3685deec84091ae044d Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 14 Jul 2021 13:41:12 -0400
Subject: [PATCH 105/452] Rev pulsar-galaxy-lib to fix composite inputs.

---
 lib/galaxy/dependencies/dev-requirements.txt    | 2 +-
 lib/galaxy/dependencies/pinned-requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index 15cbbc5001e..5290b553919 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.6
+pulsar-galaxy-lib==0.14.8
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index 9b18b2f30ae..87a30062242 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.6
+pulsar-galaxy-lib==0.14.8
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"

From 498cb22067846380ec8ed33dae8046567c868a49 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 14 Jul 2021 14:37:59 -0400
Subject: [PATCH 106/452] Fix Pulsar dynamic output collection for various
 simple tool specified patterns.

Fixes most typical hits of https://github.com/galaxyproject/pulsar/issues/212.
---
 lib/galaxy/jobs/runners/pulsar.py             |  6 ++++
 .../tool_util/parser/output_collection_def.py |  9 ++++++
 lib/galaxy/tool_util/parser/output_objects.py | 31 ++++++++++++++++++-
 lib/galaxy/tools/__init__.py                  |  8 +++++
 test/integration/test_pulsar_embedded.py      |  1 +
 5 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index 4124ec3479c..fbbf2148a0f 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -27,6 +27,8 @@ from pulsar.client import (
     submit_job as pulsar_submit_job,
     url_to_destination_params
 )
+# TODO: Perform pulsar release with this included in the client package
+from pulsar.client.staging import DEFAULT_DYNAMIC_COLLECTION_PATTERN
 
 from galaxy import model
 from galaxy.jobs import (
@@ -761,6 +763,10 @@ class PulsarJobRunner(AsynchronousJobRunner):
             # if Pulsar is doing remote metadata and the remote metadata is extended,
             # we only need to recover the final model store.
             dynamic_outputs = EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN
+        else:
+            # otherwise collect everything we might need
+            dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:]
+            dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns)
         client_outputs = ClientOutputs(
             working_directory=job_wrapper.tool_working_directory,
             metadata_directory=metadata_directory,
diff --git a/lib/galaxy/tool_util/parser/output_collection_def.py b/lib/galaxy/tool_util/parser/output_collection_def.py
index a6c05ed1e13..be13bdd9fa6 100644
--- a/lib/galaxy/tool_util/parser/output_collection_def.py
+++ b/lib/galaxy/tool_util/parser/output_collection_def.py
@@ -1,6 +1,7 @@
 """ This module define an abstract class for reasoning about Galaxy's
 dataset collection after jobs are finished.
 """
+from typing import List
 
 from galaxy.util import asbool
 from .util import is_dict
@@ -92,6 +93,10 @@ class DatasetCollectionDescription:
             'recurse': self.recurse,
         }
 
+    @property
+    def discover_patterns(self) -> List[str]:
+        return []
+
 
 class ToolProvidedMetadataDatasetCollection(DatasetCollectionDescription):
 
@@ -139,6 +144,10 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription):
         })
         return as_dict
 
+    @property
+    def discover_patterns(self):
+        return [self.pattern]
+
 
 DEFAULT_DATASET_COLLECTOR_DESCRIPTION = FilePatternDatasetCollectionDescription(
     default_dbkey=LEGACY_DEFAULT_DBKEY,
diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py
index 2750c081714..4140be8f05d 100644
--- a/lib/galaxy/tool_util/parser/output_objects.py
+++ b/lib/galaxy/tool_util/parser/output_objects.py
@@ -1,6 +1,8 @@
+from typing import List
+
 from galaxy.util.dictifiable import Dictifiable
 from .output_actions import ToolOutputActionGroup
-from .output_collection_def import dataset_collector_descriptions_from_output_dict
+from .output_collection_def import dataset_collector_descriptions_from_output_dict, DatasetCollectionDescription
 
 
 class ToolOutputBase(Dictifiable):
@@ -17,6 +19,10 @@ class ToolOutputBase(Dictifiable):
     def to_dict(self, view='collection', value_mapper=None, app=None):
         return super().to_dict(view=view, value_mapper=value_mapper)
 
+    @property
+    def output_discover_patterns(self) -> List[str]:
+        return []
+
 
 class ToolOutput(ToolOutputBase):
     """
@@ -44,6 +50,7 @@ class ToolOutput(ToolOutputBase):
         self.change_format = []
         self.implicit = implicit
         self.from_work_dir = None
+        self.dataset_collector_descriptions: List[DatasetCollectionDescription] = []
 
     # Tuple emulation
 
@@ -94,6 +101,10 @@ class ToolOutput(ToolOutputBase):
         output.dataset_collector_descriptions = dataset_collector_descriptions_from_output_dict(output_dict)
         return output
 
+    @property
+    def output_discover_patterns(self) -> List[str]:
+        return _merge_dataset_collector_descriptions_patterns(self.dataset_collector_descriptions)
+
 
 class ToolExpressionOutput(ToolOutputBase):
     dict_collection_visible_keys = ('name', 'format', 'label', 'hidden', 'output_type')
@@ -246,6 +257,10 @@ class ToolOutputCollection(ToolOutputBase):
         )
         return rval
 
+    @property
+    def output_discover_patterns(self) -> List[str]:
+        return self.structure.output_discover_patterns
+
 
 class ToolOutputCollectionStructure:
 
@@ -301,6 +316,20 @@ class ToolOutputCollectionStructure:
         )
         return structure
 
+    @property
+    def output_discover_patterns(self) -> List[str]:
+        if not self.dataset_collector_descriptions:
+            return []
+        else:
+            return _merge_dataset_collector_descriptions_patterns(self.dataset_collector_descriptions)
+
+
+def _merge_dataset_collector_descriptions_patterns(dataset_collector_descriptions: List[DatasetCollectionDescription]) -> List[str]:
+    patterns = []
+    for description in dataset_collector_descriptions:
+        patterns.extend(description.discover_patterns)
+    return patterns
+
 
 class ToolOutputCollectionPart:
 
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index c2b1a1a1b40..6de749ae0ab 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1844,6 +1844,14 @@ class Tool(Dictifiable):
         """
         return self._view.get_requirements_status({self.id: self.tool_requirements}, self.installed_tool_dependencies)
 
+    @property
+    def output_discover_patterns(self):
+        # patterns to collect for remote job execution
+        patterns = []
+        for output in self.outputs.values():
+            patterns.extend(output.output_discover_patterns)
+        return patterns
+
     def build_redirect_url_params(self, param_dict):
         """
         Substitute parameter values into self.redirect_url_params
diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py
index 411daad976e..a1b7cc792a6 100644
--- a/test/integration/test_pulsar_embedded.py
+++ b/test/integration/test_pulsar_embedded.py
@@ -26,4 +26,5 @@ test_tools = integration_util.integration_tool_runner([
     "output_filter",
     "vcf_bgzip_test",
     "environment_variables",
+    "multi_output_assign_primary_ext_dbkey",
 ])

From 2931383fb695d466e9a6366cf5b074820b4c41b6 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 14 Jul 2021 20:44:35 +0200
Subject: [PATCH 107/452] Clean a bit

- Remove unused variables
- Simplify some functions
---
 .../webapps/galaxy/api/history_contents.py    | 31 ++++++-------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index b1d4b57a284..28c0614b065 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -575,7 +575,7 @@ class HistoriesContentsService(ServiceBase):
         hdas = self.__datasets_for_update(trans, history, hda_ids, payload_dict)
         rval = []
         for hda in hdas:
-            self.__deserialize_dataset(hda, payload_dict, trans)
+            self.__deserialize_dataset(trans, hda, payload_dict)
             serialization_params["default_view"] = "summary"
             rval.append(self.hda_serializer.serialize_to_view(
                 hda, user=trans.user, trans=trans, **serialization_params
@@ -681,7 +681,7 @@ class HistoriesContentsService(ServiceBase):
         )
         hda = self.__datasets_for_update(trans, history, [decoded_id], payload_dict)[0]
         if hda:
-            self.__deserialize_dataset(hda, payload_dict, trans)
+            self.__deserialize_dataset(trans, hda, payload_dict)
             serialization_params["default_view"] = 'detailed'
             return self.hda_serializer.serialize_to_view(
                 hda, user=trans.user, trans=trans, **serialization_params
@@ -713,7 +713,7 @@ class HistoriesContentsService(ServiceBase):
 
         return hdas
 
-    def __deserialize_dataset(self, hda, payload, trans):
+    def __deserialize_dataset(self, trans, hda, payload: Dict[str, Any]):
         self.hda_deserializer.deserialize(hda, payload, user=trans.user, trans=trans)
         # TODO: this should be an effect of deleting the hda
         if payload.get('deleted', False):
@@ -1059,9 +1059,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
     hda_manager: hdas.HDAManager = depends(hdas.HDAManager)
     history_manager: histories.HistoryManager = depends(histories.HistoryManager)
     history_contents_manager: history_contents.HistoryContentsManager = depends(history_contents.HistoryContentsManager)
-    folder_manager: folders.FolderManager = depends(folders.FolderManager)
     hda_serializer: hdas.HDASerializer = depends(hdas.HDASerializer)
-    hda_deserializer: hdas.HDADeserializer = depends(hdas.HDADeserializer)
     hdca_serializer: hdcas.HDCASerializer = depends(hdcas.HDCASerializer)
     history_contents_filters: history_contents.HistoryContentsFilters = depends(history_contents.HistoryContentsFilters)
 
@@ -1145,7 +1143,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :returns:   dictionary containing detailed HDA or HDCA information
         """
         serialization_params = parse_serialization_params(**kwd)
-        contents_type = self.__get_contents_type(trans, kwd)
+        contents_type = self.__get_contents_type(kwd)
         fuzzy_count = kwd.get("fuzzy_count", None)
         if fuzzy_count:
             fuzzy_count = int(fuzzy_count)
@@ -1197,15 +1195,12 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         :rtype:     dict
         :returns:   dictionary containing jobs summary object
         """
-        contents_type = self.__get_contents_type(trans, kwd)
+        contents_type = self.__get_contents_type(kwd)
         return self.service.show_jobs_summary(trans, id, contents_type)
 
-    def __get_contents_type(self, trans, kwd):
+    def __get_contents_type(self, kwd: dict) -> HistoryContentType:
         contents_type = kwd.get('type', 'dataset')
-        if contents_type not in ['dataset', 'dataset_collection']:
-            self.__handle_unknown_contents_type(trans, contents_type)
-
-        return contents_type
+        return HistoryContentType(contents_type)
 
     @expose_api_raw_anonymous
     def download_dataset_collection(self, trans, id, history_id=None, **kwd):
@@ -1417,7 +1412,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             any values that were different from the original and, therefore, updated
         """
         serialization_params = parse_serialization_params(**kwd)
-        contents_type = self.__get_contents_type(trans, kwd)
+        contents_type = self.__get_contents_type(kwd)
         return self.service.update(
             trans, history_id, id, payload, serialization_params, contents_type
         )
@@ -1473,7 +1468,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             * purged:     if the history content was purged
         """
         serialization_params = parse_serialization_params(**kwd)
-        contents_type = self.__get_contents_type(trans, kwd)
+        contents_type = self.__get_contents_type(kwd)
         purge = util.string_as_bool(purge)
         recursive = util.string_as_bool(recursive)
         if kwd.get('payload', None):
@@ -1482,14 +1477,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             recursive = util.string_as_bool(kwd['payload'].get('recursive', recursive))
         return self.service.delete(trans, id, serialization_params, contents_type, purge, recursive)
 
-    def __handle_unknown_contents_type(self, trans, contents_type):
-        raise exceptions.UnknownContentsType(f'Unknown contents type: {type}')
-
-    def encode_type_id(self, type_id):
-        TYPE_ID_SEP = '-'
-        split = type_id.split(TYPE_ID_SEP, 1)
-        return TYPE_ID_SEP.join((split[0], self.app.security.encode_id(split[1])))
-
     @expose_api_raw
     def archive(self, trans, history_id, filename='', format='zip', dry_run=True, **kwd):
         """

From 313777f0a13c9b1c27c0b727178b39b6a531b348 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 15 Jul 2021 11:04:07 +0200
Subject: [PATCH 108/452] Avoid parsing twice the payload when updating

- Parsing the pydantic model will ignore invalid properties in the payload, but according to one of the API tests, an error must be raised in this case. In addition the payload will be parsed/deserialized anyway a bit further down by the framework.
---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 28c0614b065..c56acf7feb1 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -531,8 +531,7 @@ class HistoriesContentsService(ServiceBase):
                     any values that were different from the original and, therefore, updated
         """
         if contents_type == HistoryContentType.dataset:
-            update_hda_payload = UpdateHDAPayload.parse_obj(payload)
-            return self.__update_dataset(trans, history_id, id, update_hda_payload, serialization_params)
+            return self.__update_dataset(trans, history_id, id, payload, serialization_params)
         elif contents_type == HistoryContentType.dataset_collection:
             return self.__update_dataset_collection(trans, id, payload)
         else:
@@ -571,7 +570,7 @@ class HistoriesContentsService(ServiceBase):
                 hda_ids.append(decoded_id)
             else:
                 hdca_ids.append(item.id)
-        payload_dict = payload.dict(exclude_defaults=True, exclude_unset=True)
+        payload_dict = payload.dict(exclude_unset=True)
         hdas = self.__datasets_for_update(trans, history, hda_ids, payload_dict)
         rval = []
         for hda in hdas:
@@ -674,14 +673,13 @@ class HistoriesContentsService(ServiceBase):
     ):
         # anon user: ensure that history ids match up and the history is the current,
         #   check for uploading, and use only the subset of attribute keys manipulatable by anon users
-        payload_dict = payload.dict(exclude_defaults=True, exclude_unset=True)
         decoded_id = self.decode_id(id)
         history = self.history_manager.get_owned(
             self.decode_id(history_id), trans.user, current_history=trans.history
         )
-        hda = self.__datasets_for_update(trans, history, [decoded_id], payload_dict)[0]
+        hda = self.__datasets_for_update(trans, history, [decoded_id], payload)[0]
         if hda:
-            self.__deserialize_dataset(trans, hda, payload_dict)
+            self.__deserialize_dataset(trans, hda, payload)
             serialization_params["default_view"] = 'detailed'
             return self.hda_serializer.serialize_to_view(
                 hda, user=trans.user, trans=trans, **serialization_params

From 21b7bd0f9f00ea8ee136259f5d6c86fc3fa80903 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 15 Jul 2021 13:13:44 +0200
Subject: [PATCH 109/452] Refactor `archive` action

---
 .../webapps/galaxy/api/history_contents.py    | 209 ++++++++++--------
 1 file changed, 115 insertions(+), 94 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index c56acf7feb1..95b2c1fb280 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -643,6 +643,118 @@ class HistoriesContentsService(ServiceBase):
         else:
             raise exceptions.UnknownContentsType(f'Unknown contents type: {contents_type}')
 
+    def archive(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        filter_query_params: HistoryContentsFilterQueryParams,
+        filename: str = '',
+        dry_run: bool = True,
+    ):
+        """
+        Build and return a compressed archive of the selected history contents
+
+        :type   filename:  string
+        :param  filename:  (optional) archive name (defaults to history name)
+        :type   dry_run:   boolean
+        :param  dry_run:   (optional) if True, return the archive and file paths only
+                           as json and not an archive file
+
+        :returns:   archive file for download or json in `dry run` mode
+        """
+        # roughly from: http://stackoverflow.com/a/31976060 (windows, linux)
+        invalid_filename_char_regex = re.compile(r'[:<>|\\\/\?\* "]')
+        # path format string - dot separator between id and name
+        id_name_format = '{}.{}'
+
+        def name_to_filename(name, max_length=150, replace_with='_'):
+            # TODO: seems like shortening unicode with [:] would cause unpredictable display strings
+            return invalid_filename_char_regex.sub(replace_with, name)[0:max_length]
+
+        # given a set of parents for a dataset (HDCAs, DC, DCEs, etc.) - build a directory structure that
+        # (roughly) recreates the nesting in the contents using the parent names and ids
+        def build_path_from_parents(parents):
+            parent_names = []
+            for parent in parents:
+                # an HDCA
+                if hasattr(parent, 'hid'):
+                    name = name_to_filename(parent.name)
+                    parent_names.append(id_name_format.format(parent.hid, name))
+                # a DCE
+                elif hasattr(parent, 'element_index'):
+                    name = name_to_filename(parent.element_identifier)
+                    parent_names.append(id_name_format.format(parent.element_index, name))
+            # NOTE: DCs are skipped and use the wrapping DCE info instead
+            return parent_names
+
+        # get the history used for the contents query and check for accessibility
+        history = self.history_manager.get_accessible(trans.security.decode_id(history_id), trans.user)
+        archive_base_name = filename or name_to_filename(history.name)
+
+        # this is the fn applied to each dataset contained in the query
+        paths_and_files = []
+
+        def build_archive_files_and_paths(content, *parents):
+            archive_path = archive_base_name
+            if not self.hda_manager.is_accessible(content, trans.user):
+                # if the underlying dataset is not accessible, skip it silently
+                return
+
+            content_container_id = content.hid
+            content_name = name_to_filename(content.name)
+            if parents:
+                if hasattr(parents[0], 'element_index'):
+                    # if content is directly wrapped in a DCE, strip it from parents (and the resulting path)
+                    # and instead replace the content id and name with the DCE index and identifier
+                    parent_dce, parents = parents[0], parents[1:]
+                    content_container_id = parent_dce.element_index
+                    content_name = name_to_filename(parent_dce.element_identifier)
+                # reverse for path from parents: oldest parent first
+                archive_path = os.path.join(archive_path, *build_path_from_parents(parents)[::-1])
+                # TODO: this is brute force - building the path each time instead of re-using it
+                # possibly cache
+
+            # add the name as the last element in the archive path
+            content_id_and_name = id_name_format.format(content_container_id, content_name)
+            archive_path = os.path.join(archive_path, content_id_and_name)
+
+            # ---- for composite files, we use id and name for a directory and, inside that, ...
+            if self.hda_manager.is_composite(content):
+                # ...save the 'main' composite file (gen. html)
+                paths_and_files.append((content.file_name, os.path.join(archive_path, f"{content.name}.html")))
+                for extra_file in self.hda_manager.extra_files(content):
+                    extra_file_basename = os.path.basename(extra_file)
+                    archive_extra_file_path = os.path.join(archive_path, extra_file_basename)
+                    # ...and one for each file in the composite
+                    paths_and_files.append((extra_file, archive_extra_file_path))
+
+            # ---- for single files, we add the true extension to id and name and store that single filename
+            else:
+                # some dataset names can contain their original file extensions, don't repeat
+                if not archive_path.endswith(f".{content.extension}"):
+                    archive_path += f".{content.extension}"
+                paths_and_files.append((content.file_name, archive_path))
+
+        # filter the contents that contain datasets using any filters possible from index above and map the datasets
+        filters = self.history_contents_filters.parse_query_filters(filter_query_params)
+        self.history_contents_manager.map_datasets(history, build_archive_files_and_paths, filters=filters)
+
+        # if dry_run, return the structure as json for debugging
+        if dry_run:
+            trans.response.headers['Content-Type'] = 'application/json'
+            return safe_dumps(paths_and_files)
+
+        # create the archive, add the dataset files, then stream the archive as a download
+        archive = ZipstreamWrapper(
+            archive_name=archive_base_name,
+            upstream_mod_zip=trans.app.config.upstream_mod_zip,
+            upstream_gzip=trans.app.config.upstream_gzip,
+        )
+        for file_path, archive_path in paths_and_files:
+            archive.write(file_path, archive_path)
+
+        trans.response.headers.update(archive.get_headers())
+        return archive.response()
+
     def __delete_dataset(
         self, trans,
         id: EncodedDatabaseIdField,
@@ -1493,100 +1605,9 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
 
         .. note:: this is a volatile endpoint and settings and behavior may change.
         """
-        # roughly from: http://stackoverflow.com/a/31976060 (windows, linux)
-        invalid_filename_char_regex = re.compile(r'[:<>|\\\/\?\* "]')
-        # path format string - dot separator between id and name
-        id_name_format = '{}.{}'
-
-        def name_to_filename(name, max_length=150, replace_with='_'):
-            # TODO: seems like shortening unicode with [:] would cause unpredictable display strings
-            return invalid_filename_char_regex.sub(replace_with, name)[0:max_length]
-
-        # given a set of parents for a dataset (HDCAs, DC, DCEs, etc.) - build a directory structure that
-        # (roughly) recreates the nesting in the contents using the parent names and ids
-        def build_path_from_parents(parents):
-            parent_names = []
-            for parent in parents:
-                # an HDCA
-                if hasattr(parent, 'hid'):
-                    name = name_to_filename(parent.name)
-                    parent_names.append(id_name_format.format(parent.hid, name))
-                # a DCE
-                elif hasattr(parent, 'element_index'):
-                    name = name_to_filename(parent.element_identifier)
-                    parent_names.append(id_name_format.format(parent.element_index, name))
-            # NOTE: DCs are skipped and use the wrapping DCE info instead
-            return parent_names
-
-        # get the history used for the contents query and check for accessibility
-        history = self.history_manager.get_accessible(trans.security.decode_id(history_id), trans.user)
-        archive_base_name = filename or name_to_filename(history.name)
-
-        # this is the fn applied to each dataset contained in the query
-        paths_and_files = []
-
-        def build_archive_files_and_paths(content, *parents):
-            archive_path = archive_base_name
-            if not self.hda_manager.is_accessible(content, trans.user):
-                # if the underlying dataset is not accessible, skip it silently
-                return
-
-            content_container_id = content.hid
-            content_name = name_to_filename(content.name)
-            if parents:
-                if hasattr(parents[0], 'element_index'):
-                    # if content is directly wrapped in a DCE, strip it from parents (and the resulting path)
-                    # and instead replace the content id and name with the DCE index and identifier
-                    parent_dce, parents = parents[0], parents[1:]
-                    content_container_id = parent_dce.element_index
-                    content_name = name_to_filename(parent_dce.element_identifier)
-                # reverse for path from parents: oldest parent first
-                archive_path = os.path.join(archive_path, *build_path_from_parents(parents)[::-1])
-                # TODO: this is brute force - building the path each time instead of re-using it
-                # possibly cache
-
-            # add the name as the last element in the archive path
-            content_id_and_name = id_name_format.format(content_container_id, content_name)
-            archive_path = os.path.join(archive_path, content_id_and_name)
-
-            # ---- for composite files, we use id and name for a directory and, inside that, ...
-            if self.hda_manager.is_composite(content):
-                # ...save the 'main' composite file (gen. html)
-                paths_and_files.append((content.file_name, os.path.join(archive_path, f"{content.name}.html")))
-                for extra_file in self.hda_manager.extra_files(content):
-                    extra_file_basename = os.path.basename(extra_file)
-                    archive_extra_file_path = os.path.join(archive_path, extra_file_basename)
-                    # ...and one for each file in the composite
-                    paths_and_files.append((extra_file, archive_extra_file_path))
-
-            # ---- for single files, we add the true extension to id and name and store that single filename
-            else:
-                # some dataset names can contain their original file extensions, don't repeat
-                if not archive_path.endswith(f".{content.extension}"):
-                    archive_path += f".{content.extension}"
-                paths_and_files.append((content.file_name, archive_path))
-
-        # filter the contents that contain datasets using any filters possible from index above and map the datasets
-        filter_params = self.parse_filter_params(kwd)
-        filters = self.history_contents_filters.parse_filters(filter_params)
-        self.history_contents_manager.map_datasets(history, build_archive_files_and_paths, filters=filters)
-
-        # if dry_run, return the structure as json for debugging
-        if dry_run == 'True':
-            trans.response.headers['Content-Type'] = 'application/json'
-            return safe_dumps(paths_and_files)
-
-        # create the archive, add the dataset files, then stream the archive as a download
-        archive = ZipstreamWrapper(
-            archive_name=archive_base_name,
-            upstream_mod_zip=self.app.config.upstream_mod_zip,
-            upstream_gzip=self.app.config.upstream_gzip,
-        )
-        for file_path, archive_path in paths_and_files:
-            archive.write(file_path, archive_path)
-
-        trans.response.headers.update(archive.get_headers())
-        return archive.response()
+        dry_run = util.string_as_bool(dry_run)
+        filter_parameters = HistoryContentsFilterQueryParams(**kwd)
+        return self.service.archive(trans, history_id, filter_parameters, filename, dry_run)
 
     @expose_api_raw_anonymous
     def contents_near(self, trans, history_id, hid, limit, **kwd):

From aabac67c01a2f55d2b259acb72aa0e7fbd10e6a7 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 15 Jul 2021 13:29:08 +0200
Subject: [PATCH 110/452] Fix linting

---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 95b2c1fb280..9e9ab8ee5ad 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -72,7 +72,6 @@ from galaxy.schema.schema import (
     JobStateSummary,
     Model,
     UpdateDatasetPermissionsPayload,
-    UpdateHDAPayload,
     UpdateHistoryContentsBatchPayload,
     WorkflowInvocationStateSummary,
 )
@@ -780,7 +779,7 @@ class HistoriesContentsService(ServiceBase):
         self, trans,
         history_id: EncodedDatabaseIdField,
         id: EncodedDatabaseIdField,
-        payload: UpdateHDAPayload,
+        payload: Dict[str, Any],
         serialization_params: SerializationParams
     ):
         # anon user: ensure that history ids match up and the history is the current,

From ef77b6366b897609ebc6525392d1db769a4549ec Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 15 Jul 2021 17:26:04 +0200
Subject: [PATCH 111/452] Refactor `contents_near` action

---
 .../webapps/galaxy/api/history_contents.py    | 261 ++++++++++--------
 1 file changed, 152 insertions(+), 109 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 9e9ab8ee5ad..eb6ded8120d 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -110,6 +110,10 @@ class HistoryContentsFilterQueryParams(FilterQueryParams):
     order: Optional[str] = OrderParamField(default_order="hid-asc")
 
 
+HistoryContentFilter = List[Any]  # Lists with [attribute:str, operator:str, value:Any]
+HistoryContentsFilterList = List[HistoryContentFilter]
+
+
 class HistoryContentsIndexLegacyParamsBase(Model):
     deleted: Optional[bool] = Field(
         default=None,
@@ -754,6 +758,145 @@ class HistoriesContentsService(ServiceBase):
         trans.response.headers.update(archive.get_headers())
         return archive.response()
 
+    def contents_near(
+        self, trans,
+        history_id: EncodedDatabaseIdField,
+        serialization_params: SerializationParams,
+        filter_params: HistoryContentsFilterList,
+        hid: int,
+        limit: int,
+        since: Optional[datetime.datetime] = None,
+    ):
+        """
+        This endpoint provides random access to a large history without having
+        to know exactly how many pages are in the final query. Pick a target HID
+        and filters, and the endpoint will get LIMIT counts above and below that
+        target regardless of how many gaps may exist in the HID due to
+        filtering.
+
+        It does 2 queries, one up and one down from the target hid with a
+        result size of limit. Additional counts for total matches of both seeks
+        provided in the http headers.
+
+        I've also abandoned the wierd q/qv syntax.
+        """
+        history: History = self.history_manager.get_accessible(
+            self.decode_id(history_id), trans.user, current_history=trans.history
+        )
+
+        # while polling, check to see if the history has changed
+        # if it hasn't then we can short-circuit the poll request
+        if since:
+            # sqlalchemy DateTime columns are not timezone aware, so parse `since` into timezone-aware
+            # datetime and then convert to naive datetime object representing UTC,
+            # assuming history.update_time represents UTC time.
+            since_date = since.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+            if history.update_time <= since_date:
+                trans.response.status = 204
+                return
+
+        # parse content params
+        view = serialization_params.pop('view')
+
+        # SEEK UP, contents > hid
+        up_params = filter_params + self._hid_greater_than(hid)
+        up_order = 'hid-asc'
+        contents_up, up_count = self._seek(history, up_params, up_order, limit, serialization_params)
+
+        # SEEK DOWN, contents <= hid
+        down_params = filter_params + self._hid_less_or_equal_than(hid)
+        down_order = 'hid-dsc'
+        contents_down, down_count = self._seek(history, down_params, down_order, limit, serialization_params)
+
+        # min/max hid values
+        min_hid, max_hid = self._get_filtered_extrema(history, filter_params)
+
+        # results
+        up = self._expand_contents(trans, contents_up, serialization_params, view)
+        up.reverse()
+        down = self._expand_contents(trans, contents_down, serialization_params, view)
+        contents = up + down
+
+        # Put stats in http headers
+        trans.response.headers['matches_up'] = len(contents_up)
+        trans.response.headers['matches_down'] = len(contents_down)
+        trans.response.headers['total_matches_up'] = up_count
+        trans.response.headers['total_matches_down'] = down_count
+        trans.response.headers['max_hid'] = max_hid
+        trans.response.headers['min_hid'] = min_hid
+        trans.response.headers['history_size'] = str(history.disk_size)
+        trans.response.headers['history_empty'] = json.dumps(history.empty)  # convert to proper bool
+
+        return json.dumps(contents)
+
+    def _hid_greater_than(self, hid: int) -> HistoryContentsFilterList:
+        return [["hid", "gt", hid]]
+
+    def _hid_less_or_equal_than(self, hid: int) -> HistoryContentsFilterList:
+        return [["hid", "le", hid]]
+
+    # Perform content query and matching count
+    def _seek(self, history, filter_params, order_by_string, limit, serialization_params):
+        filters = self.history_contents_filters.parse_filters(filter_params)
+        order_by = self.build_order_by(self.history_contents_manager, order_by_string)
+
+        # actual contents
+        contents = self.history_contents_manager.contents(history,
+            filters=filters,
+            limit=limit,
+            offset=0,
+            order_by=order_by,
+            serialization_params=serialization_params)
+
+        # count of same query
+        count_filter_params = [f for f in filter_params if f[0] != 'update_time']
+        count_filters = self.history_contents_filters.parse_filters(count_filter_params)
+        contents_count = self.history_contents_manager.contents_count(history, count_filters)
+
+        return contents, contents_count
+
+    # Adds subquery details to initial contents results, perhaps better realized
+    # as a proc or view.
+    def _expand_contents(self, trans, contents, serialization_params, view):
+        rval = []
+        for content in contents:
+            if isinstance(content, HistoryDatasetAssociation):
+                dataset = self.hda_serializer.serialize_to_view(content,
+                    user=trans.user, trans=trans, view=view, **serialization_params)
+                rval.append(dataset)
+            elif isinstance(content, HistoryDatasetCollectionAssociation):
+                collection = self.hdca_serializer.serialize_to_view(content,
+                    user=trans.user, trans=trans, view=view, **serialization_params)
+                rval.append(collection)
+        return rval
+
+    def _get_filtered_extrema(self, history, filter_params):
+        extrema_params = parse_serialization_params(keys='hid', default_view='summary')
+        extrema_filter_params = [f for f in filter_params if f[0] != 'update_time']
+        extrema_filters = self.history_contents_filters.parse_filters(extrema_filter_params)
+
+        order_by_dsc = self.build_order_by(self.history_contents_manager, 'hid-dsc')
+        order_by_asc = self.build_order_by(self.history_contents_manager, 'hid-asc')
+
+        max_row_result = self.history_contents_manager.contents(history,
+            limit=1,
+            filters=extrema_filters,
+            order_by=order_by_dsc,
+            serialization_params=extrema_params)
+        max_row = max_row_result.pop() if len(max_row_result) else None
+
+        min_row_result = self.history_contents_manager.contents(history,
+            limit=1,
+            filters=extrema_filters,
+            order_by=order_by_asc,
+            serialization_params=extrema_params)
+        min_row = min_row_result.pop() if len(min_row_result) else None
+
+        max_hid = max_row.hid if max_row else None
+        min_hid = min_row.hid if min_row else None
+
+        return min_hid, max_hid
+
     def __delete_dataset(
         self, trans,
         id: EncodedDatabaseIdField,
@@ -1625,93 +1768,20 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
 
         * GET /api/histories/{history_id}/contents/near/{hid}/{limit}
         """
-        history = self.history_manager.get_accessible(self.decode_id(history_id), trans.user, current_history=trans.history)
-
-        # while polling, check to see if the history has changed
-        # if it hasn't then we can short-circuit the poll request
+        serialization_params = parse_serialization_params(default_view='betawebclient', **kwd)
+        filter_params = self._parse_rest_params(kwd)
+        hid = int(hid)
+        limit = int(limit)
         since = kwd.get('update_time-gt', None)
         if since:
-            # sqlalchemy DateTime columns are not timezone aware, so parse `since` into timezone-aware
-            # datetime and then convert to naive datetime object representing UTC,
-            # assuming history.update_time represents UTC time.
-            since_date = dateutil.parser.isoparse(since).astimezone(datetime.timezone.utc).replace(tzinfo=None)
-            if history.update_time <= since_date:
-                trans.response.status = 204
-                return
+            since = dateutil.parser.isoparse(since)
 
-        # parse content params
-        filter_params = self._parse_rest_params(kwd)
-        serialization_params = self._parse_serialization_params(kwd, 'betawebclient')
-        view = serialization_params.pop('view')
-
-        # SEEK UP, contents > hid
-        up_params = filter_params + self._parse_rest_params({'hid-gt': hid})
-        up_order = 'hid-asc'
-        contents_up, up_count = self._seek(history, up_params, up_order, limit, serialization_params)
-
-        # SEEK DOWN, contents <= hid
-        down_params = filter_params + self._parse_rest_params({'hid-le': hid})
-        down_order = 'hid-dsc'
-        contents_down, down_count = self._seek(history, down_params, down_order, limit, serialization_params)
-
-        # min/max hid values
-        min_hid, max_hid = self._get_filtered_extrema(history, filter_params)
-
-        # results
-        up = self._expand_contents(trans, contents_up, serialization_params, view)
-        up.reverse()
-        down = self._expand_contents(trans, contents_down, serialization_params, view)
-        contents = up + down
-
-        # Put stats in http headers
-        trans.response.headers['matches_up'] = len(contents_up)
-        trans.response.headers['matches_down'] = len(contents_down)
-        trans.response.headers['total_matches_up'] = up_count
-        trans.response.headers['total_matches_down'] = down_count
-        trans.response.headers['max_hid'] = max_hid
-        trans.response.headers['min_hid'] = min_hid
-        trans.response.headers['history_size'] = str(history.disk_size)
-        trans.response.headers['history_empty'] = json.dumps(history.empty)  # convert to proper bool
-
-        return json.dumps(contents)
-
-    # Perform content query and matching count
-    def _seek(self, history, filter_params, order_by_string, limit, serialization_params):
-        filters = self.history_contents_filters.parse_filters(filter_params)
-        order_by = self._parse_order_by(manager=self.history_contents_manager, order_by_string=order_by_string)
-
-        # actual contents
-        contents = self.history_contents_manager.contents(history,
-            filters=filters,
-            limit=limit,
-            offset=0,
-            order_by=order_by,
-            serialization_params=serialization_params)
-
-        # count of same query
-        count_filter_params = [f for f in filter_params if f[0] != 'update_time']
-        count_filters = self.history_contents_filters.parse_filters(count_filter_params)
-        contents_count = self.history_contents_manager.contents_count(history, count_filters)
-
-        return contents, contents_count
-
-    # Adds subquery details to initial contents results, perhaps better realized
-    # as a proc or view.
-    def _expand_contents(self, trans, contents, serialization_params, view):
-        rval = []
-        for content in contents:
-            if isinstance(content, trans.app.model.HistoryDatasetAssociation):
-                dataset = self.hda_serializer.serialize_to_view(content,
-                    user=trans.user, trans=trans, view=view, **serialization_params)
-                rval.append(dataset)
-            elif isinstance(content, trans.app.model.HistoryDatasetCollectionAssociation):
-                collection = self.hdca_serializer.serialize_to_view(content,
-                    user=trans.user, trans=trans, view=view, **serialization_params)
-                rval.append(collection)
-        return rval
+        return self.service.contents_near(
+            trans, history_id, serialization_params, filter_params, hid, limit, since,
+        )
 
     # Parsing query string according to REST standards.
-    def _parse_rest_params(self, qdict):
+    def _parse_rest_params(self, qdict: Dict[str, Any]) -> HistoryContentsFilterList:
         DEFAULT_OP = 'eq'
         splitchar = '-'
 
@@ -1724,30 +1794,3 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
             result.append([attr, op, val])
 
         return result
-
-    def _get_filtered_extrema(self, history, filter_params):
-        extrema_params = self._parse_serialization_params(dict(keys='hid'), 'summary')
-        extrema_filter_params = [f for f in filter_params if f[0] != 'update_time']
-        extrema_filters = self.history_contents_filters.parse_filters(extrema_filter_params)
-
-        order_by_dsc = self._parse_order_by(manager=self.history_contents_manager, order_by_string='hid-dsc')
-        order_by_asc = self._parse_order_by(manager=self.history_contents_manager, order_by_string='hid-asc')
-
-        max_row_result = self.history_contents_manager.contents(history,
-            limit=1,
-            filters=extrema_filters,
-            order_by=order_by_dsc,
-            serialization_params=extrema_params)
-        max_row = max_row_result.pop() if len(max_row_result) else None
-
-        min_row_result = self.history_contents_manager.contents(history,
-            limit=1,
-            filters=extrema_filters,
-            order_by=order_by_asc,
-            serialization_params=extrema_params)
-        min_row = min_row_result.pop() if len(min_row_result) else None
-
-        max_hid = max_row.hid if max_row else None
-        min_hid = min_row.hid if min_row else None
-
-        return min_hid, max_hid

From 5d0ac60726a4af15495d1cc98fab26306055d111 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 17 Dec 2020 19:15:13 -0500
Subject: [PATCH 112/452] Selenium abstractions.

---
 lib/galaxy/selenium/navigates_galaxy.py         | 17 +++++++++++------
 .../selenium/test_history_structure.py          |  3 +--
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index ca83b95a1a4..122c1f7bcbb 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -1199,15 +1199,20 @@ class NavigatesGalaxy(HasDriver):
     def click_history_options(self):
         self.components.history_panel.options_button_icon.wait_for_and_click()
 
-    def click_history_option(self, option_label):
+    def click_history_option(self, option_label_or_component):
         # Open menu
         self.click_history_options()
 
-        # Click labelled option
-        self.wait_for_visible(self.navigation.history_panel.options_menu)
-        menu_item_sizzle_selector = f"#history-options-button-menu > a:contains('{option_label}')"
-        menu_selection_element = self.wait_for_sizzle_selector_clickable(menu_item_sizzle_selector)
-        menu_selection_element.click()
+        if isinstance(option_label_or_component, str):
+            option_label = option_label_or_component
+            # Click labelled option
+            self.wait_for_visible(self.navigation.history_panel.options_menu)
+            menu_item_sizzle_selector = f"#history-options-button-menu > a:contains('{option_label}')"
+            menu_selection_element = self.wait_for_sizzle_selector_clickable(menu_item_sizzle_selector)
+            menu_selection_element.click()
+        else:
+            option_component = option_label_or_component
+            option_component.wait_for_and_click()
 
     def history_click_create_new(self):
         self.components.history_panel.new_history_button.wait_for_and_click()
diff --git a/lib/galaxy_test/selenium/test_history_structure.py b/lib/galaxy_test/selenium/test_history_structure.py
index 7450a86a817..288345acb85 100644
--- a/lib/galaxy_test/selenium/test_history_structure.py
+++ b/lib/galaxy_test/selenium/test_history_structure.py
@@ -24,9 +24,8 @@ class HistoryStructureTestCase(SeleniumTestCase):
         self.wait_for_history()
         self.components.history_structure.header.assert_absent_or_hidden()
         self.components.history_structure.dataset.assert_absent_or_hidden()
-        self.click_history_options()
 
-        self.components.history_panel.options_show_history_structure.wait_for_and_click()
+        self.click_history_option(self.components.history_panel.options_show_history_structure)
 
         # assert details are not visible before expand
         self.components.history_structure.details.assert_absent_or_hidden()

From bab5ab3502dd3c9c28883c742f9444f2ff180921 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 7 Apr 2021 15:22:38 -0400
Subject: [PATCH 113/452] Run test_history_panel.py against beta history panel.

Fix mouseover for renaming (wasn't working at all) and sync class decorators, etc.. across history implementations.
---
 .../src/components/History/HistoriesMenu.vue  |  1 +
 .../src/components/History/HistoryDetails.vue | 10 ++-
 .../src/components/History/HistoryEmpty.vue   |  2 +-
 client/src/components/PriorityMenu/Menu.vue   |  2 +-
 client/src/entry/panels/history-panel.js      |  2 +-
 client/src/mvc/history/history-view-edit.js   |  2 +-
 client/src/mvc/history/history-view.js        |  6 +-
 lib/galaxy/selenium/has_driver.py             |  3 +
 lib/galaxy/selenium/navigates_galaxy.py       | 80 ++++++++++++++-----
 lib/galaxy/selenium/navigation.yml            | 24 +++---
 lib/galaxy/selenium/smart_components.py       |  4 +
 lib/galaxy_test/selenium/framework.py         |  5 ++
 .../selenium/test_history_panel.py            | 47 ++++++++---
 13 files changed, 137 insertions(+), 51 deletions(-)

diff --git a/client/src/components/History/HistoriesMenu.vue b/client/src/components/History/HistoriesMenu.vue
index c06b506ce4b..7a23c6a120d 100644
--- a/client/src/components/History/HistoriesMenu.vue
+++ b/client/src/components/History/HistoriesMenu.vue
@@ -9,6 +9,7 @@
         <PriorityMenuItem
             key="create-new-history"
             title="Create New History"
+            class="history-new-button"
             icon="fa fa-plus"
             @click="$emit('createNewHistory')"
         />
diff --git a/client/src/components/History/HistoryDetails.vue b/client/src/components/History/HistoryDetails.vue
index 583c4226c04..a271f0e6006 100644
--- a/client/src/components/History/HistoryDetails.vue
+++ b/client/src/components/History/HistoryDetails.vue
@@ -1,14 +1,15 @@
 <template>
     <header>
-        <div class="d-flex justify-content-between align-items-center">
+        <div class="d-flex justify-content-between align-items-center actions">
             <h6 class="text-nowrap text-truncate mr-auto">
-                <span class="size">{{ history.niceSize | localize }}</span>
+                <span class="size history-size">{{ history.niceSize | localize }}</span>
             </h6>
 
             <PriorityMenu style="max-width: 50%" :starting-height="28">
                 <PriorityMenuItem
                     key="edit-history-tags"
                     title="Edit History Tags"
+                    class="history-tag-btn"
                     icon="fas fa-tags"
                     :pressed="showTags"
                     @click="showTags = !showTags"
@@ -16,6 +17,7 @@
                 <PriorityMenuItem
                     key="edit-history-annotation"
                     title="Edit Annotation"
+                    class="history-annotate-btn"
                     icon="fas fa-comment"
                     :pressed="showAnnotation"
                     @click="showAnnotation = !showAnnotation"
@@ -94,8 +96,8 @@
             class="history-title mt-2"
             tag-name="h3"
             v-model="historyName"
-            :tooltip-title="'Rename history...' | localize"
-            tooltip-placement="top"
+            :title="'Rename history...' | localize"
+            v-b-tooltip.hover="{ boundary: 'viewport', placement: 'left' }"
         />
 
         <transition name="shutterfade">
diff --git a/client/src/components/History/HistoryEmpty.vue b/client/src/components/History/HistoryEmpty.vue
index 1b1d4f5b27c..5bd60d17149 100644
--- a/client/src/components/History/HistoryEmpty.vue
+++ b/client/src/components/History/HistoryEmpty.vue
@@ -1,7 +1,7 @@
 <template>
     <b-alert show>
         <h4 class="mb-1">
-            <i class="fa fa-info-circle"></i>
+            <i class="fa fa-info-circle empty-message"></i>
             <span>{{ message | l }}</span>
         </h4>
         <p>
diff --git a/client/src/components/PriorityMenu/Menu.vue b/client/src/components/PriorityMenu/Menu.vue
index d9b591ef409..df5aad484a2 100644
--- a/client/src/components/PriorityMenu/Menu.vue
+++ b/client/src/components/PriorityMenu/Menu.vue
@@ -14,7 +14,7 @@
             <div class="overflow" v-if="overflow.length">
                 <b-dropdown no-caret right variant="link" size="sm" boundary="window" toggle-class="p-1">
                     <template v-slot:button-content>
-                        <i class="fas fa-ellipsis-v" />
+                        <i class="fas fa-ellipsis-v priority-menu-expand" />
                         <span class="sr-only">...</span>
                     </template>
                     <b-dropdown-item v-for="item in overflow" :key="item.key" v-bind="item.attrs" v-on="item.on">
diff --git a/client/src/entry/panels/history-panel.js b/client/src/entry/panels/history-panel.js
index 47455261840..665be032280 100644
--- a/client/src/entry/panels/history-panel.js
+++ b/client/src/entry/panels/history-panel.js
@@ -42,7 +42,7 @@ const HistoryPanel = Backbone.View.extend({
             this.buttonNew = new Ui.ButtonLink({
                 id: "history-new-button",
                 title: _l("Create new history"),
-                cls: "panel-header-button",
+                cls: "panel-header-button history-new-button",
                 icon: "fa fa-plus",
                 onclick: function () {
                     Galaxy.currHistoryPanel.createNewHistory();
diff --git a/client/src/mvc/history/history-view-edit.js b/client/src/mvc/history/history-view-edit.js
index 2c7111deba8..3242113c61d 100644
--- a/client/src/mvc/history/history-view-edit.js
+++ b/client/src/mvc/history/history-view-edit.js
@@ -227,7 +227,7 @@ var HistoryViewEdit = _super.extend(
             var nameSelector = "> .controls .name";
             $where
                 .find(nameSelector)
-                .attr("title", _l("Click to rename history"))
+                .attr("title", _l("Rename history..."))
                 .tooltip({ placement: "bottom" })
                 .make_text_editable({
                     on_finish: function (newName) {
diff --git a/client/src/mvc/history/history-view.js b/client/src/mvc/history/history-view.js
index 20b72c14fce..ef48b22e010 100644
--- a/client/src/mvc/history/history-view.js
+++ b/client/src/mvc/history/history-view.js
@@ -520,14 +520,14 @@ var HistoryView = _super.extend(
 HistoryView.prototype.templates = (() => {
     var mainTemplate = () =>
         `<div>
-            <div class="controls"></div>
+            <div class="controls history-details"></div>
             <ul class="list-items"></ul>
             <div class="empty-message infomessagesmall"></div>',
         </div>`;
 
     var controlsTemplate = BASE_MVC.wrapTemplate(
         [
-            '<div class="controls">',
+            '<div class="controls history-details">',
             '<div class="title">',
             '<div class="name"><%- history.name %></div>',
             "</div>",
@@ -561,7 +561,7 @@ HistoryView.prototype.templates = (() => {
 
             // add tags and annotations
             '<div class="tags-display"></div>',
-            '<div class="annotation-display"></div>',
+            '<div class="annotation-display history-annotation"></div>',
 
             '<div class="search">',
             '<div class="search-input"></div>',
diff --git a/lib/galaxy/selenium/has_driver.py b/lib/galaxy/selenium/has_driver.py
index 5466a542770..df71cb39b28 100644
--- a/lib/galaxy/selenium/has_driver.py
+++ b/lib/galaxy/selenium/has_driver.py
@@ -50,6 +50,9 @@ class HasDriver:
     def assert_absent(self, selector_template):
         assert len(self.find_elements(selector_template)) == 0
 
+    def element_absent(self, selector_template):
+        return len(self.find_elements(selector_template)) == 0
+
     def wait_for_xpath(self, xpath, **kwds):
         element = self._wait_on(
             ec.presence_of_element_located((By.XPATH, xpath)),
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 122c1f7bcbb..25a6dd03c24 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -162,7 +162,8 @@ class NavigatesGalaxy(HasDriver):
     def home(self):
         self.get()
         self.wait_for_visible(self.navigation.masthead.selector)
-        self.wait_for_visible(self.navigation.history_panel.selector)
+        if not self.is_beta_history():
+            self.wait_for_visible(self.navigation.history_panel.selector)
 
     def trs_search(self):
         self.driver.get(self.build_url('workflows/trs_search'))
@@ -225,7 +226,11 @@ class NavigatesGalaxy(HasDriver):
         }
 
     def history_panel_name_element(self):
-        return self.wait_for_present(self.navigation.history_panel.selectors.name)
+        if self.is_beta_history():
+            component = self.components.history_panel.name_beta
+        else:
+            component = self.components.history_panel.name
+        return component.wait_for_present()
 
     @retry_during_transitions
     def history_panel_name(self):
@@ -1197,7 +1202,11 @@ class NavigatesGalaxy(HasDriver):
 
     @retry_during_transitions
     def click_history_options(self):
-        self.components.history_panel.options_button_icon.wait_for_and_click()
+        if self.is_beta_history():
+            component = self.components.history_panel.options_button_icon_beta
+        else:
+            component = self.components.history_panel.options_button_icon
+        component.wait_for_and_click()
 
     def click_history_option(self, option_label_or_component):
         # Open menu
@@ -1214,6 +1223,14 @@ class NavigatesGalaxy(HasDriver):
             option_component = option_label_or_component
             option_component.wait_for_and_click()
 
+    def use_beta_history(self):
+        if not self.is_beta_history():
+            self.click_history_option(self.components.history_panel.options_use_beta_history)
+            self.components.history_panel.beta.wait_for_present()
+
+    def is_beta_history(self):
+        return not self.components.history_panel.beta.is_absent
+
     def history_click_create_new(self):
         self.components.history_panel.new_history_button.wait_for_and_click()
 
@@ -1238,14 +1255,14 @@ class NavigatesGalaxy(HasDriver):
         return names
 
     def history_panel_add_tags(self, tags):
-        tag_icon_selector = self.components.history_panel.tag_icon
-        tag_area_selector = self.components.history_panel.tag_area
-        tag_area_input_selector = self.components.history_panel.tag_area_input
+        tag_icon = self.components.history_panel.tag_icon
+        tag_area = self.components.history_panel.tag_area
+        tag_area_input = self.components.history_panel.tag_area_input
 
-        if not tag_area_selector.is_displayed:
-            tag_icon_selector.wait_for_and_click()
+        if tag_area.is_absent or not tag_area.is_displayed:
+            tag_icon.wait_for_and_click()
 
-        tag_area = tag_area_input_selector.wait_for_and_click()
+        tag_area = tag_area_input.wait_for_and_click()
 
         for tag in tags:
             tag_area.send_keys(tag)
@@ -1254,12 +1271,22 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_rename(self, new_name):
         editable_text_input_element = self.history_panel_click_to_rename()
+        if self.is_beta_history():
+            editable_text_input_element.clear()
         editable_text_input_element.send_keys(new_name)
         self.send_enter(editable_text_input_element)
 
     def history_panel_click_to_rename(self):
-        self.wait_for_and_click(self.navigation.history_panel.selectors.name)
-        return self.wait_for_visible(self.navigation.history_panel.selectors.name_edit_input)
+        history_panel = self.components.history_panel
+        if self.is_beta_history():
+            name = history_panel.name_beta
+            edit = history_panel.name_edit_input_beta
+        else:
+            name = history_panel.name
+            edit = history_panel.name_edit_input
+
+        name.wait_for_and_click()
+        return edit.wait_for_visible()
 
     def history_panel_refresh_click(self):
         self.wait_for_and_click(self.navigation.history_panel.selectors.refresh_button)
@@ -1561,24 +1588,33 @@ class NavigatesGalaxy(HasDriver):
 
     def set_history_annotation(self, annotation, clear_text=False):
         self.ensure_history_annotation_area_displayed()
+        history_panel = self.components.history_panel
 
-        self.wait_for_and_click(self.navigation.history_panel.selectors.annotation_editable_text)
-
-        annon_area_editable = self.wait_for_and_click(self.navigation.history_panel.selectors.annotation_edit)
-        anno_done_button = self.wait_for_clickable(self.navigation.history_panel.selectors.annotation_done)
+        if self.is_beta_history():
+            editable = history_panel.annotation_editable_text_beta
+            edit = history_panel.annotation_edit_beta
+        else:
+            editable = history_panel.annotation_editable_text
+            edit = history_panel.annotation_edit
 
+        editable.wait_for_and_click()
+        edit_el = edit.wait_for_and_click()
         if clear_text:
-            annon_area_editable.clear()
+            edit_el.clear()
+        edit_el.send_keys(annotation)
 
-        annon_area_editable.send_keys(annotation)
-        anno_done_button.click()
+        if self.is_beta_history():
+            # no button, just click away
+            self.click_center()
+        else:
+            history_panel.annotation_done.wait_for_and_click()
 
     def ensure_history_annotation_area_displayed(self):
-        annotation_area_selector = self.navigation.history_panel.selectors.annotation_area
-        annotation_icon_selector = self.navigation.history_panel.selectors.annotation_icon
+        annotation_area = self.components.history_panel.annotation_area
+        annotation_icon = self.components.history_panel.annotation_icon
 
-        if not self.is_displayed(annotation_area_selector):
-            self.wait_for_and_click(annotation_icon_selector)
+        if annotation_area.is_absent or not annotation_area.is_displayed:
+            annotation_icon.wait_for_and_click()
 
     def select2_set_value(self, container_selector_or_elem, value, with_click=True, clear_value=False):
         # There are two hacky was to select things from the select2 widget -
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 46a476e85a0..9df487c6c9c 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -165,36 +165,42 @@ history_panel:
       list_items: '.dataset-collection-panel .list-items .list-item'
 
   selectors:
+    beta: 'section.history'
     _: '#current-history-panel'
     search: '#current-history-panel input.search-query'
     refresh_button: '#history-refresh-button'
     name: '.title .name'
+    name_beta: '.history-title span:last-child'
     name_edit_input: '.name input'
+    name_edit_input_beta: '.history-title input'
     contents: '#current-history-panel .list-items div.history-content'
 
     empty_message: '.empty-message'
     size: '.history-size'
     tag_icon: '.actions .history-tag-btn'
-    tag_area: '.controls .tags-display'
-    tag_area_input: '.controls .tags-display .tags-input input'
+    tag_area: '.history-details .tags-display'
+    tag_area_input: '.history-details .tags-display .tags-input input'
     tag_close_btn: '.tags-display .ti-icon-close'
     tags: 'li.ti-tag.ti-valid .tag-name'
     annotation_icon: '.actions .history-annotate-btn'
-    annotation_area: '.controls .annotation-display'
-    annotation_editable_text: '.controls .annotation-display .annotation.editable-text'
-    annotation_edit: '.controls .annotation-display .annotation textarea'
-    annotation_done: '.controls .annotation-display .annotation button'
+    annotation_area: '.history-details .history-annotation'
+    annotation_editable_text: '.history-details .history-annotation .annotation.editable-text'
+    annotation_editable_text_beta: '.history-details .history-annotation.clickToEdit'
+    annotation_edit: '.history-details .history-annotation .annotation textarea'
+    annotation_edit_beta: '.history-details .history-annotation textarea'
+    annotation_done: '.history-details .history-annotation .annotation button'
 
     options_button: '#history-options-button'
     options_button_icon: '#history-options-button span.fa-cog'
+    options_button_icon_beta: 'section.history-details .priority-menu-expand'
     options_menu: '#history-options-button-menu'
     options_show_history_structure:
       type: xpath
-      selector: '//a[text()="Show Structure"]'
+      selector: '//a[contains(text(), "Show Structure")]'
     options_show_export_history_to_file:
       type: xpath
       selector: '//a[text()="Export History to File"]'
-    new_history_button: '#history-new-button'
+    new_history_button: '.history-new-button'
     multi_view_button: '#history-view-multi-button'
 
     pagination_pages: '.list-pagination .pages'
@@ -205,7 +211,7 @@ history_panel:
 
 
   text:
-    tooltip_name: 'Click to rename history'
+    tooltip_name: 'Rename history...'
     new_name: 'Unnamed history'
     new_size: '(empty)'
 
diff --git a/lib/galaxy/selenium/smart_components.py b/lib/galaxy/selenium/smart_components.py
index 580eff9f8a4..58daeb5a277 100644
--- a/lib/galaxy/selenium/smart_components.py
+++ b/lib/galaxy/selenium/smart_components.py
@@ -75,6 +75,10 @@ class SmartTarget:
     def is_displayed(self):
         return self._has_driver.is_displayed(self._target)
 
+    @property
+    def is_absent(self):
+        return self._has_driver.element_absent(self._target)
+
     def wait_for_absent_or_hidden(self, **kwds):
         self._has_driver.wait_for_absent_or_hidden(self._target, **kwds)
 
diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py
index 8e471d910ba..1903d1c0d85 100644
--- a/lib/galaxy_test/selenium/framework.py
+++ b/lib/galaxy_test/selenium/framework.py
@@ -217,6 +217,8 @@ class TestWithSeleniumMixin(GalaxyTestSeleniumContext, UsesApiTestCaseMixin):
     # will be used to login.
     ensure_registered = False
 
+    ensure_beta_history = False
+
     # Override this in subclasses to annotate that an admin user
     # is required for the test to run properly. Override admin user
     # login info with GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL /
@@ -256,6 +258,8 @@ class TestWithSeleniumMixin(GalaxyTestSeleniumContext, UsesApiTestCaseMixin):
         """
         if self.ensure_registered:
             self.login()
+            if self.ensure_beta_history:
+                self.use_beta_history()
 
     def tear_down_selenium(self):
         self.tear_down_driver()
@@ -363,6 +367,7 @@ class TestWithSeleniumMixin(GalaxyTestSeleniumContext, UsesApiTestCaseMixin):
         initial_size_str = self.components.history_panel.new_size.text
         size_selector = self.components.history_panel.size
         size_text = size_selector.wait_for_text()
+
         assert initial_size_str in size_text, f"{initial_size_str} not in {size_text}"
 
         self.components.history_panel.empty_message.wait_for_visible()
diff --git a/lib/galaxy_test/selenium/test_history_panel.py b/lib/galaxy_test/selenium/test_history_panel.py
index 4de659b5de2..99537965ddf 100644
--- a/lib/galaxy_test/selenium/test_history_panel.py
+++ b/lib/galaxy_test/selenium/test_history_panel.py
@@ -1,8 +1,13 @@
+import pytest
+
 from .framework import (
+    retry_assertion_during_transitions,
     selenium_test,
-    SeleniumTestCase
+    SeleniumTestCase,
 )
 
+NEW_HISTORY_NAME = "New History Name"
+
 
 class HistoryPanelTestCase(SeleniumTestCase):
 
@@ -24,26 +29,32 @@ class HistoryPanelTestCase(SeleniumTestCase):
     @selenium_test
     def test_history_panel_rename(self):
         editable_text_input_element = self.history_panel_click_to_rename()
-        editable_text_input_element.send_keys("New History Name")
+        if self.is_beta_history():
+            editable_text_input_element.clear()
+        editable_text_input_element.send_keys(NEW_HISTORY_NAME)
         self.send_enter(editable_text_input_element)
 
-        assert "New History Name" in self.history_panel_name()
+        self.assert_name_changed()
 
     @selenium_test
     def test_history_rename_confirm_with_click(self):
         editable_text_input_element = self.history_panel_click_to_rename()
-        editable_text_input_element.send_keys("New History Name")
+        if self.is_beta_history():
+            editable_text_input_element.clear()
+        editable_text_input_element.send_keys(NEW_HISTORY_NAME)
         self.click_center()
         self.assert_absent(self.navigation.history_panel.selectors.name_edit_input)
-        assert "New History Name" in self.history_panel_name()
+        self.assert_name_changed()
 
     @selenium_test
     def test_history_rename_cancel_with_escape(self):
         editable_text_input_element = self.history_panel_click_to_rename()
-        editable_text_input_element.send_keys("New History Name")
+        if self.is_beta_history():
+            editable_text_input_element.clear()
+        editable_text_input_element.send_keys(NEW_HISTORY_NAME)
         self.send_escape(editable_text_input_element)
         self.assert_absent(self.navigation.history_panel.selectors.name_edit_input)
-        assert "New History Name" not in self.history_panel_name()
+        assert NEW_HISTORY_NAME not in self.history_panel_name()
 
     @selenium_test
     def test_history_tags_and_annotations_buttons(self):
@@ -79,10 +90,17 @@ class HistoryPanelTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_history_panel_annotations_change(self):
+        history_panel = self.components.history_panel
 
+        @retry_assertion_during_transitions
         def assert_current_annotation(expected, error_message="History annotation",
                                       is_equal=True):
-            current_annotation = self.components.history_panel.annotation_editable_text.wait_for_visible()
+
+            if self.is_beta_history():
+                text_component = history_panel.annotation_editable_text_beta
+            else:
+                text_component = history_panel.annotation_editable_text
+            current_annotation = text_component.wait_for_visible()
             error_message += " given: [%s] expected [%s] "
             if is_equal:
                 assert current_annotation.text == expected, error_message % (
@@ -97,7 +115,7 @@ class HistoryPanelTestCase(SeleniumTestCase):
             return random_annotation
 
         # assert that annotation wasn't set before
-        self.components.history_panel.annotation_area.assert_absent_or_hidden()
+        history_panel.annotation_area.assert_absent_or_hidden()
 
         # assign annotation random text
         initial_annotation = set_random_annotation()
@@ -160,6 +178,8 @@ class HistoryPanelTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_refresh_preserves_state(self):
+        if self.is_beta_history():
+            raise pytest.skip("Beta History Panel does not preserve state")
         self.perform_upload(self.get_filename("1.txt"))
         self.wait_for_history()
 
@@ -180,3 +200,12 @@ class HistoryPanelTestCase(SeleniumTestCase):
         self.sleep_for(self.wait_types.UX_TRANSITION)
         self.wait_for_selector_clickable(self.history_panel_item_selector(hid=1))
         assert not self.history_panel_item_showing_details(hid=1)
+
+    @retry_assertion_during_transitions
+    def assert_name_changed(self):
+        name = self.history_panel_name()
+        self.assertEqual(name, NEW_HISTORY_NAME)
+
+
+class BetaHistoryPanelTestCase(HistoryPanelTestCase):
+    ensure_beta_history = True

From 157b0dc9b2781f9540dad643df5a5c5d947affb1 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 3 May 2021 20:40:21 -0400
Subject: [PATCH 114/452] Sepearate set of tests for beta history panel.

---
 .github/workflows/selenium_beta.yaml          | 47 +++++++++++++++++++
 lib/galaxy_test/selenium/framework.py         |  3 +-
 .../selenium/test_history_panel.py            |  4 --
 3 files changed, 49 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/selenium_beta.yaml

diff --git a/.github/workflows/selenium_beta.yaml b/.github/workflows/selenium_beta.yaml
new file mode 100644
index 00000000000..345683fee09
--- /dev/null
+++ b/.github/workflows/selenium_beta.yaml
@@ -0,0 +1,47 @@
+name: Selenium tests (beta history panel)
+on: [push, pull_request]
+env:
+  GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
+  GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR: 'true'
+  GALAXY_TEST_SELENIUM_RETRIES: 1
+  GALAXY_TEST_SELENIUM_BETA_HISTORY: 1
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7']
+        chunk: [0, 1, 2]
+    continue-on-error: true
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip dir
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - uses: nanasess/setup-chromedriver@master
+      - name: Run tests
+        run: ./run_tests.sh -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }}
+        working-directory: 'galaxy root'
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Selenium test results (${{ matrix.python-version }}, ${{ matrix.chunk }})
+          path: 'galaxy root/database/test_errors'
diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py
index 1903d1c0d85..ad871a28b35 100644
--- a/lib/galaxy_test/selenium/framework.py
+++ b/lib/galaxy_test/selenium/framework.py
@@ -63,6 +63,7 @@ GALAXY_TEST_SELENIUM_USER_EMAIL = os.environ.get("GALAXY_TEST_SELENIUM_USER_EMAI
 GALAXY_TEST_SELENIUM_USER_PASSWORD = os.environ.get("GALAXY_TEST_SELENIUM_USER_PASSWORD", None)
 GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL = os.environ.get("GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL", DEFAULT_ADMIN_USER)
 GALAXY_TEST_SELENIUM_ADMIN_USER_PASSWORD = os.environ.get("GALAXY_TEST_SELENIUM_ADMIN_USER_PASSWORD", DEFAULT_ADMIN_PASSWORD)
+GALAXY_TEST_SELENIUM_BETA_HISTORY = os.environ.get("GALAXY_TEST_SELENIUM_BETA_HISTORY", "0") == "1"
 
 # JS code to execute in Galaxy JS console to setup localStorage of session for logging and
 # logging "flatten" messages because it seems Selenium (with Chrome at least) only grabs
@@ -217,7 +218,7 @@ class TestWithSeleniumMixin(GalaxyTestSeleniumContext, UsesApiTestCaseMixin):
     # will be used to login.
     ensure_registered = False
 
-    ensure_beta_history = False
+    ensure_beta_history = GALAXY_TEST_SELENIUM_BETA_HISTORY
 
     # Override this in subclasses to annotate that an admin user
     # is required for the test to run properly. Override admin user
diff --git a/lib/galaxy_test/selenium/test_history_panel.py b/lib/galaxy_test/selenium/test_history_panel.py
index 99537965ddf..3c0a69d0b02 100644
--- a/lib/galaxy_test/selenium/test_history_panel.py
+++ b/lib/galaxy_test/selenium/test_history_panel.py
@@ -205,7 +205,3 @@ class HistoryPanelTestCase(SeleniumTestCase):
     def assert_name_changed(self):
         name = self.history_panel_name()
         self.assertEqual(name, NEW_HISTORY_NAME)
-
-
-class BetaHistoryPanelTestCase(HistoryPanelTestCase):
-    ensure_beta_history = True

From 39813a6c56027c5862e06fed736e689e25e8d5c4 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 4 May 2021 09:27:56 -0400
Subject: [PATCH 115/452] Was this ever working?

---
 lib/galaxy/selenium/navigation.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 9df487c6c9c..64a946014bb 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -200,6 +200,9 @@ history_panel:
     options_show_export_history_to_file:
       type: xpath
       selector: '//a[text()="Export History to File"]'
+    options_use_beta_history:
+      type: xpath
+      selector: '//a[text()="Use Beta History Panel"]'
     new_history_button: '.history-new-button'
     multi_view_button: '#history-view-multi-button'
 

From c8ffe01277208c677c1126b2c3f67fcdcd1e6405 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 4 May 2021 09:30:25 -0400
Subject: [PATCH 116/452] Try not to fail build for beta history.

---
 .github/workflows/selenium_beta.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/selenium_beta.yaml b/.github/workflows/selenium_beta.yaml
index 345683fee09..743fad838f6 100644
--- a/.github/workflows/selenium_beta.yaml
+++ b/.github/workflows/selenium_beta.yaml
@@ -38,10 +38,9 @@ jobs:
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
       - uses: nanasess/setup-chromedriver@master
       - name: Run tests
-        run: ./run_tests.sh -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }}
+        run: ./run_tests.sh -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }} || true
         working-directory: 'galaxy root'
       - uses: actions/upload-artifact@v2
-        if: failure()
         with:
           name: Selenium test results (${{ matrix.python-version }}, ${{ matrix.chunk }})
           path: 'galaxy root/database/test_errors'

From c9bd18dd397d07f141ed066f0780e559d01cd6f5 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 4 May 2021 12:46:15 -0400
Subject: [PATCH 117/452] Publish test results.

---
 .github/workflows/selenium_beta.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/selenium_beta.yaml b/.github/workflows/selenium_beta.yaml
index 743fad838f6..74a0b2c7b4a 100644
--- a/.github/workflows/selenium_beta.yaml
+++ b/.github/workflows/selenium_beta.yaml
@@ -42,5 +42,10 @@ jobs:
         working-directory: 'galaxy root'
       - uses: actions/upload-artifact@v2
         with:
-          name: Selenium test results (${{ matrix.python-version }}, ${{ matrix.chunk }})
+          name: Selenium beta history panel test results (${{ matrix.python-version }}, ${{ matrix.chunk }})
           path: 'galaxy root/database/test_errors'
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Selenium beta history panel test results (${{ matrix.python-version }})
+          path: 'galaxy root/run_selenium_tests.html'
+ 
\ No newline at end of file

From f664e5ab6bd2adbec22085e4142f4e630230c6f4 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 4 May 2021 22:35:20 +0200
Subject: [PATCH 118/452] Reduce selenium logging

This should result in much smaller html results and more readable logs
overall.
---
 lib/galaxy/selenium/driver_factory.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/selenium/driver_factory.py b/lib/galaxy/selenium/driver_factory.py
index 549761cafa3..6a9bdea74de 100644
--- a/lib/galaxy/selenium/driver_factory.py
+++ b/lib/galaxy/selenium/driver_factory.py
@@ -1,3 +1,4 @@
+import logging
 import os
 
 try:
@@ -10,6 +11,8 @@ from selenium.webdriver.chrome.options import Options as ChromeOptions
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.remote.webdriver import WebDriver
 
+logger = logging.getLogger('selenium.webdriver.remote.remote_connection')
+logger.setLevel(logging.WARNING)
 
 DEFAULT_BROWSER = "auto"
 DEFAULT_DOWNLOAD_PATH = '/tmp/'

From e5eee36d4a8630c5379abfcb46924a989362dddd Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 10 May 2021 10:31:40 -0400
Subject: [PATCH 119/452] IDs -> Classes for history panel buttons, consistency
 accross history panels.

---
 client/src/components/History/HistoriesMenu.vue   | 4 +++-
 client/src/entry/panels/history-panel.js          | 4 ++--
 lib/galaxy/selenium/navigation.yml                | 4 ++--
 lib/galaxy_test/selenium/test_navigates_galaxy.py | 2 +-
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/client/src/components/History/HistoriesMenu.vue b/client/src/components/History/HistoriesMenu.vue
index 7a23c6a120d..ceb38a7df17 100644
--- a/client/src/components/History/HistoriesMenu.vue
+++ b/client/src/components/History/HistoriesMenu.vue
@@ -17,6 +17,7 @@
             key="view-all-histories"
             title="View All Histories"
             icon="fa fa-columns"
+            class="history-view-multi-button"
             @click="redirect('/history/view_multiple')"
         />
         <PriorityMenuItem
@@ -28,7 +29,8 @@
         <PriorityMenuItem
             key="clear-history-cache"
             title="Refresh History"
-            icon="fa fa-sync"
+            icon="fa fa-refresh"
+            class="history-refresh-button"
             @click.stop="clearCache"
         />
         <PriorityMenuItem
diff --git a/client/src/entry/panels/history-panel.js b/client/src/entry/panels/history-panel.js
index 665be032280..ed7ee2d3147 100644
--- a/client/src/entry/panels/history-panel.js
+++ b/client/src/entry/panels/history-panel.js
@@ -30,7 +30,7 @@ const HistoryPanel = Backbone.View.extend({
         this.buttonRefresh = new Ui.ButtonLink({
             id: "history-refresh-button",
             title: _l("Refresh history"),
-            cls: "panel-header-button",
+            cls: "panel-header-button history-refresh-button",
             icon: "fa fa-sync",
             onclick: () => {
                 this.historyView.loadCurrentHistory();
@@ -54,7 +54,7 @@ const HistoryPanel = Backbone.View.extend({
         this.buttonViewMulti = new Ui.ButtonLink({
             id: "history-view-multi-button",
             title: _l("View all histories"),
-            cls: "panel-header-button",
+            cls: "panel-header-button history-view-multi-button",
             icon: "fa fa-columns",
             href: `${this.root}history/view_multiple`,
         });
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 64a946014bb..3017a84522b 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -168,7 +168,7 @@ history_panel:
     beta: 'section.history'
     _: '#current-history-panel'
     search: '#current-history-panel input.search-query'
-    refresh_button: '#history-refresh-button'
+    refresh_button: '.history-refresh-button'
     name: '.title .name'
     name_beta: '.history-title span:last-child'
     name_edit_input: '.name input'
@@ -204,7 +204,7 @@ history_panel:
       type: xpath
       selector: '//a[text()="Use Beta History Panel"]'
     new_history_button: '.history-new-button'
-    multi_view_button: '#history-view-multi-button'
+    multi_view_button: '.history-view-multi-button'
 
     pagination_pages: '.list-pagination .pages'
     pagination_pages_options: '.list-pagination .pages option'
diff --git a/lib/galaxy_test/selenium/test_navigates_galaxy.py b/lib/galaxy_test/selenium/test_navigates_galaxy.py
index c7906791ac2..8e1f7c2fcc1 100644
--- a/lib/galaxy_test/selenium/test_navigates_galaxy.py
+++ b/lib/galaxy_test/selenium/test_navigates_galaxy.py
@@ -22,7 +22,7 @@ class NavigatesGalaxySeleniumTestCase(SeleniumTestCase):
         # Open the details, verify they are open and do a refresh.
         exception = None
         try:
-            refresh_button = self.wait_for_selector("#history-refresh-button")
+            refresh_button = self.wait_for_selector(".history-refresh-button")
             refresh_button.click()
         except Exception as e:
             exception = e

From 23a0bd53e0d8a6cb4ba7b641480d82130da9fd8e Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 10 May 2021 13:57:38 -0400
Subject: [PATCH 120/452] Restore legacy id behavior for history panel list.

---
 .../src/components/History/ContentItem/Dataset/DatasetUI.vue  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index f77640cc9cf..75fb739e712 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -8,6 +8,7 @@ either through the props, and make updates through the events -->
 <template>
     <div
         class="dataset"
+        :id="typedId"
         :class="{ expanded, collapsed, selected }"
         :data-state="dataset.state"
         @keydown.arrow-left.self.stop="$emit('update:expanded', false)"
@@ -143,6 +144,9 @@ export default {
         ok() {
             return this.dataset.state == "ok";
         },
+        typedId() {
+            return `dataset-${this.dataset.id}`;
+        },
     },
     methods: {
         onStatusClick(evt) {

From d66b9a3c1cf98f0f41644787bad8b390efde7128 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 10 May 2021 18:32:13 -0400
Subject: [PATCH 121/452] Unified dataset state selectors using data
 attributes.

---
 client/src/mvc/dataset/dataset-li.js    |  1 +
 lib/galaxy/selenium/components.py       | 14 ++++++++++++--
 lib/galaxy/selenium/navigates_galaxy.py |  3 ++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/client/src/mvc/dataset/dataset-li.js b/client/src/mvc/dataset/dataset-li.js
index 33fae7ef956..2aba4151794 100644
--- a/client/src/mvc/dataset/dataset-li.js
+++ b/client/src/mvc/dataset/dataset-li.js
@@ -135,6 +135,7 @@ export var DatasetListItemView = _super.extend(
             _super.prototype._swapNewRender.call(this, $newRender);
             if (this.model.has("state")) {
                 this.$el.addClass(`state-${this.model.get("state")}`);
+                this.$el.attr("data-state", this.model.get("state"));
             }
             return this.$el;
         },
diff --git a/lib/galaxy/selenium/components.py b/lib/galaxy/selenium/components.py
index 44bc3a9b8a9..182643cb527 100644
--- a/lib/galaxy/selenium/components.py
+++ b/lib/galaxy/selenium/components.py
@@ -23,12 +23,13 @@ class Target(metaclass=ABCMeta):
 
 class SelectorTemplate(Target):
 
-    def __init__(self, selector, selector_type, children=None, kwds=None, with_classes=None):
+    def __init__(self, selector, selector_type, children=None, kwds=None, with_classes=None, with_data=None):
         self._selector = selector
         self.selector_type = selector_type
         self._children = children or {}
         self.__kwds = kwds or {}
         self.with_classes = with_classes or []
+        self._with_data = with_data or {}
 
     @staticmethod
     def from_dict(raw_value, children=None):
@@ -40,7 +41,13 @@ class SelectorTemplate(Target):
 
     def with_class(self, class_):
         assert self.selector_type == "css"
-        return SelectorTemplate(self._selector, self.selector_type, kwds=self.__kwds, with_classes=self.with_classes + [class_], children=self._children)
+        return SelectorTemplate(self._selector, self.selector_type, kwds=self.__kwds, with_classes=self.with_classes + [class_], with_data=self._with_data.copy(), children=self._children)
+
+    def with_data(self, key, value):
+        assert self.selector_type == "css"
+        with_data = self._with_data.copy()
+        with_data[key] = value
+        return SelectorTemplate(self._selector, self.selector_type, kwds=self.__kwds, with_classes=self.with_classes, with_data=with_data, children=self._children)
 
     def descendant(self, has_selector):
         assert self.selector_type == "css"
@@ -72,6 +79,9 @@ class SelectorTemplate(Target):
         if self.__kwds is not None:
             selector = string.Template(selector).substitute(self.__kwds)
         selector = selector + "".join(f".{c}" for c in self.with_classes)
+        if self._with_data:
+            for key, value in self._with_data.items():
+                selector = selector + f'[data-{key}="{value}"]'
         return selector
 
     @property
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 25a6dd03c24..d3377caa8ec 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -357,7 +357,8 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_wait_for_hid_state(self, hid, state, allowed_force_refreshes=0):
         history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes)
-        history_item_selector_state = history_item_selector.with_class(f"state-{state}")
+        # history_item_selector_state = history_item_selector.with_class(f"state-{state}")
+        history_item_selector_state = history_item_selector.with_data("state", state)
         try:
             self.history_item_wait_for(history_item_selector_state, allowed_force_refreshes)
         except self.TimeoutException as e:

From 94c4615817051e890970c77231208c3a5b2af7d9 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 13 May 2021 14:06:31 -0400
Subject: [PATCH 122/452] Rebase into state decorating.

---
 client/src/mvc/collection/collection-li.js | 1 +
 client/src/mvc/history/hdca-li.js          | 1 +
 2 files changed, 2 insertions(+)

diff --git a/client/src/mvc/collection/collection-li.js b/client/src/mvc/collection/collection-li.js
index 6fdef8ea7ba..148588bfa19 100644
--- a/client/src/mvc/collection/collection-li.js
+++ b/client/src/mvc/collection/collection-li.js
@@ -255,6 +255,7 @@ var NestedDCDCEListItemView = DCListItemView.extend(
             DCListItemView.prototype._swapNewRender.call(this, $newRender);
             var state = this.model.get("state") || "ok";
             this.$el.addClass(`state-${state}`);
+            this.$el.attr("data-state", state);
             return this.$el;
         },
 
diff --git a/client/src/mvc/history/hdca-li.js b/client/src/mvc/history/hdca-li.js
index f5b7aec7c2c..6570ae7013e 100644
--- a/client/src/mvc/history/hdca-li.js
+++ b/client/src/mvc/history/hdca-li.js
@@ -75,6 +75,7 @@ var HDCAListItemView = _super.extend(
                 state = this.model.get("populated_state") ? STATES.OK : STATES.RUNNING;
             }
             this.$el.addClass(`state-${state}`);
+            this.$el.attr("data-state", state);
             const collection = this.model;
             const stateContainer = this.$el.find(".state-description")[0];
             mountCollectionJobStates({ jobStatesSummary, collection }, stateContainer);

From 65828149e663730877d3f523414a756e516154b3 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 20 May 2021 12:42:18 -0400
Subject: [PATCH 123/452] Try to fix history element selection when invocation
 is also present.

---
 lib/galaxy/selenium/navigation.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 3017a84522b..8a4dc229d9d 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -113,7 +113,9 @@ history_panel:
 
   item:
     selectors:
-      _: '#${history_content_type}-${id}'
+      # These now appear other places :_( - e.g. in the invocation view so we'll try
+      # prefixing ids with #current-history-panel but obviously we need to switch to classes or data
+      _: '#current-history-panel #${history_content_type}-${id}'
 
       title: '${_} .title'
       hid: '${_} .hid'

From fc0ac12e264653d4341810ae2a99d20866370b18 Mon Sep 17 00:00:00 2001
From: Chilton <jxc755@E1-054863.local>
Date: Tue, 25 May 2021 13:55:23 -0400
Subject: [PATCH 124/452] try to fix multi-history panel selection stuff in
 navigates galaxy.

---
 lib/galaxy/selenium/navigates_galaxy.py | 21 ++++++++++++---------
 lib/galaxy/selenium/navigation.yml      |  2 ++
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index d3377caa8ec..fdbdbc5345a 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -292,16 +292,19 @@ class NavigatesGalaxy(HasDriver):
     def history_panel_wait_for_hid_ok(self, hid, allowed_force_refreshes=0):
         self.history_panel_wait_for_hid_state(hid, 'ok', allowed_force_refreshes=allowed_force_refreshes)
 
-    def history_panel_item_component(self, history_item=None, hid=None):
+    def history_panel_item_component(self, history_item=None, hid=None, multi_history_panel=False):
         if history_item is None:
             assert hid
             history_item = self.hid_to_history_item(hid)
-        return self.components.history_panel.item.selector(
+        item = self.components.history_panel.item
+        if multi_history_panel:
+            item = self.components.multi_history_view.item
+        return item.selector(
             history_content_type=history_item["history_content_type"],
             id=history_item["id"]
         )
 
-    def history_panel_wait_for_hid_visible(self, hid, allowed_force_refreshes=0):
+    def history_panel_wait_for_hid_visible(self, hid, allowed_force_refreshes=0, multi_history_panel=False):
         current_history_id = self.current_history_id()
 
         def history_has_hid(driver):
@@ -311,7 +314,7 @@ class NavigatesGalaxy(HasDriver):
         timeout = self.timeout_for(wait_type=WAIT_TYPES.JOB_COMPLETION)
         self.wait(timeout).until(history_has_hid)
         history_item = self.hid_to_history_item(hid, current_history_id=current_history_id)
-        history_item_selector = self.history_panel_item_component(history_item)
+        history_item_selector = self.history_panel_item_component(history_item, multi_history_panel=multi_history_panel)
         try:
             self.history_item_wait_for(history_item_selector, allowed_force_refreshes)
         except self.TimeoutException as e:
@@ -349,14 +352,14 @@ class NavigatesGalaxy(HasDriver):
         # (even if empty).
         self.wait_for_visible(self.navigation.history_panel.selectors.search, wait_type=WAIT_TYPES.DATABASE_OPERATION)
 
-    def history_panel_wait_for_hid_hidden(self, hid):
+    def history_panel_wait_for_hid_hidden(self, hid, multi_history_panel=False):
         history_item = self.hid_to_history_item(hid)
-        history_item_selector = self.history_panel_item_component(history_item)
+        history_item_selector = self.history_panel_item_component(history_item, multi_history_panel=multi_history_panel)
         self.wait_for_absent_or_hidden(history_item_selector, wait_type=WAIT_TYPES.JOB_COMPLETION)
         return history_item_selector
 
-    def history_panel_wait_for_hid_state(self, hid, state, allowed_force_refreshes=0):
-        history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes)
+    def history_panel_wait_for_hid_state(self, hid, state, allowed_force_refreshes=0, multi_history_panel=False):
+        history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes, multi_history_panel=multi_history_panel)
         # history_item_selector_state = history_item_selector.with_class(f"state-{state}")
         history_item_selector_state = history_item_selector.with_data("state", state)
         try:
@@ -1315,7 +1318,7 @@ class NavigatesGalaxy(HasDriver):
     def history_multi_view_display_collection_contents(self, collection_hid, collection_type="list"):
         self.components.history_panel.multi_view_button.wait_for_and_click()
 
-        selector = self.history_panel_wait_for_hid_state(collection_hid, "ok")
+        selector = self.history_panel_wait_for_hid_state(collection_hid, "ok", multi_history_panel=True)
         self.click(selector)
         next_level_element_selector = selector
         for _ in range(len(collection_type.split(":")) - 1):
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 8a4dc229d9d..df65131297d 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -254,6 +254,8 @@ multi_history_view:
 
   selectors:
     _: '.multi-panel-history'
+    item: '.multi-panel-history #${history_content_type}-${id}'
+
     histories: '.middle .history-column'
     current_label: '.current-label'
     create_new_button: '.create-new'

From d54dc106494855e27d38ae38592d13ad3059c83b Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 26 May 2021 10:10:37 -0400
Subject: [PATCH 125/452] more fixes.

---
 lib/galaxy_test/selenium/test_history_multi_view.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py
index b5886453c97..43e3fa1d9bf 100644
--- a/lib/galaxy_test/selenium/test_history_multi_view.py
+++ b/lib/galaxy_test/selenium/test_history_multi_view.py
@@ -16,11 +16,10 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
 
         self.home()
 
-        hdca_selector = self.history_panel_wait_for_hid_state(input_hid, "ok")
-
         self.components.history_panel.multi_view_button.wait_for_and_click()
         self.components.multi_history_view.create_new_button.wait_for_and_click()
         self.components.multi_history_view.drag_drop_help.wait_for_visible()
+        hdca_selector = self.history_panel_wait_for_hid_state(input_hid, "ok", multi_history_panel=True)
         self.wait_for_visible(hdca_selector)
         self.screenshot("multi_history_collection")
 
@@ -144,6 +143,6 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
         self.home()
         self.components.history_panel.multi_view_button.wait_for_and_click()
 
-        selector = self.history_panel_wait_for_hid_state(collection_hid, "ok")
+        selector = self.history_panel_wait_for_hid_state(collection_hid, "ok", multi_history_panel=True)
         self.click(selector)
         return selector

From d36363d10fc7955d52ed8859992ac1a01d693cf4 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 26 May 2021 16:39:35 -0400
Subject: [PATCH 126/452] Another fix.

---
 lib/galaxy/selenium/navigates_galaxy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index fdbdbc5345a..2f5d22bce62 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -296,10 +296,10 @@ class NavigatesGalaxy(HasDriver):
         if history_item is None:
             assert hid
             history_item = self.hid_to_history_item(hid)
-        item = self.components.history_panel.item
+        item = self.components.history_panel.item.selector
         if multi_history_panel:
             item = self.components.multi_history_view.item
-        return item.selector(
+        return item(
             history_content_type=history_item["history_content_type"],
             id=history_item["id"]
         )

From 22835f1357737d245769d65bb8ea424f4667b2b2 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 27 May 2021 09:59:06 -0400
Subject: [PATCH 127/452] better error message while waiting for hids to appear
 in histories.

---
 lib/galaxy/selenium/navigates_galaxy.py       | 22 ++++++++++++++-----
 .../selenium/test_history_multi_view.py       |  4 ++--
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 2f5d22bce62..0f4e15c88ab 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -304,15 +304,27 @@ class NavigatesGalaxy(HasDriver):
             id=history_item["id"]
         )
 
-    def history_panel_wait_for_hid_visible(self, hid, allowed_force_refreshes=0, multi_history_panel=False):
-        current_history_id = self.current_history_id()
+    def wait_for_history_to_have_hid(self, history_id, hid):
+
+        def get_hids():
+            contents = self.api_get(f"histories/{history_id}/contents")
+            return [d["hid"] for d in contents]
 
         def history_has_hid(driver):
-            contents = self.api_get(f"histories/{current_history_id}/contents")
-            return any([d for d in contents if d["hid"] == hid])
+            hids = get_hids()
+            return any([h == hid for h in hids])
 
         timeout = self.timeout_for(wait_type=WAIT_TYPES.JOB_COMPLETION)
-        self.wait(timeout).until(history_has_hid)
+        try:
+            self.wait(timeout).until(history_has_hid)
+        except self.TimeoutException as e:
+            hids = get_hids()
+            message = f"Timeout waiting for history {history_id} to have hid {hid} - have hids {hids}"
+            raise self.prepend_timeout_message(e, message)
+
+    def history_panel_wait_for_hid_visible(self, hid, allowed_force_refreshes=0, multi_history_panel=False):
+        current_history_id = self.current_history_id()
+        self.wait_for_history_to_have_hid(current_history_id, hid)
         history_item = self.hid_to_history_item(hid, current_history_id=current_history_id)
         history_item_selector = self.history_panel_item_component(history_item, multi_history_panel=multi_history_panel)
         try:
diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py
index 43e3fa1d9bf..7c0991dfc39 100644
--- a/lib/galaxy_test/selenium/test_history_multi_view.py
+++ b/lib/galaxy_test/selenium/test_history_multi_view.py
@@ -17,10 +17,10 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
         self.home()
 
         self.components.history_panel.multi_view_button.wait_for_and_click()
-        self.components.multi_history_view.create_new_button.wait_for_and_click()
-        self.components.multi_history_view.drag_drop_help.wait_for_visible()
         hdca_selector = self.history_panel_wait_for_hid_state(input_hid, "ok", multi_history_panel=True)
         self.wait_for_visible(hdca_selector)
+        self.components.multi_history_view.create_new_button.wait_for_and_click()
+        self.components.multi_history_view.drag_drop_help.wait_for_visible()
         self.screenshot("multi_history_collection")
 
     @selenium_test

From 5d66d9bcc6930911455b66b3c93700f37163ca63 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 9 Jun 2021 11:50:20 -0400
Subject: [PATCH 128/452] PR comments for #10965

---
 .github/workflows/selenium_beta.yaml           | 2 +-
 client/src/components/PriorityMenu/Menu.vue    | 2 +-
 client/src/entry/panels/history-panel.js       | 4 ++--
 lib/galaxy/selenium/navigates_galaxy.py        | 7 ++-----
 lib/galaxy/selenium/navigation.yml             | 8 ++++++--
 lib/galaxy_test/selenium/test_history_panel.py | 2 +-
 6 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/selenium_beta.yaml b/.github/workflows/selenium_beta.yaml
index 74a0b2c7b4a..a0a90bee180 100644
--- a/.github/workflows/selenium_beta.yaml
+++ b/.github/workflows/selenium_beta.yaml
@@ -2,7 +2,7 @@ name: Selenium tests (beta history panel)
 on: [push, pull_request]
 env:
   GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
-  GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR: 'true'
+  GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR: 1
   GALAXY_TEST_SELENIUM_RETRIES: 1
   GALAXY_TEST_SELENIUM_BETA_HISTORY: 1
 jobs:
diff --git a/client/src/components/PriorityMenu/Menu.vue b/client/src/components/PriorityMenu/Menu.vue
index df5aad484a2..6759d889bc2 100644
--- a/client/src/components/PriorityMenu/Menu.vue
+++ b/client/src/components/PriorityMenu/Menu.vue
@@ -14,7 +14,7 @@
             <div class="overflow" v-if="overflow.length">
                 <b-dropdown no-caret right variant="link" size="sm" boundary="window" toggle-class="p-1">
                     <template v-slot:button-content>
-                        <i class="fas fa-ellipsis-v priority-menu-expand" />
+                        <i class="fas fa-ellipsis-v priority-menu-expand menu-expand-button" />
                         <span class="sr-only">...</span>
                     </template>
                     <b-dropdown-item v-for="item in overflow" :key="item.key" v-bind="item.attrs" v-on="item.on">
diff --git a/client/src/entry/panels/history-panel.js b/client/src/entry/panels/history-panel.js
index ed7ee2d3147..6780cd37987 100644
--- a/client/src/entry/panels/history-panel.js
+++ b/client/src/entry/panels/history-panel.js
@@ -63,7 +63,7 @@ const HistoryPanel = Backbone.View.extend({
         this.buttonOptions = new Ui.ButtonLink({
             id: "history-options-button",
             title: _l("History options"),
-            cls: "panel-header-button",
+            cls: "panel-header-button menu-expand-button",
             target: "galaxy_main",
             icon: "fa fa-cog",
             href: `${this.root}root/history_options`,
@@ -72,7 +72,7 @@ const HistoryPanel = Backbone.View.extend({
 
         this.model = new Backbone.Model({
             // define components
-            cls: "history-right-panel",
+            cls: "history-right-panel history-details",
             title: _l("History"),
             buttons: panelHeaderButtons,
         });
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 0f4e15c88ab..cbb3b3a462a 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -1218,10 +1218,7 @@ class NavigatesGalaxy(HasDriver):
 
     @retry_during_transitions
     def click_history_options(self):
-        if self.is_beta_history():
-            component = self.components.history_panel.options_button_icon_beta
-        else:
-            component = self.components.history_panel.options_button_icon
+        component = self.components.history_panel.options_button_icon
         component.wait_for_and_click()
 
     def click_history_option(self, option_label_or_component):
@@ -1232,7 +1229,7 @@ class NavigatesGalaxy(HasDriver):
             option_label = option_label_or_component
             # Click labelled option
             self.wait_for_visible(self.navigation.history_panel.options_menu)
-            menu_item_sizzle_selector = f"#history-options-button-menu > a:contains('{option_label}')"
+            menu_item_sizzle_selector = self.navigation.history_panel.options_menu_item(option_label=option_label).selector
             menu_selection_element = self.wait_for_sizzle_selector_clickable(menu_item_sizzle_selector)
             menu_selection_element.click()
         else:
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index df65131297d..10b5dddb99f 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -193,9 +193,13 @@ history_panel:
     annotation_done: '.history-details .history-annotation .annotation button'
 
     options_button: '#history-options-button'
-    options_button_icon: '#history-options-button span.fa-cog'
-    options_button_icon_beta: 'section.history-details .priority-menu-expand'
+    options_button_icon: '.history-details .menu-expand-button'
+    options_button_icon_beta: '.history-details .menu-expand-button'
     options_menu: '#history-options-button-menu'
+    options_menu_item:
+      type: sizzle
+      selector: '#history-options-button-menu > a:contains("${option_label}")'
+
     options_show_history_structure:
       type: xpath
       selector: '//a[contains(text(), "Show Structure")]'
diff --git a/lib/galaxy_test/selenium/test_history_panel.py b/lib/galaxy_test/selenium/test_history_panel.py
index 3c0a69d0b02..d6008255a24 100644
--- a/lib/galaxy_test/selenium/test_history_panel.py
+++ b/lib/galaxy_test/selenium/test_history_panel.py
@@ -179,7 +179,7 @@ class HistoryPanelTestCase(SeleniumTestCase):
     @selenium_test
     def test_refresh_preserves_state(self):
         if self.is_beta_history():
-            raise pytest.skip("Beta History Panel does not preserve state")
+            raise pytest.skip("Beta History Panel does not preserve state xref https://github.com/galaxyproject/galaxy/issues/12119")
         self.perform_upload(self.get_filename("1.txt"))
         self.wait_for_history()
 

From 1a909f9c2f728004932182f7b0f72e41124a1943 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 1 Jul 2021 14:21:54 -0400
Subject: [PATCH 129/452] Progress on porting selenium tests.

---
 .../components/History/ContentItem/Dataset/DatasetUI.vue   | 2 +-
 .../ContentItem/DatasetCollection/DatasetCollection.vue    | 2 +-
 .../History/ContentItem/DatasetCollection/DscUI.vue        | 6 ++++++
 client/src/components/History/ContentOperations.vue        | 1 +
 client/src/components/History/CurrentHistoryPanel.vue      | 2 +-
 lib/galaxy/selenium/has_driver.py                          | 6 +++++-
 lib/galaxy/selenium/navigates_galaxy.py                    | 7 ++++++-
 lib/galaxy/selenium/navigation.yml                         | 7 ++++++-
 8 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 75fb739e712..6c609f5c36d 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -7,7 +7,7 @@ either through the props, and make updates through the events -->
 
 <template>
     <div
-        class="dataset"
+        class="dataset history-content"
         :id="typedId"
         :class="{ expanded, collapsed, selected }"
         :data-state="dataset.state"
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue b/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue
index 86e54a534df..c8b44f8d3d9 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue
@@ -3,7 +3,7 @@
         v-if="dsc"
         v-bind="$attrs"
         v-on="$listeners"
-        class="dataset-collection"
+        class="dataset-collection history-content"
         :dsc="dsc"
         @update:dsc="onUpdate"
         @delete="onDelete"
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index 9a551ba24f1..631c8a5df6b 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -5,6 +5,7 @@
     <div
         class="dataset dataset-collection collapsed"
         :class="{ selected }"
+        :id="typedId"
         :data-state="dsc.state"
         @keydown.arrow-right.self.stop="$emit('viewCollection')"
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
@@ -99,6 +100,11 @@ export default {
             console.log("onStatusClick", ...arguments);
         },
     },
+    computed: {
+        typedId() {
+            return this.dsc.type_id;
+        },
+    },
 };
 </script>
 
diff --git a/client/src/components/History/ContentOperations.vue b/client/src/components/History/ContentOperations.vue
index d1b66ef6216..a43ef2363cc 100644
--- a/client/src/components/History/ContentOperations.vue
+++ b/client/src/components/History/ContentOperations.vue
@@ -27,6 +27,7 @@
 
                 <PriorityMenuItem
                     key="copy-datasets"
+                    class="copy-datasets-menu-item"
                     title="Copy Datasets"
                     icon="fas fa-copy"
                     @click="iframeRedirect('/dataset/copy_datasets')"
diff --git a/client/src/components/History/CurrentHistoryPanel.vue b/client/src/components/History/CurrentHistoryPanel.vue
index 0adeefd884d..7490292c41c 100644
--- a/client/src/components/History/CurrentHistoryPanel.vue
+++ b/client/src/components/History/CurrentHistoryPanel.vue
@@ -3,7 +3,7 @@
         <UserHistories v-if="user" :user="user" v-slot="{ currentHistory, histories, handlers }">
             <HistoryPanel v-if="currentHistory" :history="currentHistory" v-on="handlers">
                 <template v-slot:nav>
-                    <div>
+                    <div id="current-history-panel">
                         <HistorySelector
                             :histories="histories"
                             :current-history="currentHistory"
diff --git a/lib/galaxy/selenium/has_driver.py b/lib/galaxy/selenium/has_driver.py
index df71cb39b28..c8b1226bb30 100644
--- a/lib/galaxy/selenium/has_driver.py
+++ b/lib/galaxy/selenium/has_driver.py
@@ -208,8 +208,12 @@ class HasDriver:
         submit_button.click()
 
     def prepend_timeout_message(self, timeout_exception, message):
+        msg = message
+        timeout_msg = timeout_exception.msg
+        if timeout_msg:
+            msg += f" {timeout_msg}"
         return TimeoutException(
-            msg=message + (timeout_exception.msg or ''),
+            msg=msg,
             screen=timeout_exception.screen,
             stacktrace=timeout_exception.stacktrace,
         )
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index cbb3b3a462a..a9e107aee9f 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -299,6 +299,8 @@ class NavigatesGalaxy(HasDriver):
         item = self.components.history_panel.item.selector
         if multi_history_panel:
             item = self.components.multi_history_view.item
+        elif self.is_beta_history():
+            item = self.components.history_panel.beta_item.selector
         return item(
             history_content_type=history_item["history_content_type"],
             id=history_item["id"]
@@ -1248,7 +1250,10 @@ class NavigatesGalaxy(HasDriver):
         self.components.history_panel.new_history_button.wait_for_and_click()
 
     def history_panel_click_copy_elements(self):
-        self.click_history_option("Copy Datasets")
+        if self.is_beta_history():
+            self.components.history_panel.options_button_copy_datasets_beta.wait_for_and_click()
+        else:
+            self.click_history_option("Copy Datasets")
 
     @retry_during_transitions
     def histories_click_advanced_search(self):
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 10b5dddb99f..f3531eb559f 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -111,6 +111,10 @@ history_panel:
     labels:
       new: 'Create New'
 
+  beta_item:
+    selectors:
+      _: ' #${history_content_type}-${id}' 
+
   item:
     selectors:
       # These now appear other places :_( - e.g. in the invocation view so we'll try
@@ -175,7 +179,7 @@ history_panel:
     name_beta: '.history-title span:last-child'
     name_edit_input: '.name input'
     name_edit_input_beta: '.history-title input'
-    contents: '#current-history-panel .list-items div.history-content'
+    contents: '#current-history-panel .history-content'
 
     empty_message: '.empty-message'
     size: '.history-size'
@@ -195,6 +199,7 @@ history_panel:
     options_button: '#history-options-button'
     options_button_icon: '.history-details .menu-expand-button'
     options_button_icon_beta: '.history-details .menu-expand-button'
+    options_button_copy_datasets_beta: '.copy-datasets-menu-item'
     options_menu: '#history-options-button-menu'
     options_menu_item:
       type: sizzle

From 6c076cc78107e8b811098e1439fdb565214553e2 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 1 Jul 2021 15:00:55 -0400
Subject: [PATCH 130/452] Progress on history panel test convergence.

---
 .../History/ContentItem/Dataset/DatasetUI.vue       |  7 ++++++-
 .../History/ContentItem/DatasetCollection/DscUI.vue |  7 ++++++-
 client/src/components/History/ContentOperations.vue | 13 +++++++++++--
 client/src/mvc/history/history-view-edit.js         |  2 +-
 client/src/mvc/history/history-view.js              |  9 +++++----
 client/src/mvc/list/list-view.js                    |  3 ++-
 lib/galaxy/selenium/navigation.yml                  |  6 +++---
 7 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 6c609f5c36d..aae19514302 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -18,7 +18,12 @@ either through the props, and make updates through the events -->
         <!-- name, state buttons, menus -->
         <nav class="content-top-menu" @click.stop="$emit('update:expanded', !expanded)">
             <div class="content-status-indicators" @click.stop>
-                <b-check v-if="showSelection" :checked="selected" @change="$emit('update:selected', $event)" />
+                <b-check
+                    class="selector"
+                    v-if="showSelection"
+                    :checked="selected"
+                    @change="$emit('update:selected', $event)"
+                />
                 <StatusIcon v-if="!ok" class="status-icon px-1" :state="dataset.state" @click.stop="onStatusClick" />
                 <StateBtn
                     v-if="!dataset.visible"
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index 631c8a5df6b..1795656350f 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -13,7 +13,12 @@
     >
         <nav class="content-top-menu">
             <div class="content-status-indicators mr-1" @click.stop>
-                <b-check v-if="showSelection" :checked="selected" @change="$emit('update:selected', $event)" />
+                <b-check
+                    class="selector"
+                    v-if="showSelection"
+                    :checked="selected"
+                    @change="$emit('update:selected', $event)"
+                />
 
                 <StatusIcon
                     v-if="dsc.state != 'ok'"
diff --git a/client/src/components/History/ContentOperations.vue b/client/src/components/History/ContentOperations.vue
index a43ef2363cc..3b3bd01cff4 100644
--- a/client/src/components/History/ContentOperations.vue
+++ b/client/src/components/History/ContentOperations.vue
@@ -20,6 +20,7 @@
                 <PriorityMenuItem
                     key="selection"
                     title="Operations on multiple datasets"
+                    class="show-history-content-selectors-btn"
                     icon="fas fa-check-square"
                     @click="$emit('update:show-selection', !showSelection)"
                     :pressed="showSelection"
@@ -68,7 +69,10 @@
         </transition>
 
         <transition name="shutterfade">
-            <b-button-toolbar v-if="showSelection" class="content-selection justify-content-between mt-1">
+            <b-button-toolbar
+                v-if="showSelection"
+                class="content-selection justify-content-between mt-1 list-action-menu"
+            >
                 <b-button-group>
                     <b-button size="sm" @click="$emit('selectAllContent')">
                         {{ "Select All" | localize }}
@@ -78,7 +82,12 @@
                     </b-button>
                 </b-button-group>
 
-                <b-dropdown class="ml-auto" size="sm" text="With Selected" :disabled="!hasSelection">
+                <b-dropdown
+                    class="ml-auto history-contents-list-action-menu-btn"
+                    size="sm"
+                    text="With Selected"
+                    :disabled="!hasSelection"
+                >
                     <b-dropdown-item @click="hideSelected">
                         {{ "Hide Datasets" | localize }}
                     </b-dropdown-item>
diff --git a/client/src/mvc/history/history-view-edit.js b/client/src/mvc/history/history-view-edit.js
index 3242113c61d..0a3d24e9f32 100644
--- a/client/src/mvc/history/history-view-edit.js
+++ b/client/src/mvc/history/history-view-edit.js
@@ -485,7 +485,7 @@ var HistoryViewEdit = _super.extend(
         // ------------------------------------------------------------------------ panel events
         /** event map */
         events: _.extend(_.clone(_super.prototype.events), {
-            "click .show-selectors-btn": "toggleSelectors",
+            "click .show-history-content-selectors-btn": "toggleSelectors",
             "click .toggle-deleted-link": function (ev) {
                 this.toggleShowDeleted();
             },
diff --git a/client/src/mvc/history/history-view.js b/client/src/mvc/history/history-view.js
index ef48b22e010..a240ad1781f 100644
--- a/client/src/mvc/history/history-view.js
+++ b/client/src/mvc/history/history-view.js
@@ -32,6 +32,7 @@ var _super = LIST_VIEW.ModelListPanel;
 var HistoryView = _super.extend(
     /** @lends HistoryView.prototype */ {
         _logNamespace: "history",
+        actionButtonClass: "history-contents-list-action-menu-btn",
 
         /** class to use for constructing the HDA views */
         HDAViewClass: HDA_LI.HDAListItemView,
@@ -172,18 +173,18 @@ var HistoryView = _super.extend(
             // do not render (and remove even) if nothing to select
             if (!this.views.length) {
                 this.hideSelectors();
-                $where.find(".controls .actions .show-selectors-btn").remove();
+                $where.find(".controls .actions .show-history-content-selectors-btn").remove();
                 return null;
             }
             // don't bother rendering if there's one already
-            var $existing = $where.find(".controls .actions .show-selectors-btn");
+            var $existing = $where.find(".controls .actions .show-history-content-selectors-btn");
             if ($existing.length) {
                 return $existing;
             }
 
             return faIconButton({
                 title: _l("Operations on multiple datasets"),
-                classes: "show-selectors-btn",
+                classes: "show-history-content-selectors-btn",
                 faIcon: "fa-check-square-o",
                 tooltipConfig: { placement: "top" },
             }).prependTo($where.find(".controls .actions"));
@@ -347,7 +348,7 @@ var HistoryView = _super.extend(
         // ------------------------------------------------------------------------ panel events
         /** event map */
         events: _.extend(_.clone(_super.prototype.events), {
-            "click .show-selectors-btn": "toggleSelectors",
+            "click .show-history-content-selectors-btn": "toggleSelectors",
             "click > .controls .prev": "_clickPrevPage",
             "click > .controls .next": "_clickNextPage",
             "change > .controls .pages": "_changePageSelect",
diff --git a/client/src/mvc/list/list-view.js b/client/src/mvc/list/list-view.js
index 4ed554f22dd..16bed0aa546 100644
--- a/client/src/mvc/list/list-view.js
+++ b/client/src/mvc/list/list-view.js
@@ -36,6 +36,7 @@ var ListPanel = Backbone.View.extend(BASE_MVC.LoggableMixin).extend(
 
         tagName: "div",
         className: "list-panel",
+        actionButtonClass: "list-action-menu-btn",
 
         /** (in ms) that jquery effects will use */
         fxSpeed: "fast",
@@ -297,7 +298,7 @@ var ListPanel = Backbone.View.extend(BASE_MVC.LoggableMixin).extend(
             }
             var $newMenu = $(
                 `<div class="list-action-menu btn-group">
-                    <button class="list-action-menu-btn btn btn-secondary dropdown-toggle" data-toggle="dropdown">
+                    <button class="${this.actionButtonClass} btn btn-secondary dropdown-toggle" data-toggle="dropdown">
                         ${_l("For all selected")}...
                     </button>
                     <div class="dropdown-menu float-right" role="menu"/>
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index f3531eb559f..0fd81523ccf 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -149,9 +149,9 @@ history_panel:
 
   multi_operations:
     selectors:
-      show_button: '#current-history-panel .actions .show-selectors-btn'
-      action_button: '#current-history-panel .list-action-menu .list-action-menu-btn'
-      action_menu: '.list-action-menu div.dropdown-menu'
+      show_button: '.show-history-content-selectors-btn'
+      action_button: '.history-contents-list-action-menu-btn'
+      action_menu: '.list-action-menu .dropdown-menu'
 
     labels:
       build_pair: "Build Dataset Pair"

From 609fb65a53d8194539a4643adf8a1d6d631faede Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 1 Jul 2021 17:01:41 -0400
Subject: [PATCH 131/452] Progress on beta history panel testing.

---
 .../selenium/test_collection_builders.py      | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy_test/selenium/test_collection_builders.py b/lib/galaxy_test/selenium/test_collection_builders.py
index ef1eb596a3e..0bf5ee41959 100644
--- a/lib/galaxy_test/selenium/test_collection_builders.py
+++ b/lib/galaxy_test/selenium/test_collection_builders.py
@@ -19,7 +19,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_set_name("my cool list")
         self.screenshot("collection_builder_list")
         self.collection_builder_create()
-        self.history_panel_wait_for_hid_ok(2)
+        self._collection_builder_wait_for_hid_ok(2)
 
     @selenium_test
     def test_build_list_and_show_items(self):
@@ -34,7 +34,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_set_name("my cool list")
 
         self.collection_builder_create()
-        self.history_panel_wait_for_hid_ok(3)
+        self._collection_builder_wait_for_hid_ok(3)
 
     @selenium_test
     def test_build_pair_simple_hidden(self):
@@ -49,7 +49,17 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_set_name("my awesome pair")
         self.screenshot("collection_builder_pair")
         self.collection_builder_create()
-        self.history_panel_wait_for_hid_ok(3)
+        self._collection_builder_wait_for_hid_ok(3)
+
+    def _collection_builder_wait_for_hid_ok(self, hid):
+        # a hacked up version of super().history_panel_wait_for_hid_ok for beta history panel not refreshing correctly.
+        allowed_force_refreshes = 0
+        if self.use_beta_history:
+            # so collection builder no longer forces refreshes of history panel.
+            # Why doesn't the panel pick it up automatically - an sqlite limitation?
+            # Or maybe refresh doesn't work if things are selected in the panel?
+            allowed_force_refreshes = 2
+        self.history_panel_wait_for_hid_ok(hid, allowed_force_refreshes=allowed_force_refreshes)
 
     @selenium_test
     def test_build_paired_list_simple(self):
@@ -68,7 +78,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_set_name("my awesome paired list")
         self.screenshot("collection_builder_paired_list")
         self.collection_builder_create()
-        self.history_panel_wait_for_hid_ok(3)
+        self._collection_builder_wait_for_hid_ok(3)
         self.history_panel_wait_for_hid_hidden(1)
         self.history_panel_wait_for_hid_hidden(2)
 
@@ -98,7 +108,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_set_name("my awesome paired list")
 
         self.collection_builder_create()
-        self.history_panel_wait_for_hid_ok(5)
+        self._collection_builder_wait_for_hid_ok(5)
         self.history_panel_refresh_click()
         self.history_panel_wait_for_hid_ok(1)
         self.history_panel_wait_for_hid_ok(2)

From a4cc01e3d1c50b6e87685606a132ce4e890c0b1f Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 1 Jul 2021 18:16:07 -0400
Subject: [PATCH 132/452] Fix export link in beta history panel.

---
 client/src/mvc/history/options-menu.js | 1 +
 client/src/mvc/ui/popup-menu.js        | 3 ++-
 lib/galaxy/selenium/navigation.yml     | 4 +---
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/mvc/history/options-menu.js b/client/src/mvc/history/options-menu.js
index 8a8b7b752f3..5da382e86cc 100644
--- a/client/src/mvc/history/options-menu.js
+++ b/client/src/mvc/history/options-menu.js
@@ -219,6 +219,7 @@ function buildMenu(isAnon, purgeAllowed, urlRoot) {
             menuOption.href = urlRoot + menuOption.href;
             menuOption.target = menuOption.target || "galaxy_main";
         }
+        menuOption.title = menuOption.html;
 
         if (menuOption.confirm) {
             menuOption.func = () => {
diff --git a/client/src/mvc/ui/popup-menu.js b/client/src/mvc/ui/popup-menu.js
index cd3e2376047..0fff0e0a888 100644
--- a/client/src/mvc/ui/popup-menu.js
+++ b/client/src/mvc/ui/popup-menu.js
@@ -84,7 +84,8 @@ const PopupMenu = Backbone.View.extend({
             const href = option.href || "javascript:void(0);";
             const target = option.target ? `target="${option.target}"` : "";
             const check = option.checked ? '<span class="fa fa-check mr-1"/>' : "";
-            return `<a class="popupmenu-option dropdown-item" href="${href}" ${target}>${check}${option.html}</a>`;
+            const title = option.title ? `title="${option.title}"` : "";
+            return `<a class="popupmenu-option dropdown-item" href="${href}" ${title} ${target}>${check}${option.html}</a>`;
         }).join("");
     },
 
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 0fd81523ccf..1d706052580 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -208,9 +208,7 @@ history_panel:
     options_show_history_structure:
       type: xpath
       selector: '//a[contains(text(), "Show Structure")]'
-    options_show_export_history_to_file:
-      type: xpath
-      selector: '//a[text()="Export History to File"]'
+    options_show_export_history_to_file: 'a[title="Export History to File"]'
     options_use_beta_history:
       type: xpath
       selector: '//a[text()="Use Beta History Panel"]'

From 9388d0276c7e86205d4cd870f0f71395201b01a1 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 2 Jul 2021 09:49:49 -0400
Subject: [PATCH 133/452] Progress on history panel selenium tests.

---
 .../History/ContentItem/Dataset/DatasetUI.vue          |  2 +-
 client/src/components/History/CurrentHistoryPanel.vue  |  4 ++--
 client/src/mvc/history/hda-li.js                       |  2 +-
 client/src/mvc/history/hdca-li.js                      |  2 +-
 lib/galaxy/selenium/navigates_galaxy.py                | 10 +++++++---
 lib/galaxy/selenium/navigation.yml                     |  6 +-----
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index aae19514302..59fe2466f95 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -63,7 +63,7 @@ either through the props, and make updates through the events -->
         </div>
 
         <!-- expanded view with editors -->
-        <header v-if="expanded" class="p-2">
+        <header v-if="expanded" class="p-2 details">
             <ClickToEdit
                 v-if="dataset.canEditName"
                 tag-name="h4"
diff --git a/client/src/components/History/CurrentHistoryPanel.vue b/client/src/components/History/CurrentHistoryPanel.vue
index 7490292c41c..6b58fe67d01 100644
--- a/client/src/components/History/CurrentHistoryPanel.vue
+++ b/client/src/components/History/CurrentHistoryPanel.vue
@@ -1,9 +1,9 @@
 <template>
     <CurrentUser class="d-flex flex-column" v-slot="{ user }">
         <UserHistories v-if="user" :user="user" v-slot="{ currentHistory, histories, handlers }">
-            <HistoryPanel v-if="currentHistory" :history="currentHistory" v-on="handlers">
+            <HistoryPanel v-if="currentHistory" :history="currentHistory" v-on="handlers" id="current-history-panel">
                 <template v-slot:nav>
-                    <div id="current-history-panel">
+                    <div>
                         <HistorySelector
                             :histories="histories"
                             :current-history="currentHistory"
diff --git a/client/src/mvc/history/hda-li.js b/client/src/mvc/history/hda-li.js
index d5e13f82810..b70b07be075 100644
--- a/client/src/mvc/history/hda-li.js
+++ b/client/src/mvc/history/hda-li.js
@@ -33,7 +33,7 @@ HDAListItemView.prototype.templates = (() => {
     var titleBarTemplate = (dataset) => `
         <div class="title-bar clear" tabindex="0">
             <span class="state-icon"></span>
-            <div class="title">
+            <div class="title content-title">
                 <span class="hid">${dataset.hid}</span>
                 <span class="name">${_.escape(dataset.name)}</span>
             </div>
diff --git a/client/src/mvc/history/hdca-li.js b/client/src/mvc/history/hdca-li.js
index 6570ae7013e..d48fa1f0f35 100644
--- a/client/src/mvc/history/hdca-li.js
+++ b/client/src/mvc/history/hdca-li.js
@@ -104,7 +104,7 @@ HDCAListItemView.prototype.templates = (() => {
     var titleBarTemplate = (collection) => `
         <div class="title-bar clear" tabindex="0">
             <span class="state-icon"></span>
-            <div class="title">
+            <div class="title content-title">
                 <span class="hid">${collection.hid}</span>
                 <span class="name">${_.escape(collection.name)}</span>
             </div>
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index a9e107aee9f..2809ffc35e9 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -12,6 +12,7 @@ from functools import partial, wraps
 
 import requests
 import yaml
+from selenium.webdriver.common.keys import Keys
 
 from galaxy.util import DEFAULT_SOCKET_TIMEOUT
 from . import sizzle
@@ -299,8 +300,6 @@ class NavigatesGalaxy(HasDriver):
         item = self.components.history_panel.item.selector
         if multi_history_panel:
             item = self.components.multi_history_view.item
-        elif self.is_beta_history():
-            item = self.components.history_panel.beta_item.selector
         return item(
             history_content_type=history_item["history_content_type"],
             id=history_item["id"]
@@ -1618,7 +1617,12 @@ class NavigatesGalaxy(HasDriver):
         editable.wait_for_and_click()
         edit_el = edit.wait_for_and_click()
         if clear_text:
-            edit_el.clear()
+            # previously this was just edit_el.clear() but
+            # .clear() doesn't work with beta history panel
+            action_chains = self.action_chains()
+            for _ in range(40):
+                action_chains.send_keys(Keys.BACKSPACE)
+            action_chains.perform()
         edit_el.send_keys(annotation)
 
         if self.is_beta_history():
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 1d706052580..8140a4715b0 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -111,17 +111,13 @@ history_panel:
     labels:
       new: 'Create New'
 
-  beta_item:
-    selectors:
-      _: ' #${history_content_type}-${id}' 
-
   item:
     selectors:
       # These now appear other places :_( - e.g. in the invocation view so we'll try
       # prefixing ids with #current-history-panel but obviously we need to switch to classes or data
       _: '#current-history-panel #${history_content_type}-${id}'
 
-      title: '${_} .title'
+      title: '${_} .content-title'
       hid: '${_} .hid'
       name: '${_} .name'
       details: '${_} .details'

From fa8e6746f5962f81e22641ae3d3d42a630bad775 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 2 Jul 2021 12:06:36 -0400
Subject: [PATCH 134/452] More progress on sycning histories.

---
 client/src/components/History/CurrentCollection/Details.vue | 2 +-
 client/src/components/History/CurrentCollection/Panel.vue   | 2 +-
 client/src/components/History/CurrentCollection/TopNav.vue  | 4 ++--
 lib/galaxy/selenium/navigates_galaxy.py                     | 2 +-
 lib/galaxy/selenium/navigation.yml                          | 2 +-
 lib/galaxy_test/selenium/test_history_panel_collections.py  | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/client/src/components/History/CurrentCollection/Details.vue b/client/src/components/History/CurrentCollection/Details.vue
index 1d4218184cb..32b4fbb2ec5 100644
--- a/client/src/components/History/CurrentCollection/Details.vue
+++ b/client/src/components/History/CurrentCollection/Details.vue
@@ -1,5 +1,5 @@
 <template>
-    <section class="history-details">
+    <section class="history-details controls">
         <ClickToEdit
             :value="dsc.name"
             v-if="writable"
diff --git a/client/src/components/History/CurrentCollection/Panel.vue b/client/src/components/History/CurrentCollection/Panel.vue
index 7fd5cd1c3ee..298ba97e2be 100644
--- a/client/src/components/History/CurrentCollection/Panel.vue
+++ b/client/src/components/History/CurrentCollection/Panel.vue
@@ -8,7 +8,7 @@
                 :get-item-key="(item) => item.type_id"
                 v-slot="{ isExpanded, setExpanded }"
             >
-                <Layout>
+                <Layout class="dataset-collection-panel">
                     <template v-slot:nav>
                         <TopNav :history="history" :selected-collections="selectedCollections" v-on="$listeners" />
                     </template>
diff --git a/client/src/components/History/CurrentCollection/TopNav.vue b/client/src/components/History/CurrentCollection/TopNav.vue
index 1e9a3090756..85f324d930b 100644
--- a/client/src/components/History/CurrentCollection/TopNav.vue
+++ b/client/src/components/History/CurrentCollection/TopNav.vue
@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div class="navigation">
         <b-dropdown v-if="breadCrumbOptions.length" class="mr-auto" size="sm" text="Go to..." boundary="viewport">
             <b-dropdown-item @click="close"> History: {{ history.name }} </b-dropdown-item>
             <b-dropdown-item v-for="option in breadCrumbOptions" :key="option.key" @click="reselect(option.value)">
@@ -7,7 +7,7 @@
             </b-dropdown-item>
         </b-dropdown>
 
-        <b-button v-else size="sm" class="mr-auto" @click="close"> Back to: {{ history.name }} </b-button>
+        <b-button v-else size="sm" class="mr-auto back" @click="close"> Back to: {{ history.name }} </b-button>
 
         <PriorityMenu :starting-height="27">
             <PriorityMenuItem
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 2809ffc35e9..a588542594a 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -1400,10 +1400,10 @@ class NavigatesGalaxy(HasDriver):
     def history_panel_click_item_title(self, hid, **kwds):
         item_component = self.history_panel_item_component(hid=hid)
         details_component = item_component.details
-        details_displayed = details_component.is_displayed
         item_component.title.wait_for_and_click()
 
         if kwds.get("wait", False):
+            details_displayed = details_component.is_displayed
             if details_displayed:
                 details_component.wait_for_absent_or_hidden()
             else:
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 8140a4715b0..413c90912fc 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -157,7 +157,7 @@ history_panel:
 
   collection_view:
     selectors:
-      _: '.list-panel.dataset-collection-panel'
+      _: '.dataset-collection-panel'
       back: '.navigation .back'
       title: '.dataset-collection-panel .controls .title .editable-text'
       title_input: '.dataset-collection-panel .controls .title input'
diff --git a/lib/galaxy_test/selenium/test_history_panel_collections.py b/lib/galaxy_test/selenium/test_history_panel_collections.py
index a3a2fd9a1a2..b69378e947d 100644
--- a/lib/galaxy_test/selenium/test_history_panel_collections.py
+++ b/lib/galaxy_test/selenium/test_history_panel_collections.py
@@ -158,7 +158,7 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
         collection_view = self._click_and_wait_for_collection_view(collection_hid)
 
         # the space on the end of the parent_selector is important
-        self.tagging_add(["#moo"], parent_selector="#current-history-panel .dataset-collection-panel .controls ")
+        self.tagging_add(["#moo"], parent_selector=".dataset-collection-panel .controls ")
 
         self.screenshot("history_panel_collection_view_add_nametag")
         collection_view.back.wait_for_and_click()

From f7b5ec906f07e37ca8e64b2bebc90d730c4c583d Mon Sep 17 00:00:00 2001
From: Marius van den Beek <m.vandenbeek@gmail.com>
Date: Fri, 16 Jul 2021 17:26:23 +0200
Subject: [PATCH 135/452] Update issue templates

---
 .github/ISSUE_TEMPLATE/bug_report.md | 31 ++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 00000000000..6ae457a2aeb
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Galaxy Version and/or server at which you observed the bug**
+Galaxy Version: (check <galaxy_url>/api/version if you don't know)
+Commit: (run `git rev-parse HEAD` if you run this Galaxy server)
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

From 2cc1d1d274305d7aa67b3772de5183e74db1dc16 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Fri, 16 Jul 2021 15:03:46 +0200
Subject: [PATCH 136/452] add linting for validators

checking for incompatibilities of:

- parameter and validator types
- present attributes and validator type
---
 lib/galaxy/tool_util/linters/inputs.py    | 43 ++++++++++++++++++++++-
 lib/galaxy/tool_util/xsd/galaxy.xsd       |  7 ++--
 lib/galaxy/tools/parameters/validation.py |  2 +-
 test/unit/tool_util/test_tool_linters.py  | 22 ++++++++++++
 4 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tool_util/linters/inputs.py b/lib/galaxy/tool_util/linters/inputs.py
index 2dde74cab8c..dbe1a041363 100644
--- a/lib/galaxy/tool_util/linters/inputs.py
+++ b/lib/galaxy/tool_util/linters/inputs.py
@@ -17,6 +17,33 @@ FILTER_TYPES = [
     'sort_by',
 ]
 
+ATTRIB_VALIDATOR_COMPATIBILITY = {
+    "check": ["metadata"],
+    "expression": ["regex"],
+    "table_name": ["dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "value_in_data_table", "value_not_in_data_table"],
+    "filename": ["dataset_metadata_in_file"],
+    "metadata_name": ["dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "dataset_metadata_in_file"],
+    "metadata_column": ["dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "value_in_data_table", "value_not_in_data_table", "dataset_metadata_in_file options"],
+    "line_startswith": ["dataset_metadata_in_file", "dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "value_in_data_table", "value_not_in_data_table"],
+    "min": ["in_range"],
+    "max": ["in_range"],
+    "exclude_min": ["in_range"],
+    "exclude_max": ["in_range"],
+    "split": ["dataset_metadata_in_file"],
+    "skip": ["metadata"]
+}
+
+PARAMETER_VALIDATOR_TYPE_COMPATIBILITY = {
+    "integer": ["in_range", "expression"],
+    "float": ["in_range", "expression"],
+    "data": ["metadata", "unspecified_build", "dataset_ok_validator", "dataset_metadata_in_range", "dataset_metadata_in_file", "dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "expression"],
+    "data_collection": ["metadata", "unspecified_build", "dataset_ok_validator", "dataset_metadata_in_range", "dataset_metadata_in_file", "dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "expression"],
+    "text": ["regex", "length", "empty_field", "value_in_data_table", "value_not_in_data_table", "expression"],
+    "select": ["no_options", "regex", "length", "empty_field", "value_in_data_table", "value_not_in_data_table", "expression"],
+    "drill_down": ["no_options", "regex", "length", "empty_field", "value_in_data_table", "value_not_in_data_table", "expression"],
+    "data_column": ["no_options", "regex", "length", "empty_field", "value_in_data_table", "value_not_in_data_table", "expression"]
+}
+
 
 def lint_inputs(tool_xml, lint_ctx):
     """Lint parameters in a tool's inputs block."""
@@ -120,9 +147,23 @@ def lint_inputs(tool_xml, lint_ctx):
                     lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with multiple="true"')
                 if string_as_bool(param_attrib.get("optional", "false")):
                     lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with optional="true"')
-
         # TODO: Validate type, much more...
 
+        # validate validators :)
+        validators = param.findall("./validator")
+        for validator in validators:
+            vtype = validator.attrib['type']
+            if vtype == "expression" and validator.text == "":
+                lint_ctx.error(f"Parameter [{param_name}]: expression validator without content")
+            if vtype != "expression" and validator.text != "":
+                lint_ctx.warn(f"Parameter [{param_name}]: '{vtype}' validators are not expected to contain text")
+            if param_type in PARAMETER_VALIDATOR_TYPE_COMPATIBILITY:
+                if vtype not in PARAMETER_VALIDATOR_TYPE_COMPATIBILITY[param_type]:
+                    lint_ctx.error(f"Parameter [{param_name}]: validator with an incompatible type '{vtype}'")
+            for attrib in ATTRIB_VALIDATOR_COMPATIBILITY:
+                if attrib in validator.attrib and vtype not in ATTRIB_VALIDATOR_COMPATIBILITY[attrib]:
+                    lint_ctx.error(f"Parameter [{param_name}]: attribute '{attrib}' is in compatible with validator of type '{vtype}'")
+
     conditional_selects = tool_xml.findall("./inputs//conditional")
     for conditional in conditional_selects:
         conditional_name = conditional.get('name')
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index df5dc269929..29d2bb5c8c4 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -3753,7 +3753,7 @@ specified by the ``skip`` attribute.</xs:documentation>
         <xs:attribute name="table_name" type="xs:string">
           <xs:annotation>
             <xs:documentation xml:lang="en">Tool data table name to check against
-if ``type`` is ``dataset_metadata_in_tool_data``, ``dataset_metadata_not_in_tool_data``, ``value_in_tool_data``, or ``value_not_in_tool_data``. See the documentation for
+if ``type`` is ``dataset_metadata_in_data_table``, ``dataset_metadata_not_in_data_table``, ``value_in_data_table``, or ``value_not_in_data_table``. See the documentation for
 [tool data tables](https://galaxyproject.org/admin/tools/data-tables)
 and [data managers](https://galaxyproject.org/admin/tools/data-managers/) for
 more information.</xs:documentation>
@@ -3775,14 +3775,15 @@ if ``type`` is ``dataset_metadata_in_file``. File should be present Galaxy's
         <xs:attribute name="metadata_column" type="xs:string">
           <xs:annotation>
             <xs:documentation xml:lang="en">Target column for metadata attribute
-in ``dataset_metadata_in_data_table``, ``dataset_metadata_not_in_data_table``, ``value_in_data_table``, ``value__not_in_data_table``, and ``dataset_metadata_in_file`` options.
+in ``dataset_metadata_in_data_table``, ``dataset_metadata_not_in_data_table``, ``value_in_data_table``, ``value_not_in_data_table``, and ``dataset_metadata_in_file`` options.
 This can be an integer index to the column or a column name.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="line_startswith" type="xs:string">
           <xs:annotation>
             <xs:documentation xml:lang="en">Used to indicate lines in the file
-being used for validation start with a this attribute value.</xs:documentation>
+being used for validation start with a this attribute value.
+For use with validators of type ``dataset_metadata_in_file``, ``dataset_metadata_in_data_table``, ``dataset_metadata_not_in_data_table``, ``value_in_data_table``, ``value_not_in_data_tabl``</xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="min" type="xs:decimal">
diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py
index de5184205f3..a5212ac8748 100644
--- a/lib/galaxy/tools/parameters/validation.py
+++ b/lib/galaxy/tools/parameters/validation.py
@@ -612,7 +612,7 @@ validator_types = dict(
     dataset_metadata_not_in_data_table=MetadataNotInDataTableColumnValidator,
     dataset_metadata_in_range=MetadataInRangeValidator,
     value_in_data_table=ValueInDataTableColumnValidator,
-    value_not_in_data_table=ValueInDataTableColumnValidator,
+    value_not_in_data_table=ValueNotInDataTableColumnValidator,
     dataset_ok_validator=DatasetOkValidator,
 )
 
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index 31b2e082ba5..0e3c708848e 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -121,6 +121,19 @@ WHITESPACE_IN_VERSIONS_AND_NAMES = """
 </tool>
 """
 
+VALIDATOR_INCOMPATIBILITIES = """
+<tool name="BWA Mapper" id="bwa" version="1.0.1" is_multi_byte="true" display_interface="true" require_login="true" hidden="true">
+    <description>The BWA Mapper</description>
+    <version_command interpreter="python">bwa.py --version</version_command>
+    <inputs>
+        <param name="param_name" type="text">
+            <validator type="in_range">TEXT</validator>
+            <validator type="regex" filename="blah"/>
+        </param>
+    </inputs>
+</tool>
+"""
+
 TESTS = [
     (
         NO_SECTIONS_XML, inputs.lint_inputs,
@@ -179,6 +192,14 @@ TESTS = [
             and "Tool ID contains whitespace - this is discouraged: [bwa tool]."
             and len(x.warn_messages) == 4 and len(x.error_messages) == 0
     ),
+    (
+        VALIDATOR_INCOMPATIBILITIES, inputs.lint_inputs,
+        lambda x:
+            "Parameter [param_name]: validator with an incompatible type 'in_range'" in x.error_messages
+            and "Parameter [param_name]: attribute 'filename' is in compatible with validator of type 'regex'" in x.error_messages
+            and "Parameter [param_name]: 'in_range' validators are not expected to contain text" in x.warn_messages
+            and len(x.warn_messages) == 0 and len(x.error_messages) == 2
+    ),
 ]
 
 TEST_IDS = [
@@ -189,6 +210,7 @@ TEST_IDS = [
     'select deprecations',
     'select option definitions',
     'hazardous whitespace',
+    'validator imcompatibilities'
 ]
 
 

From 33ce8a78e9f80b22793d2a7ce3c822cd5498929c Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Fri, 16 Jul 2021 17:55:23 +0100
Subject: [PATCH 137/452] Move PR template to `.github`

---
 pull_request_template.md => .github/pull_request_template.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename pull_request_template.md => .github/pull_request_template.md (100%)

diff --git a/pull_request_template.md b/.github/pull_request_template.md
similarity index 100%
rename from pull_request_template.md
rename to .github/pull_request_template.md

From 5d3548a63812c3f2ef1f960af4abd726ab9bb9f8 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Fri, 16 Jul 2021 20:19:19 +0100
Subject: [PATCH 138/452] save current page, intoduce link to the page, manage
 on new window

---
 .../Libraries/LibraryFolder/LibraryFolder.vue | 23 +++++++++++++++----
 .../LibraryFolderDatasetPermissions.vue       |  2 +-
 .../Libraries/LibraryFolderRouter.js          | 11 +++++++--
 3 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
index 03181ab7812..5c699be7722 100644
--- a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
+++ b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
@@ -220,10 +220,10 @@
                         </b-button>
                         <b-button
                             v-if="row.item.can_manage && !row.item.deleted"
-                            @click="navigateToPermission(row.item)"
                             size="sm"
                             class="lib-btn permission_lib_btn"
                             :title="`Permissions of ${row.item.name}`"
+                            :to="{ path: `${navigateToPermission(row.item)}` }"
                         >
                             <font-awesome-icon icon="users" />
                             Manage
@@ -241,10 +241,17 @@
                     </div>
                 </template>
             </b-table>
+            <!-- hide pagination if the table is loading-->
             <b-container>
-                <b-row class="justify-content-md-center">
+                <b-row align-v="center" class="justify-content-md-center">
                     <b-col md="auto">
+                        <div v-if="isBusy">
+                            <b-spinner small type="grow"></b-spinner>
+                            <b-spinner small type="grow"></b-spinner>
+                            <b-spinner small type="grow"></b-spinner>
+                        </div>
                         <b-pagination
+                            v-else
                             v-model="currentPage"
                             :total-rows="total_rows"
                             :per-page="perPage"
@@ -252,6 +259,7 @@
                         >
                         </b-pagination>
                     </b-col>
+
                     <b-col cols="1.5">
                         <table>
                             <tr>
@@ -304,7 +312,6 @@ function initialFolderState() {
         include_deleted: false,
         search_text: "",
         isAllSelectedMode: false,
-        currentPage: 1,
     };
 }
 export default {
@@ -313,6 +320,11 @@ export default {
             type: String,
             required: true,
         },
+        page: {
+            type: Number,
+            default: 1,
+            required: false,
+        },
     },
     components: {
         FolderTopBar,
@@ -323,6 +335,7 @@ export default {
         return {
             ...initialFolderState(),
             ...{
+                currentPage: this.page,
                 current_folder_id: null,
                 error: null,
                 isBusy: false,
@@ -486,9 +499,9 @@ export default {
         },
         navigateToPermission(element) {
             if (element.type === "file") {
-                this.$router.push({ path: `${this.folder_id}/dataset/${element.id}/permissions` });
+                return `/folders/${this.folder_id}/dataset/${element.id}/permissions`;
             } else if (element.type === "folder") {
-                this.$router.push({ path: `${element.id}/permissions` });
+                return `/folders/${element.id}/permissions`;
             }
         },
         getMessage(element) {
diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue
index 8c843e06cae..6137a6f1f40 100644
--- a/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue
+++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue
@@ -9,7 +9,7 @@
                     <template v-for="path_item in this.dataset.full_path">
                         <b-breadcrumb-item
                             :key="path_item[0]"
-                            :to="{ path: `/folders/${path_item[0]}` }"
+                            :to="{ path: `/folders/${path_item[0]}/` }"
                             :active="path_item[0] === dataset_id"
                             href="#"
                             >{{ path_item[1] }}</b-breadcrumb-item
diff --git a/client/src/components/Libraries/LibraryFolderRouter.js b/client/src/components/Libraries/LibraryFolderRouter.js
index 38c2dec8863..c2164dc6e0f 100644
--- a/client/src/components/Libraries/LibraryFolderRouter.js
+++ b/client/src/components/Libraries/LibraryFolderRouter.js
@@ -25,10 +25,17 @@ export default new VueRouter({
             props: true,
         },
         {
-            path: "/folders/:folder_id",
+            path: "/folders/:folder_id/(page/)?:page?",
+            // path: "/folders/:folder_id(/?)(page/)?:page?",
             name: "LibraryFolder",
             component: LibraryFolder,
-            props: true,
+            props(route) {
+                const props = { ...route.params };
+                if (props.page) {
+                    props.page = +props.page;
+                }
+                return props;
+            },
         },
         {
             path: "/folders/:folder_id/permissions",

From 42c6616e436e7e6103e9bca86d770e9c211c87d9 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Fri, 16 Jul 2021 19:16:14 +0200
Subject: [PATCH 139/452] more linting of validators

some fixed in xsd
---
 lib/galaxy/tool_util/linters/inputs.py    | 24 +++++++++++++++--------
 lib/galaxy/tool_util/xsd/galaxy.xsd       |  8 ++++----
 lib/galaxy/tools/parameters/validation.py |  2 +-
 test/unit/tool_util/test_tool_linters.py  | 10 ++++++----
 4 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/lib/galaxy/tool_util/linters/inputs.py b/lib/galaxy/tool_util/linters/inputs.py
index dbe1a041363..f8a843fb11a 100644
--- a/lib/galaxy/tool_util/linters/inputs.py
+++ b/lib/galaxy/tool_util/linters/inputs.py
@@ -25,8 +25,8 @@ ATTRIB_VALIDATOR_COMPATIBILITY = {
     "metadata_name": ["dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "dataset_metadata_in_file"],
     "metadata_column": ["dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "value_in_data_table", "value_not_in_data_table", "dataset_metadata_in_file options"],
     "line_startswith": ["dataset_metadata_in_file", "dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table", "value_in_data_table", "value_not_in_data_table"],
-    "min": ["in_range"],
-    "max": ["in_range"],
+    "min": ["in_range", "length"],
+    "max": ["in_range", "length"],
     "exclude_min": ["in_range"],
     "exclude_max": ["in_range"],
     "split": ["dataset_metadata_in_file"],
@@ -149,20 +149,28 @@ def lint_inputs(tool_xml, lint_ctx):
                     lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with optional="true"')
         # TODO: Validate type, much more...
 
-        # validate validators :)
+        # lint validators
         validators = param.findall("./validator")
         for validator in validators:
             vtype = validator.attrib['type']
-            if vtype == "expression" and validator.text == "":
-                lint_ctx.error(f"Parameter [{param_name}]: expression validator without content")
-            if vtype != "expression" and validator.text != "":
-                lint_ctx.warn(f"Parameter [{param_name}]: '{vtype}' validators are not expected to contain text")
             if param_type in PARAMETER_VALIDATOR_TYPE_COMPATIBILITY:
                 if vtype not in PARAMETER_VALIDATOR_TYPE_COMPATIBILITY[param_type]:
                     lint_ctx.error(f"Parameter [{param_name}]: validator with an incompatible type '{vtype}'")
             for attrib in ATTRIB_VALIDATOR_COMPATIBILITY:
                 if attrib in validator.attrib and vtype not in ATTRIB_VALIDATOR_COMPATIBILITY[attrib]:
-                    lint_ctx.error(f"Parameter [{param_name}]: attribute '{attrib}' is in compatible with validator of type '{vtype}'")
+                    lint_ctx.error(f"Parameter [{param_name}]: attribute '{attrib}' is incompatible with validator of type '{vtype}'")
+            if vtype == "expression" and validator.text is None:
+                lint_ctx.error(f"Parameter [{param_name}]: expression validator without content")
+            if vtype != "expression" and validator.text is not None:
+                lint_ctx.warn(f"Parameter [{param_name}]: '{vtype}' validators are not expected to contain text (found '{validator.text}')")
+            if vtype == "regex" and "expression" not in validator.attrib:
+                lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define an 'expression' attribute")
+            if vtype in ["in_range", "length", "dataset_metadata_in_range"] and ("min" not in validator.attrib or "max" not in validator.attrib):
+                lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'min' or 'max' attribute(s)")
+            if vtype in ["metadata"] and ("check" not in validator.attrib or "skip" not in validator.attrib):
+                lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'check' or 'skip' attribute(s)")
+            if vtype in ["value_in_data_table", "value_not_in_data_table", "dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table"] and "table_name" not in validator.attrib:
+                lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'table_name' attribute")
 
     conditional_selects = tool_xml.findall("./inputs//conditional")
     for conditional in conditional_selects:
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 29d2bb5c8c4..578e6067ef1 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -3789,25 +3789,25 @@ For use with validators of type ``dataset_metadata_in_file``, ``dataset_metadata
         <xs:attribute name="min" type="xs:decimal">
           <xs:annotation>
             <xs:documentation xml:lang="en">When the ``type`` attribute value is
-``in_range`` - this is the minimum number allowed.</xs:documentation>
+``in_range``, ``length``, or ``dataset_metadata_in_range`` - this is the minimum number allowed.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="max" type="xs:decimal">
           <xs:annotation>
             <xs:documentation xml:lang="en">When the ``type`` attribute value is
-``in_range`` - this is the maximum number allowed.</xs:documentation>
+``in_range``, ``length``, or ``dataset_metadata_in_range`` - this is the maximum number allowed.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="exclude_min" type="xs:boolean" default="false">
           <xs:annotation>
             <xs:documentation xml:lang="en">When the ``type`` attribute value is
-``in_range`` - this boolean indicates if the ``min`` value is allowed.</xs:documentation>
+``in_range``, ``length``, or ``dataset_metadata_in_range`` - this boolean indicates if the ``min`` value is allowed.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="exclude_max" type="xs:boolean" default="false">
           <xs:annotation>
             <xs:documentation xml:lang="en">When the ``type`` attribute value is
-``in_range`` - this boolean indicates if the ``max`` value is allowed.</xs:documentation>
+``in_range``, ``length``, or ``dataset_metadata_in_range`` - this boolean indicates if the ``max`` value is allowed.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="split" type="xs:string">
diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py
index a5212ac8748..9b4d86e226d 100644
--- a/lib/galaxy/tools/parameters/validation.py
+++ b/lib/galaxy/tools/parameters/validation.py
@@ -466,7 +466,7 @@ class ValueInDataTableColumnValidator(Validator):
         if not value:
             return
         if not self._tool_data_table.is_current_version(self._data_table_content_version):
-            log.debug('MetadataInDataTableColumnValidator values are out of sync with data table (%s), updating validator.', self._tool_data_table.name)
+            log.debug('ValueInDataTableColumnValidator: values are out of sync with data table (%s), updating validator.', self._tool_data_table.name)
             self._load_values()
         if value in self.valid_values:
             return
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index 0e3c708848e..5323ea820a8 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -195,10 +195,12 @@ TESTS = [
     (
         VALIDATOR_INCOMPATIBILITIES, inputs.lint_inputs,
         lambda x:
-            "Parameter [param_name]: validator with an incompatible type 'in_range'" in x.error_messages
-            and "Parameter [param_name]: attribute 'filename' is in compatible with validator of type 'regex'" in x.error_messages
-            and "Parameter [param_name]: 'in_range' validators are not expected to contain text" in x.warn_messages
-            and len(x.warn_messages) == 0 and len(x.error_messages) == 2
+            "Parameter [param_name]: 'in_range' validators are not expected to contain text (found 'TEXT')" in x.warn_messages
+            and "Parameter [param_name]: validator with an incompatible type 'in_range'" in x.error_messages
+            and "Parameter [param_name]: 'in_range' validators need to define the 'min' or 'max' attribute(s)" in x.error_messages
+            and "Parameter [param_name]: attribute 'filename' is incompatible with validator of type 'regex'" in x.error_messages
+            and "Parameter [param_name]: 'regex' validators need to define an 'expression' attribute" in x.error_messages
+            and len(x.warn_messages) == 1 and len(x.error_messages) == 4
     ),
 ]
 

From d7cf4acd7c19a6b595d3413c32fb6844ba9a26f2 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 16 Jul 2021 16:32:56 -0400
Subject: [PATCH 140/452] Implement galaxy.json dynamic file collection in
 Pulsar.

---
 lib/galaxy/dependencies/dev-requirements.txt    | 2 +-
 lib/galaxy/dependencies/pinned-requirements.txt | 2 +-
 lib/galaxy/jobs/runners/pulsar.py               | 7 +++++++
 pyproject.toml                                  | 2 +-
 test/integration/test_pulsar_embedded.py        | 1 +
 5 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index 5290b553919..d1346637796 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.8
+pulsar-galaxy-lib==0.14.10
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index 87a30062242..b3db1a5e313 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.8
+pulsar-galaxy-lib==0.14.10
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index fbbf2148a0f..bf7c9d9a2a3 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -767,6 +767,12 @@ class PulsarJobRunner(AsynchronousJobRunner):
             # otherwise collect everything we might need
             dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:]
             dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns)
+        tool = job_wrapper.tool
+        tool_provided_metadata_file_path = tool.provided_metadata_file
+        tool_provided_metadata_style = tool.provided_metadata_style
+        dynamic_file_sources = [
+            {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"}
+        ]
         client_outputs = ClientOutputs(
             working_directory=job_wrapper.tool_working_directory,
             metadata_directory=metadata_directory,
@@ -774,6 +780,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
             output_files=output_files,
             version_file=job_wrapper.get_version_string_path(),
             dynamic_outputs=dynamic_outputs,
+            dynamic_file_sources=dynamic_file_sources,
         )
         return client_outputs
 
diff --git a/pyproject.toml b/pyproject.toml
index 70b0bf64603..1019ed57a0e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ paramiko = "*"
 Parsley = "*"
 Paste = "*"
 psutil = "*"
-pulsar-galaxy-lib = ">=0.14.5"
+pulsar-galaxy-lib = ">=0.14.9"
 pycryptodome = "*"
 pykwalify = "*"
 pyparsing = "*"
diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py
index 282b5fa64e1..db9bd8baedf 100644
--- a/test/integration/test_pulsar_embedded.py
+++ b/test/integration/test_pulsar_embedded.py
@@ -21,6 +21,7 @@ class EmbeddedPulsarIntegrationInstance(integration_util.IntegrationInstance):
 instance = integration_util.integration_module_instance(EmbeddedPulsarIntegrationInstance)
 
 test_tools = integration_util.integration_tool_runner([
+    "collection_creates_dynamic_nested_from_json",
     "composite",
     "simple_constructs",
     "multi_data_param",

From 4d46efb4592dd4740f85cd2b5d9618f663d15fe2 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 16 Jul 2021 16:32:56 -0400
Subject: [PATCH 141/452] Implement galaxy.json dynamic file collection in
 Pulsar.

---
 lib/galaxy/dependencies/dev-requirements.txt    | 2 +-
 lib/galaxy/dependencies/pinned-requirements.txt | 2 +-
 lib/galaxy/jobs/runners/pulsar.py               | 7 +++++++
 pyproject.toml                                  | 2 +-
 test/integration/test_pulsar_embedded.py        | 1 +
 5 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index 515f44c07b2..fa0c98b11c3 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -144,7 +144,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.8
+pulsar-galaxy-lib==0.14.10
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index d134630e25f..bd4d0587550 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -127,7 +127,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.8
+pulsar-galaxy-lib==0.14.10
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"
diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index 326c36ebe46..5d7dfa19830 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -767,6 +767,12 @@ class PulsarJobRunner(AsynchronousJobRunner):
             # otherwise collect everything we might need
             dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:]
             dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns)
+        tool = job_wrapper.tool
+        tool_provided_metadata_file_path = tool.provided_metadata_file
+        tool_provided_metadata_style = tool.provided_metadata_style
+        dynamic_file_sources = [
+            {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"}
+        ]
         client_outputs = ClientOutputs(
             working_directory=job_wrapper.tool_working_directory,
             metadata_directory=metadata_directory,
@@ -774,6 +780,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
             output_files=output_files,
             version_file=job_wrapper.get_version_string_path(),
             dynamic_outputs=dynamic_outputs,
+            dynamic_file_sources=dynamic_file_sources,
         )
         return client_outputs
 
diff --git a/pyproject.toml b/pyproject.toml
index 83c4fa50742..e85ffdbee53 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ paramiko = "*"
 Parsley = "*"
 Paste = "*"
 psutil = "*"
-pulsar-galaxy-lib = ">=0.14.5"
+pulsar-galaxy-lib = ">=0.14.9"
 pycryptodome = "*"
 pykwalify = "*"
 pyparsing = "*"
diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py
index 282b5fa64e1..db9bd8baedf 100644
--- a/test/integration/test_pulsar_embedded.py
+++ b/test/integration/test_pulsar_embedded.py
@@ -21,6 +21,7 @@ class EmbeddedPulsarIntegrationInstance(integration_util.IntegrationInstance):
 instance = integration_util.integration_module_instance(EmbeddedPulsarIntegrationInstance)
 
 test_tools = integration_util.integration_tool_runner([
+    "collection_creates_dynamic_nested_from_json",
     "composite",
     "simple_constructs",
     "multi_data_param",

From fc575dd3d8c276244b21ee7141514b865f102fcb Mon Sep 17 00:00:00 2001
From: Kjetil Klepper <kjetil.klepper@ntnu.no>
Date: Mon, 19 Jul 2021 12:57:40 +0200
Subject: [PATCH 142/452] Added a necessary check to install the 'fs.sshfs'
 package if a data source of type 'ssh' is configured in
 'file_sources_conf.yml'

---
 lib/galaxy/dependencies/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py
index bf48e79c09c..aab3f19988f 100644
--- a/lib/galaxy/dependencies/__init__.py
+++ b/lib/galaxy/dependencies/__init__.py
@@ -227,6 +227,9 @@ class ConditionalDependencies:
         # pyfilesystem plugin access to terra on anvil
         return 'anvil' in self.file_sources
 
+    def check_fs_sshfs(self):
+        return 'ssh' in self.file_sources
+
     def check_s3fs(self):
         # use s3fs directly (skipping pyfilesystem) for direct access to more options
         return 's3fs' in self.file_sources

From b8269be30c010a3145be2fcb48cae9fe2eb21a4f Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 14:53:00 +0200
Subject: [PATCH 143/452] add redirect in order to handle :folder_id and
 :folder/

---
 client/src/components/Libraries/LibraryFolderRouter.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Libraries/LibraryFolderRouter.js b/client/src/components/Libraries/LibraryFolderRouter.js
index c2164dc6e0f..5eae95c1b98 100644
--- a/client/src/components/Libraries/LibraryFolderRouter.js
+++ b/client/src/components/Libraries/LibraryFolderRouter.js
@@ -24,9 +24,10 @@ export default new VueRouter({
             component: LibraryPermissions,
             props: true,
         },
+        // redirect to the 1st page
+        { path: "/folders/:folder_id", redirect: "/folders/:folder_id/page/1" },
         {
-            path: "/folders/:folder_id/(page/)?:page?",
-            // path: "/folders/:folder_id(/?)(page/)?:page?",
+            path: "/folders/:folder_id/page/:page?",
             name: "LibraryFolder",
             component: LibraryFolder,
             props(route) {

From a0b71d136d9cd93e68c18c22e6113fafd8f66130 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 16:23:45 +0200
Subject: [PATCH 144/452] dynamically change route and save page

---
 .../Libraries/LibraryFolder/LibraryFolder.vue | 20 +++++++++----------
 .../Libraries/LibraryFolderRouter.js          |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
index 5c699be7722..94e57224156 100644
--- a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
+++ b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
@@ -252,9 +252,10 @@
                         </div>
                         <b-pagination
                             v-else
-                            v-model="currentPage"
+                            :value="currentPage"
                             :total-rows="total_rows"
                             :per-page="perPage"
+                            @input="changePage"
                             aria-controls="folder_list_body"
                         >
                         </b-pagination>
@@ -335,7 +336,7 @@ export default {
         return {
             ...initialFolderState(),
             ...{
-                currentPage: this.page,
+                currentPage: null,
                 current_folder_id: null,
                 error: null,
                 isBusy: false,
@@ -351,11 +352,12 @@ export default {
     },
     created() {
         this.services = new Services({ root: this.root });
-        this.initFolder(this.folder_id);
+        this.getFolder(this.folder_id, this.page);
     },
     methods: {
-        initFolder(folder_id) {
+        getFolder(folder_id, page) {
             this.current_folder_id = folder_id;
+            this.currentPage = page;
             this.resetData();
             this.fetchFolderContents(this.include_deleted);
         },
@@ -594,6 +596,9 @@ export default {
                 );
             }
         },
+        changePage(page) {
+            this.$router.push({ name: `LibraryFolder`, params: { folder_id: this.folder_id, page: page } });
+        },
 
         /*
          Former Backbone code, adopted to work with Vue
@@ -637,11 +642,6 @@ export default {
         },
     },
     watch: {
-        currentPage: {
-            handler: function (value) {
-                this.fetchFolderContents(this.include_deleted);
-            },
-        },
         perPage: {
             handler: function (value) {
                 this.fetchFolderContents(this.include_deleted);
@@ -649,7 +649,7 @@ export default {
         },
     },
     beforeRouteUpdate(to, from, next) {
-        this.initFolder(to.params.folder_id);
+        this.getFolder(to.params.folder_id, to.params.page);
         next();
     },
 };
diff --git a/client/src/components/Libraries/LibraryFolderRouter.js b/client/src/components/Libraries/LibraryFolderRouter.js
index 5eae95c1b98..40c7da11ac7 100644
--- a/client/src/components/Libraries/LibraryFolderRouter.js
+++ b/client/src/components/Libraries/LibraryFolderRouter.js
@@ -27,7 +27,7 @@ export default new VueRouter({
         // redirect to the 1st page
         { path: "/folders/:folder_id", redirect: "/folders/:folder_id/page/1" },
         {
-            path: "/folders/:folder_id/page/:page?",
+            path: "/folders/:folder_id/page/:page",
             name: "LibraryFolder",
             component: LibraryFolder,
             props(route) {

From 3a99a136768f4c0206a0b1096a0b1c97c0aa7a17 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 16:34:35 +0200
Subject: [PATCH 145/452] remove separate 'back' button in permissions

---
 .../LibraryFolderPermissions.vue                      |  2 +-
 .../LibraryPermissions/LibraryPermissions.vue         |  2 +-
 .../LibraryPermissions/PermissionsHeader.vue          | 11 +----------
 3 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderPermissions.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderPermissions.vue
index a7d19e87b3e..d4b8eaaf847 100644
--- a/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderPermissions.vue
+++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderPermissions.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <PermissionsHeader v-if="folder" :name="folder.name" :path="`/folders/${this.folder.parent_id}`" />
+        <PermissionsHeader v-if="folder" :name="folder.name" />
         <b-container fluid>
             <div class="dataset_table">
                 <h2 class="text-center">Folder permissions</h2>
diff --git a/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue b/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue
index fba18bdb0c9..50bbf13a59a 100644
--- a/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue
+++ b/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <PermissionsHeader v-if="library" :name="library.name" path="/" />
+        <PermissionsHeader v-if="library" :name="library.name"/>
         <h2 class="text-center">Library permissions</h2>
         <PermissionsInputField
             v-if="access_library_role_list"
diff --git a/client/src/components/Libraries/LibraryPermissions/PermissionsHeader.vue b/client/src/components/Libraries/LibraryPermissions/PermissionsHeader.vue
index b07d369bb93..8b27506e3c6 100644
--- a/client/src/components/Libraries/LibraryPermissions/PermissionsHeader.vue
+++ b/client/src/components/Libraries/LibraryPermissions/PermissionsHeader.vue
@@ -1,9 +1,6 @@
 <template>
     <div>
         <div v-if="name">
-            <b-button variant="primary" title="back to parent folder" :to="{ path: `${this.path}` }">
-                <font-awesome-icon icon="angle-double-left" />
-            </b-button>
             <div>
                 <div class="header text-center">{{ name }}</div>
             </div>
@@ -15,8 +12,7 @@
 
 <script>
 import { getGalaxyInstance } from "app";
-import LibraryPermissionsWarning from "components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryPermissionsWarning.vue";
-import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import LibraryPermissionsWarning from "components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryPermissionsWarning";
 
 export default {
     props: {
@@ -24,14 +20,9 @@ export default {
             type: String,
             required: true,
         },
-        path: {
-            type: String,
-            required: true,
-        },
     },
     components: {
         LibraryPermissionsWarning,
-        FontAwesomeIcon,
     },
     data() {
         return {

From 1ea47aee5702ec31c563bcdeb1202810c0985e45 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 16:54:18 +0200
Subject: [PATCH 146/452] redo typo

---
 .../LibraryFolderDatasetPermissions.vue                         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue
index 6137a6f1f40..8c843e06cae 100644
--- a/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue
+++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderPermissions/LibraryFolderDatasetPermissions.vue
@@ -9,7 +9,7 @@
                     <template v-for="path_item in this.dataset.full_path">
                         <b-breadcrumb-item
                             :key="path_item[0]"
-                            :to="{ path: `/folders/${path_item[0]}/` }"
+                            :to="{ path: `/folders/${path_item[0]}` }"
                             :active="path_item[0] === dataset_id"
                             href="#"
                             >{{ path_item[1] }}</b-breadcrumb-item

From 1cc033ee29001afb5206a65747c9e3c00ece71c2 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 17:05:38 +0200
Subject: [PATCH 147/452] lint

---
 .../Libraries/LibraryPermissions/LibraryPermissions.vue         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue b/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue
index 50bbf13a59a..1d3f2511297 100644
--- a/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue
+++ b/client/src/components/Libraries/LibraryPermissions/LibraryPermissions.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <PermissionsHeader v-if="library" :name="library.name"/>
+        <PermissionsHeader v-if="library" :name="library.name" />
         <h2 class="text-center">Library permissions</h2>
         <PermissionsInputField
             v-if="access_library_role_list"

From 22cc496afa6d125c1cfa5f47e3c1971cdb8cf868 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 18:13:51 +0200
Subject: [PATCH 148/452] cast page and page_limit to int

---
 lib/galaxy/model/security.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/galaxy/model/security.py b/lib/galaxy/model/security.py
index ddad0310c0e..dd60593feea 100644
--- a/lib/galaxy/model/security.py
+++ b/lib/galaxy/model/security.py
@@ -108,6 +108,10 @@ class GalaxyRBACAgent(RBACAgent):
         else:
             search_query = None
         # Limit the query only to get the page needed
+        if isinstance(page, str):
+            page = int(page)
+        if isinstance(page_limit, str):
+            page_limit = int(page_limit)
         if page is not None and page_limit is not None:
             limit = page * page_limit
         else:

From 1f279b049167928a6ec0bd77f4934dabe2218bbf Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Mon, 19 Jul 2021 18:55:02 +0200
Subject: [PATCH 149/452] fix selenium test

---
 lib/galaxy/selenium/navigation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 46a476e85a0..8c7b6f29617 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -668,7 +668,7 @@ libraries:
       import_dir_btn:
         type: xpath
         selector: '//button[contains(text(), "Import")]'
-      manage_dataset_permissions_btn: 'button[title="Permissions of ${name}"]'
+      manage_dataset_permissions_btn: 'a[title="Permissions of ${name}"]'
       make_private_btn: '#make-private'
       access_dataset_roles: '.access_dataset_roles .multiselect__tag span'
       private_dataset_icon: '.fa-key'

From f24331d2f426d9144980a5db60d0f9591c3fade1 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Tue, 20 Jul 2021 09:33:07 +0200
Subject: [PATCH 150/452] fix edta metadata setting

fix for:

```
UnboundLocalError: local variable 'tpe' referenced before assignment
```

was discovered in the tests running here: https://github.com/galaxyproject/galaxy/pull/11043
---
 lib/galaxy/datatypes/proteomics.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py
index 3ace82f15b2..b4deb40e34d 100644
--- a/lib/galaxy/datatypes/proteomics.py
+++ b/lib/galaxy/datatypes/proteomics.py
@@ -526,6 +526,7 @@ class Edta(TabularData):
     def set_meta(self, dataset, **kwd):
         data_lines = 0
         delim = None
+        tpe = None
         if dataset.has_data():
             with open(dataset.file_name) as dtafile:
                 for idx, line in enumerate(dtafile):
@@ -550,7 +551,7 @@ class Edta(TabularData):
         dataset.metadata.data_lines = data_lines
         dataset.metadata.comment_lines = 0
         dataset.metadata.columns = len(dataset.metadata.column_names)
-        if tpe > 0:
+        if tpe in not None and tpe > 0:
             dataset.metadata.comment_lines += 1
             dataset.metadata.data_lines -= 1
 

From 4cc6fe6e1c6a0fd086807cb953ec769c56d87357 Mon Sep 17 00:00:00 2001
From: M Bernt <m.bernt@ufz.de>
Date: Tue, 20 Jul 2021 09:59:00 +0200
Subject: [PATCH 151/452] fix typo

Co-authored-by: Marius van den Beek <m.vandenbeek@gmail.com>
---
 lib/galaxy/datatypes/proteomics.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py
index b4deb40e34d..0b4d6b49947 100644
--- a/lib/galaxy/datatypes/proteomics.py
+++ b/lib/galaxy/datatypes/proteomics.py
@@ -551,7 +551,7 @@ class Edta(TabularData):
         dataset.metadata.data_lines = data_lines
         dataset.metadata.comment_lines = 0
         dataset.metadata.columns = len(dataset.metadata.column_names)
-        if tpe in not None and tpe > 0:
+        if tpe is not None and tpe > 0:
             dataset.metadata.comment_lines += 1
             dataset.metadata.data_lines -= 1
 

From c052ee503426ab40e2f7d0adc09fe3ad8f66d6e8 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 20 Jul 2021 11:11:54 +0200
Subject: [PATCH 152/452] Improve handling of unexpected values in
 is_empty_file

Should help with https://github.com/galaxyproject/galaxy/issues/12274
and display when submitting a problematic job.
---
 lib/galaxy/tools/parameters/basic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index cf8653c162d..8dcb0272593 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -1378,8 +1378,14 @@ class ColumnListParameter(SelectToolParameter):
             # Use representative dataset if a dataset collection is parsed
             if isinstance(dataset, trans.app.model.HistoryDatasetCollectionAssociation):
                 dataset = dataset.to_hda_representative()
-            if is_runtime_value(dataset) or not dataset.has_data():
+            if isinstance(dataset, trans.app.model.DatasetInstance):
+                return not dataset.has_data()
+            if is_runtime_value(dataset):
                 return True
+            else:
+                msg = f"Dataset '{dataset}' for data_ref attribute '{self.data_ref}' of parameter '{self.name}' is not a DatasetInstance"
+                log.debug(msg, exc_info=True)
+                raise ParameterValueError(msg, self.name)
         return False
 
     def get_dependencies(self):

From 9b463ac1f0d3c96f444e20c1429a0eab1b1ab3a5 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Tue, 20 Jul 2021 14:41:48 +0200
Subject: [PATCH 153/452] move cast to upper function

---
 lib/galaxy/managers/folders.py | 6 +++++-
 lib/galaxy/model/security.py   | 4 ----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/managers/folders.py b/lib/galaxy/managers/folders.py
index 9ae332192cb..94ae8ddcf2f 100644
--- a/lib/galaxy/managers/folders.py
+++ b/lib/galaxy/managers/folders.py
@@ -405,7 +405,11 @@ class FoldersService:
         is_admin = trans.user_is_admin
         decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
         folder = self.folder_manager.get(trans, decoded_folder_id)
-
+        if isinstance(page, str):
+            page = int(page)
+        if isinstance(page_limit, str):
+            page_limit = int(page_limit)
+            
         if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, folder)):
             raise InsufficientPermissionsException('You do not have proper permission to access permissions of this folder.')
 
diff --git a/lib/galaxy/model/security.py b/lib/galaxy/model/security.py
index dd60593feea..ddad0310c0e 100644
--- a/lib/galaxy/model/security.py
+++ b/lib/galaxy/model/security.py
@@ -108,10 +108,6 @@ class GalaxyRBACAgent(RBACAgent):
         else:
             search_query = None
         # Limit the query only to get the page needed
-        if isinstance(page, str):
-            page = int(page)
-        if isinstance(page_limit, str):
-            page_limit = int(page_limit)
         if page is not None and page_limit is not None:
             limit = page * page_limit
         else:

From 44c096ad7b11b43bc48fe87343ebed55eb818f92 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 20 Jul 2021 15:26:30 +0100
Subject: [PATCH 154/452] Raise exception if GET request returned an
 unsuccessful status code

---
 lib/galaxy/tool_util/deps/mulled/mulled_search.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_search.py b/lib/galaxy/tool_util/deps/mulled/mulled_search.py
index ce0f9e14408..90aabe0d843 100755
--- a/lib/galaxy/tool_util/deps/mulled/mulled_search.py
+++ b/lib/galaxy/tool_util/deps/mulled/mulled_search.py
@@ -145,8 +145,9 @@ class GitHubSearch():
         Takes search_string variable and return results from the bioconda-recipes github repository in JSON format
         """
         response = requests.get(
-            f"https://api.github.com/search/code?q={search_string}+in:path+repo:bioconda/bioconda-recipes+path:recipes", timeout=MULLED_SOCKET_TIMEOUT).json()
-        return response
+            f"https://api.github.com/search/code?q={search_string}+in:path+repo:bioconda/bioconda-recipes+path:recipes", timeout=MULLED_SOCKET_TIMEOUT)
+        response.raise_for_status()
+        return response.json()
 
     def process_json(self, json, search_string):
         """

From 947cd0bef03d2cfba20a2d60cff5d080148d2060 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 20 Jul 2021 15:29:07 +0100
Subject: [PATCH 155/452] Don't use standard library name as variable name

---
 lib/galaxy/tool_util/deps/mulled/mulled_search.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_search.py b/lib/galaxy/tool_util/deps/mulled/mulled_search.py
index 90aabe0d843..b02d8296331 100755
--- a/lib/galaxy/tool_util/deps/mulled/mulled_search.py
+++ b/lib/galaxy/tool_util/deps/mulled/mulled_search.py
@@ -149,17 +149,12 @@ class GitHubSearch():
         response.raise_for_status()
         return response.json()
 
-    def process_json(self, json, search_string):
+    def process_json(self, json_response, search_string):
         """
         Take JSON input and process it, returning the required data
         """
-        json = json['items'][0:10]  # get top ten results
-
-        results = []
-
-        for result in json:
-            results.append({'name': result['name'], 'path': result['path']})
-        return results
+        top_10_items = json_response['items'][0:10]  # get top ten results
+        return [{'name': result['name'], 'path': result['path']} for result in top_10_items]
 
     def recipe_present(self, search_string):
         """

From 8d7af8816827949508d7b67bd05bf33949c4bd79 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 20 Jul 2021 15:30:19 +0100
Subject: [PATCH 156/452] Small code refactorings

---
 lib/galaxy/tool_util/deps/mulled/mulled_search.py | 6 ++----
 test/unit/tool_util/mulled/test_mulled_search.py  | 9 ++++++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_search.py b/lib/galaxy/tool_util/deps/mulled/mulled_search.py
index b02d8296331..4fa927f018c 100755
--- a/lib/galaxy/tool_util/deps/mulled/mulled_search.py
+++ b/lib/galaxy/tool_util/deps/mulled/mulled_search.py
@@ -160,10 +160,8 @@ class GitHubSearch():
         """
         Check if a recipe exists in bioconda-recipes which matches search_string exactly
         """
-        if requests.get(f"https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/{search_string}", timeout=MULLED_SOCKET_TIMEOUT).status_code == 200:
-            return True
-        else:
-            return False
+        response = requests.get(f"https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/{search_string}", timeout=MULLED_SOCKET_TIMEOUT)
+        return response.status_code == 200
 
 
 def get_package_hash(packages, versions):
diff --git a/test/unit/tool_util/mulled/test_mulled_search.py b/test/unit/tool_util/mulled/test_mulled_search.py
index 8caaed59694..88e4efcee55 100644
--- a/test/unit/tool_util/mulled/test_mulled_search.py
+++ b/test/unit/tool_util/mulled/test_mulled_search.py
@@ -1,6 +1,13 @@
 import pytest
 
-from galaxy.tool_util.deps.mulled.mulled_search import CondaSearch, get_package_hash, GitHubSearch, QuaySearch, run_command, singularity_search
+from galaxy.tool_util.deps.mulled.mulled_search import (
+    CondaSearch,
+    get_package_hash,
+    GitHubSearch,
+    QuaySearch,
+    run_command,
+    singularity_search
+)
 from ..util import external_dependency_management
 
 

From 4cd77d07b84fe17c1de01039c464fda6649cb59b Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Tue, 20 Jul 2021 22:43:18 +0530
Subject: [PATCH 157/452] Added additional guard conditions in k8s runner to
 prevent index out of bounds

---
 lib/galaxy/jobs/runners/kubernetes.py       | 17 ++++++++++-------
 lib/galaxy/jobs/runners/util/pykube_util.py |  4 ++--
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py
index 9e0c089a2ba..f9cd9df9a27 100644
--- a/lib/galaxy/jobs/runners/kubernetes.py
+++ b/lib/galaxy/jobs/runners/kubernetes.py
@@ -845,10 +845,13 @@ class KubernetesJobRunner(AsynchronousJobRunner):
         self._handle_metadata_externally(job_state.job_wrapper, resolve_requirements=True)
         super().finish_job(job_state)
         jobs = find_job_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace'])
-        if len(jobs.response['items']) != 1:
-            log.warning("More than one job matches selector. Possible configuration error"
-                        " in job id '%s'", job_state.job_id)
-        job = Job(self._pykube_api, jobs.response['items'][0])
-        if job_state.job_wrapper.guest_ports:
-            self.__cleanup_k8s_interactivetools(job_state.job_wrapper, job)
-        self.__cleanup_k8s_job(job)
+        if len(jobs.response['items']) > 1:
+            log.warning("More than one job matches selector: %s. Possible configuration error"
+                        " in job id '%s'" % (jobs.response['items'], job_state.job_id))
+        elif len(jobs.response['items']) == 0:
+            log.warning("No k8s job found which matches job id '%s'. Ignoring...", job_state.job_id)
+        else:
+            job = Job(self._pykube_api, jobs.response['items'][0])
+            if job_state.job_wrapper.guest_ports:
+                self.__cleanup_k8s_interactivetools(job_state.job_wrapper, job)
+            self.__cleanup_k8s_job(job)
diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py
index 5f873f30d00..066cd8f9bca 100644
--- a/lib/galaxy/jobs/runners/util/pykube_util.py
+++ b/lib/galaxy/jobs/runners/util/pykube_util.py
@@ -89,8 +89,8 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None):
 
 
 def is_pod_unschedulable(pykube_api, pod, namespace=None):
-    is_unschedulable = bool(len([c for c in pod.obj['status']['conditions'] if c.get("reason") == "Unschedulable"]))
-    if pod.obj['status']['phase'] == "Pending" and is_unschedulable:
+    is_unschedulable = bool(len([c for c in pod.obj['status'].get('conditions', []) if c.get("reason") == "Unschedulable"]))
+    if pod.obj['status'].get('phase') == "Pending" and is_unschedulable:
         return True
 
     return False

From bb8713b7e3cd2b95041c05a9623b3d8c22dd3173 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Wed, 21 Jul 2021 12:16:21 +0100
Subject: [PATCH 158/452] Move casts to `FoldersController`

---
 lib/galaxy/managers/folders.py           |  6 +-----
 lib/galaxy/webapps/galaxy/api/folders.py | 12 ++++++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/managers/folders.py b/lib/galaxy/managers/folders.py
index 94ae8ddcf2f..9ae332192cb 100644
--- a/lib/galaxy/managers/folders.py
+++ b/lib/galaxy/managers/folders.py
@@ -405,11 +405,7 @@ class FoldersService:
         is_admin = trans.user_is_admin
         decoded_folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
         folder = self.folder_manager.get(trans, decoded_folder_id)
-        if isinstance(page, str):
-            page = int(page)
-        if isinstance(page_limit, str):
-            page_limit = int(page_limit)
-            
+
         if not (is_admin or trans.app.security_agent.can_manage_library_item(current_user_roles, folder)):
             raise InsufficientPermissionsException('You do not have proper permission to access permissions of this folder.')
 
diff --git a/lib/galaxy/webapps/galaxy/api/folders.py b/lib/galaxy/webapps/galaxy/api/folders.py
index 46ba96ccb6f..f02c39cfca2 100644
--- a/lib/galaxy/webapps/galaxy/api/folders.py
+++ b/lib/galaxy/webapps/galaxy/api/folders.py
@@ -246,10 +246,14 @@ class FoldersController(BaseGalaxyAPIController):
 
         :raises: InsufficientPermissionsException
         """
-        scope = kwd.get('scope', None)
-        page = kwd.get('page', None)
-        page_limit = kwd.get('page_limit', None)
-        query = kwd.get('q', None)
+        scope = kwd.get('scope')
+        page = kwd.get('page')
+        if isinstance(page, str):
+            page = int(page)
+        page_limit = kwd.get('page_limit')
+        if isinstance(page_limit, str):
+            page_limit = int(page_limit)
+        query = kwd.get('q')
         return self.service.get_permissions(trans, encoded_folder_id, scope, page, page_limit, query)
 
     @expose_api

From b3354bdca7a6a415902a8a4cc0e04e8b96a89978 Mon Sep 17 00:00:00 2001
From: Kjetil Klepper <kjetil.klepper@ntnu.no>
Date: Mon, 19 Jul 2021 12:57:40 +0200
Subject: [PATCH 159/452] Added a necessary check to install the 'fs.sshfs'
 package if a data source of type 'ssh' is configured in
 'file_sources_conf.yml'

---
 lib/galaxy/dependencies/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py
index 7fe031be839..5762516a1d9 100644
--- a/lib/galaxy/dependencies/__init__.py
+++ b/lib/galaxy/dependencies/__init__.py
@@ -227,6 +227,9 @@ class ConditionalDependencies:
         # pyfilesystem plugin access to terra on anvil
         return 'anvil' in self.file_sources
 
+    def check_fs_sshfs(self):
+        return 'ssh' in self.file_sources
+
     def check_s3fs(self):
         # use s3fs directly (skipping pyfilesystem) for direct access to more options
         return 's3fs' in self.file_sources

From 46fed3d3441ece2a60175d2af3903e52ef24c272 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 21 Jul 2021 15:21:26 -0400
Subject: [PATCH 160/452] Fix Pulsar's ability to read tool provided metadata
 when path is overridden.

---
 lib/galaxy/jobs/runners/pulsar.py        | 11 ++++++++---
 test/integration/test_pulsar_embedded.py |  2 ++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index bf7c9d9a2a3..15a6aa3c977 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -6,6 +6,7 @@ More information on Pulsar can be found at https://pulsar.readthedocs.io/ .
 import errno
 import logging
 import os
+import re
 import subprocess
 from time import sleep
 
@@ -758,6 +759,10 @@ class PulsarJobRunner(AsynchronousJobRunner):
         output_files = self.get_output_files(job_wrapper)
         metadata_directory = os.path.join(job_wrapper.working_directory, "metadata")
         metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None)
+        tool = job_wrapper.tool
+        tool_provided_metadata_file_path = tool.provided_metadata_file
+        tool_provided_metadata_style = tool.provided_metadata_style
+
         dynamic_outputs = None  # use default
         if metadata_strategy == "extended" and PulsarJobRunner.__remote_metadata(client):
             # if Pulsar is doing remote metadata and the remote metadata is extended,
@@ -766,10 +771,10 @@ class PulsarJobRunner(AsynchronousJobRunner):
         else:
             # otherwise collect everything we might need
             dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:]
+            # grab discovered outputs...
             dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns)
-        tool = job_wrapper.tool
-        tool_provided_metadata_file_path = tool.provided_metadata_file
-        tool_provided_metadata_style = tool.provided_metadata_style
+            # grab too provided metadata (galaxy.json) also...
+            dynamic_outputs.append(re.escape(tool_provided_metadata_file_path))
         dynamic_file_sources = [
             {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"}
         ]
diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py
index db9bd8baedf..75d1df29276 100644
--- a/test/integration/test_pulsar_embedded.py
+++ b/test/integration/test_pulsar_embedded.py
@@ -29,4 +29,6 @@ test_tools = integration_util.integration_tool_runner([
     "vcf_bgzip_test",
     "environment_variables",
     "multi_output_assign_primary_ext_dbkey",
+    "strict_shell",
+    "tool_provided_metadata_9",
 ])

From 3c88705c4bc1f6231999b436cbd9dd0c734fb586 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Thu, 22 Jul 2021 15:53:15 +0530
Subject: [PATCH 161/452] Some refactoring of interactive tool handling in k8s

---
 lib/galaxy/jobs/runners/kubernetes.py | 93 ++++++++++++++++-----------
 1 file changed, 57 insertions(+), 36 deletions(-)

diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py
index 9e0c089a2ba..ca8561354b9 100644
--- a/lib/galaxy/jobs/runners/kubernetes.py
+++ b/lib/galaxy/jobs/runners/kubernetes.py
@@ -150,36 +150,24 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             log.exception(f"({job_wrapper.get_id_tag()}) failure writing job script")
             return
 
-        # Construction of the Kubernetes Job object follows: http://kubernetes.io/docs/user-guide/persistent-volumes/
+        # Construction of Kubernetes objects follow: https://kubernetes.io/docs/concepts/workloads/controllers/job/
+        if self.__is_interactive_tool(ajs):
+            try:
+                self.__configure_interactive_tool_services(ajs)
+            except HTTPError:
+                log.exception("Kubernetes failed to create interactive tool services, HTTP exception encountered")
+                ajs.runner_state = JobState.runner_states.UNKNOWN_ERROR
+                ajs.fail_message = "Kubernetes failed to create interactive tool services."
+                self.mark_as_failed(ajs)
+                return
+
         k8s_job_prefix = self.__produce_k8s_job_prefix()
-        guest_ports = ajs.job_wrapper.guest_ports
-        ports_dict = {}
-        for guest_port in guest_ports:
-            ports_dict[str(guest_port)] = dict(host='manual', port=guest_port, protocol="https")
-        eps = None
-        if ajs.job_wrapper.guest_ports:
-            k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix, ajs.job_wrapper)
-            log.debug(f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}')
-            k8s_service_obj = service_object_dict(
-                self.runner_params,
-                k8s_job_name,
-                self.__get_k8s_service_spec(ajs)
-            )
-            eps = self.app.interactivetool_manager.configure_entry_points(ajs.job_wrapper.get_job(), ports_dict)
-            k8s_ingress_obj = ingress_object_dict(
-                self.runner_params,
-                k8s_job_name,
-                self.__get_k8s_ingress_spec(ajs, eps)
-            )
-            service = Service(self._pykube_api, k8s_service_obj)
-            service.create()
-            ingress = Ingress(self._pykube_api, k8s_ingress_obj)
-            ingress.create()
         k8s_job_obj = job_object_dict(
             self.runner_params,
             k8s_job_prefix,
-            self.__get_k8s_job_spec(ajs, eps)
+            self.__get_k8s_job_spec(ajs)
         )
+
         job = Job(self._pykube_api, k8s_job_obj)
         try:
             job.create()
@@ -203,6 +191,37 @@ class KubernetesJobRunner(AsynchronousJobRunner):
         job_wrapper.set_external_id(job_id)
         self.monitor_queue.put(ajs)
 
+    def __is_interactive_tool(self, ajs):
+        return bool(ajs.job_wrapper.guest_ports)
+
+    def __configure_interactive_tool_services(self, ajs):
+        # Configure interactive tool entry points first
+        guest_ports = ajs.job_wrapper.guest_ports
+        ports_dict = {}
+        for guest_port in guest_ports:
+            ports_dict[str(guest_port)] = dict(host='manual', port=guest_port, protocol="https")
+        self.app.interactivetool_manager.configure_entry_points(ajs.job_wrapper.get_job(), ports_dict)
+
+        # Configure additional k8s service and ingress for interactive tool
+        k8s_job_prefix = self.__produce_k8s_job_prefix()
+        k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix, ajs.job_wrapper)
+        log.debug(f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}')
+        k8s_service_obj = service_object_dict(
+            self.runner_params,
+            k8s_job_name,
+            self.__get_k8s_service_spec(ajs)
+        )
+
+        k8s_ingress_obj = ingress_object_dict(
+            self.runner_params,
+            k8s_job_name,
+            self.__get_k8s_ingress_spec(ajs)
+        )
+        service = Service(self._pykube_api, k8s_service_obj)
+        service.create()
+        ingress = Ingress(self._pykube_api, k8s_ingress_obj)
+        ingress.create()
+
     def __get_overridable_params(self, job_wrapper, param_key):
         dest_params = self.__get_destination_params(job_wrapper)
         return dest_params.get(param_key, self.runner_params[param_key])
@@ -265,10 +284,10 @@ class KubernetesJobRunner(AsynchronousJobRunner):
         instance_id = self._galaxy_instance_id or ''
         return produce_k8s_job_prefix(app_prefix='gxy', instance_id=instance_id)
 
-    def __get_k8s_job_spec(self, ajs, eps=None):
+    def __get_k8s_job_spec(self, ajs):
         """Creates the k8s Job spec. For a Job spec, the only requirement is to have a .spec.template.
         If the job hangs around unlimited it will be ended after k8s wall time limit, which sets activeDeadlineSeconds"""
-        k8s_job_spec = {"template": self.__get_k8s_job_spec_template(ajs, eps),
+        k8s_job_spec = {"template": self.__get_k8s_job_spec_template(ajs),
                         "activeDeadlineSeconds": int(self.runner_params['k8s_walltime_limit'])}
         job_ttl = self.runner_params["k8s_job_ttl_secs_after_finished"]
         if self.runner_params["k8s_cleanup_job"] != "never" and job_ttl is not None:
@@ -289,7 +308,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             label_val += 'x'
         return label_val
 
-    def __get_k8s_job_spec_template(self, ajs, eps=None):
+    def __get_k8s_job_spec_template(self, ajs):
         """The k8s spec template is nothing but a Pod spec, except that it is nested and does not have an apiversion
         nor kind. In addition to required fields for a Pod, a pod template in a job must specify appropriate labels
         (see pod selector) and an appropriate restart policy."""
@@ -314,7 +333,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             "spec": {
                 "volumes": self.runner_params['k8s_mountable_volumes'],
                 "restartPolicy": self.__get_k8s_restart_policy(ajs.job_wrapper),
-                "containers": self.__get_k8s_containers(ajs, eps),
+                "containers": self.__get_k8s_containers(ajs),
                 "priorityClassName": self.runner_params['k8s_pod_priority_class'],
                 "tolerations": yaml.safe_load(self.runner_params['k8s_tolerations'] or "[]"),
                 "affinity": yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper,
@@ -363,19 +382,20 @@ class KubernetesJobRunner(AsynchronousJobRunner):
         }
         return k8s_spec_template
 
-    def __get_k8s_ingress_spec(self, ajs, eps=None):
+    def __get_k8s_ingress_spec(self, ajs):
         """The k8s spec template is nothing but a Ingress spec, except that it is nested and does not have an apiversion
         nor kind."""
         guest_ports = ajs.job_wrapper.guest_ports
         if len(guest_ports) > 0:
             entry_points = []
-            for entry_point in eps.get('configured', []):
+            configured_eps = [ep for ep in ajs.job_wrapper.get_job().interactivetool_entry_points if ep.configured]
+            for entry_point in configured_eps:
                 # sending in self.app as `trans` since it's only used for `.security` so seems to work
                 entry_point_path = self.app.interactivetool_manager.get_entry_point_path(self.app, entry_point)
                 if '?' in entry_point_path:
                     # Removing all the parameters from the ingress path, but they will still be in the database
                     # so the link that the user clicks on will still have them
-                    log.warn("IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S")
+                    log.warning("IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S")
                     entry_point_path = entry_point_path.split('?')[0]
                 entry_point_domain = f'{self.app.config.interactivetools_proxy_host}'
                 if entry_point.requires_domain:
@@ -432,7 +452,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
         """The default Kubernetes restart policy for Jobs"""
         return "Never"
 
-    def __get_k8s_containers(self, ajs, eps=None):
+    def __get_k8s_containers(self, ajs):
         """Fills in all required for setting up the docker containers to be used, including setting a pull policy if
            this has been set.
            $GALAXY_VIRTUAL_ENV is set to None to avoid the galaxy virtualenv inside the tool container.
@@ -480,14 +500,15 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             extra_envs = yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper, 'k8s_extra_job_envs') or "{}")
             for key in extra_envs:
                 envs.append({'name': key, 'value': extra_envs[key]})
-            if eps:
-                for entry_point in eps.get('configured', []):
+            if self.__is_interactive_tool(ajs):
+                configured_eps = [ep for ep in ajs.job_wrapper.get_job().interactivetool_entry_points if ep.configured]
+                for entry_point in configured_eps:
                     # sending in self.app as `trans` since it's only used for `.security` so seems to work
                     entry_point_path = self.app.interactivetool_manager.get_entry_point_path(self.app, entry_point)
                     if '?' in entry_point_path:
                         # Removing all the parameters from the ingress path, but they will still be in the database
                         # so the link that the user clicks on will still have them
-                        log.warn("IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S")
+                        log.warning("IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S")
                         entry_point_path = entry_point_path.split('?')[0]
                     entry_point_domain = f'{self.app.config.interactivetools_proxy_host}'
                     if entry_point.requires_domain:

From 8d744dfcb8bdbfba82528d979f47f8f8d6a45f9c Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 22 Jul 2021 18:54:21 -0400
Subject: [PATCH 162/452] Make autocomplete field input field options
 scrollable, fix filtering

---
 client/src/mvc/ui/ui-misc.js  | 4 +---
 client/src/style/scss/ui.scss | 7 +++++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/client/src/mvc/ui/ui-misc.js b/client/src/mvc/ui/ui-misc.js
index 2106d2ff464..d3a4a655c5b 100644
--- a/client/src/mvc/ui/ui-misc.js
+++ b/client/src/mvc/ui/ui-misc.js
@@ -105,9 +105,7 @@ export var Input = Backbone.View.extend({
         var datalist = this.model.get("datalist");
         if (Array.isArray(datalist) && datalist.length > 0) {
             this.$el.autocomplete({
-                source: function (request, response) {
-                    response(self.model.get("datalist"));
-                },
+                source: self.model.get("datalist"),
                 change: function () {
                     self._onchange();
                 },
diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss
index a752574eb50..a3951b25415 100644
--- a/client/src/style/scss/ui.scss
+++ b/client/src/style/scss/ui.scss
@@ -38,6 +38,13 @@ $ui-margin-horizontal: $margin-h;
 $ui-margin-horizontal-small: $margin-h * 0.5;
 $ui-margin-horizontal-large: $margin-v * 2;
 
+.ui-autocomplete {
+    max-height: 9.5rem;
+    position: fixed;
+    overflow-y: auto;
+    overflow-x: hidden;
+}
+
 .ui-modal {
     display: none;
     overflow: auto;

From 08dc137ccb126bddda3226f7909de882bf72a698 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 16 Jun 2021 13:11:42 -0400
Subject: [PATCH 163/452] Add workflow parameters to workflow run component

---
 .../Workflow/Run/WorkflowRunForm.vue          | 23 ++++++++++++++++++-
 client/src/mvc/tool/tool-form-composite.js    |  2 +-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index c645ed2cae3..9f81a779394 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,11 +1,24 @@
 <template>
-    <div ref="form" />
+    <div>
+        <FormCard v-if="wpInputsAvailable" title="Workflow Parameters" icon="">
+            <template v-slot:body>
+                <Form :inputs="wpInputsArray" :options="model" />
+            </template>
+        </FormCard>
+        <div ref="form" />
+    </div>
 </template>
 
 <script>
 import ToolFormComposite from "mvc/tool/tool-form-composite";
+import Form from "components/Form/Form";
+import FormCard from "components/Form/FormCard";
 
 export default {
+    components: {
+        Form,
+        FormCard,
+    },
     props: {
         model: {
             type: Object,
@@ -21,6 +34,14 @@ export default {
             runForm: null,
         };
     },
+    computed: {
+        wpInputsAvailable() {
+            return this.wpInputsArray.length > 0;
+        },
+        wpInputsArray() {
+            return Object.keys(this.model.wpInputs).map((k) => this.model.wpInputs[k]);
+        },
+    },
     mounted() {
         this.$nextTick(() => {
             const el = this.$refs["form"];
diff --git a/client/src/mvc/tool/tool-form-composite.js b/client/src/mvc/tool/tool-form-composite.js
index a68b8dae6f2..abff56d190d 100644
--- a/client/src/mvc/tool/tool-form-composite.js
+++ b/client/src/mvc/tool/tool-form-composite.js
@@ -62,7 +62,7 @@ var View = Backbone.View.extend({
                     });
                 },
             });
-            this._append(this.$el.empty(), this.wp_form.$el);
+            //this._append(this.$el.empty(), this.wp_form.$el);
         }
     },
 

From 5ee3dd7c8ad0831e049dbf068a60cc5b851e0c47 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 16 Jun 2021 13:18:49 -0400
Subject: [PATCH 164/452] Add history options to workflow run component

---
 client/src/components/Form/FormCard.vue       |  2 +-
 .../Workflow/Run/WorkflowRunForm.vue          | 36 +++++++++++++++++--
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 1562b2e8161..69399b90a83 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -29,7 +29,7 @@ export default {
         },
         icon: {
             type: String,
-            default: "fa-wrench",
+            default: "",
         },
     },
 };
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 9f81a779394..6fc700674ac 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,8 +1,13 @@
 <template>
     <div>
-        <FormCard v-if="wpInputsAvailable" title="Workflow Parameters" icon="">
+        <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
-                <Form :inputs="wpInputsArray" :options="model" />
+                <Form :inputs="wpInputsArray" />
+            </template>
+        </FormCard>
+        <FormCard title="History Options">
+            <template v-slot:body>
+                <Form :inputs="historyInputs" />
             </template>
         </FormCard>
         <div ref="form" />
@@ -31,7 +36,32 @@ export default {
     },
     data() {
         return {
-            runForm: null,
+            historyInputs: [
+                {
+                    type: "conditional",
+                    name: "new_history",
+                    test_param: {
+                        name: "check",
+                        label: "Send results to a new history",
+                        type: "boolean",
+                        value: "false",
+                        help: "",
+                    },
+                    cases: [
+                        {
+                            value: "true",
+                            inputs: [
+                                {
+                                    name: "name",
+                                    label: "History name",
+                                    type: "text",
+                                    value: this.model.name,
+                                },
+                            ],
+                        },
+                    ],
+                },
+            ],
         };
     },
     computed: {

From 0e9ffe230bba000cf91d0fd75d6f220e6db4ca1b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 16 Jun 2021 14:20:34 -0400
Subject: [PATCH 165/452] Add chached jobs form to run workflow component

---
 .../Workflow/Run/WorkflowRunForm.vue          | 56 +++++++++++++++++--
 client/src/mvc/tool/tool-form-composite.js    |  2 +-
 2 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 6fc700674ac..06624d69446 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -2,7 +2,7 @@
     <div>
         <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
-                <Form :inputs="wpInputsArray" />
+                <Form :inputs="wpInputs" />
             </template>
         </FormCard>
         <FormCard title="History Options">
@@ -10,11 +10,22 @@
                 <Form :inputs="historyInputs" />
             </template>
         </FormCard>
+        <FormCard v-if="cacheInputsAvailable" title="Job re-use Options">
+            <template v-slot:body>
+                <Form :inputs="cacheInputs" />
+            </template>
+        </FormCard>
+        <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
+            <template v-slot:body>
+                <Form :inputs="resourceInputs" />
+            </template>
+        </FormCard>
         <div ref="form" />
     </div>
 </template>
 
 <script>
+import { getGalaxyInstance } from "app";
 import ToolFormComposite from "mvc/tool/tool-form-composite";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
@@ -65,11 +76,43 @@ export default {
         };
     },
     computed: {
-        wpInputsAvailable() {
-            return this.wpInputsArray.length > 0;
+        cacheInputsAvailable() {
+            const Galaxy = getGalaxyInstance();
+            var extra_user_preferences = {};
+            if (Galaxy.user.attributes.preferences && "extra_user_preferences" in Galaxy.user.attributes.preferences) {
+                extra_user_preferences = JSON.parse(Galaxy.user.attributes.preferences.extra_user_preferences);
+            }
+            return;
+            "use_cached_job|use_cached_job_checkbox" in extra_user_preferences
+                ? extra_user_preferences["use_cached_job|use_cached_job_checkbox"] === "true"
+                : false;
         },
-        wpInputsArray() {
-            return Object.keys(this.model.wpInputs).map((k) => this.model.wpInputs[k]);
+        cacheInputs() {
+            return [
+                {
+                    type: "conditional",
+                    name: "use_cached_job",
+                    test_param: {
+                        name: "check",
+                        label: "Attempt to reuse jobs with identical parameters?",
+                        type: "boolean",
+                        value: "false",
+                        help: "This may skip executing jobs that you have already run.",
+                    },
+                },
+            ];
+        },
+        resourceInputsAvailable() {
+            return this.resourceInputs.length > 0;
+        },
+        resourceInputs() {
+            return this.toArray(this.model.workflowResourceParameters);
+        },
+        wpInputsAvailable() {
+            return this.wpInputs.length > 0;
+        },
+        wpInputs() {
+            return this.toArray(this.model.wpInputs);
         },
     },
     mounted() {
@@ -91,6 +134,9 @@ export default {
         handleInvocations(invocations) {
             this.$emit("submissionSuccess", invocations);
         },
+        toArray(obj) {
+            return obj ? Object.keys(obj).map((k) => obj[k]) : [];
+        },
     },
 };
 </script>
diff --git a/client/src/mvc/tool/tool-form-composite.js b/client/src/mvc/tool/tool-form-composite.js
index abff56d190d..57043fcfb8d 100644
--- a/client/src/mvc/tool/tool-form-composite.js
+++ b/client/src/mvc/tool/tool-form-composite.js
@@ -98,7 +98,7 @@ var View = Backbone.View.extend({
                 },
             ],
         });
-        this._append(this.$el, this.history_form.$el);
+        //this._append(this.$el, this.history_form.$el);
     },
 
     /** Render Workflow Options */

From a37fef2aa9ebfa4faf07afd19cc1af48a4ca3f9a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 16 Jun 2021 15:52:22 -0400
Subject: [PATCH 166/452] Add initial components for data and tool steps to
 workflow run component

---
 .../Workflow/Run/WorkflowRunDefaultStep.vue   | 57 ++++++++++++++++++
 .../Workflow/Run/WorkflowRunForm.vue          | 12 ++++
 .../Workflow/Run/WorkflowRunToolStep.vue      | 60 +++++++++++++++++++
 client/src/mvc/tool/tool-form-composite.js    |  8 +--
 4 files changed, 133 insertions(+), 4 deletions(-)
 create mode 100644 client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
 create mode 100644 client/src/components/Workflow/Run/WorkflowRunToolStep.vue

diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
new file mode 100644
index 00000000000..0d878cafb7a
--- /dev/null
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -0,0 +1,57 @@
+<template>
+    <div>
+        <FormCard :title="model.fixed_title">
+            <template v-slot:body>
+                <Form :inputs="inputs" />
+            </template>
+        </FormCard>
+    </div>
+</template>
+
+<script>
+import { getGalaxyInstance } from "app";
+import Form from "components/Form/Form";
+import FormCard from "components/Form/FormCard";
+
+export default {
+    components: {
+        Form,
+        FormCard,
+    },
+    props: {
+        model: {
+            type: Object,
+            required: true,
+        },
+    },
+    data() {
+        return {};
+    },
+    computed: {
+        isSimpleInput() {
+            return (
+                this.model.step_type.startsWith("data_input") ||
+                this.model.step_type.startsWith("data_collection_input")
+            );
+        },
+        inputs() {
+            this.model.inputs.forEach((input) => {
+                input.flavor = "module";
+                input.hide_label = this.isSimpleInput;
+            });
+            if (this.model.inputs && this.model.inputs.length > 0) {
+                return this.model.inputs;
+            } else {
+                return [
+                    {
+                        type: "hidden",
+                        name: "No options available.",
+                        ignore: null,
+                    },
+                ];
+            }
+        },
+    },
+    methods: {},
+};
+</script>
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 06624d69446..bd04fc61cb5 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -21,6 +21,14 @@
             </template>
         </FormCard>
         <div ref="form" />
+        <div v-for="step in model.steps">
+            <div v-if="step.step_type.startsWith('data_input') || step.step_type.startsWith('data_collection_input')">
+                <WorkflowRunDefaultStep :model="step" />
+            </div>
+            <div v-else>
+                <WorkflowRunToolStep :model="step" />
+            </div>
+        </div>
     </div>
 </template>
 
@@ -29,11 +37,15 @@ import { getGalaxyInstance } from "app";
 import ToolFormComposite from "mvc/tool/tool-form-composite";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
+import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
+import WorkflowRunToolStep from "./WorkflowRunToolStep";
 
 export default {
     components: {
         Form,
         FormCard,
+        WorkflowRunDefaultStep,
+        WorkflowRunToolStep,
     },
     props: {
         model: {
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
new file mode 100644
index 00000000000..09195fa97ac
--- /dev/null
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -0,0 +1,60 @@
+<template>
+    <div>
+        <FormCard title="model.label">
+            <template v-slot:body>
+                <Form :inputs="inputs" />
+            </template>
+        </FormCard>
+    </div>
+</template>
+
+<script>
+import { getGalaxyInstance } from "app";
+import Form from "components/Form/Form";
+import FormCard from "components/Form/FormCard";
+
+export default {
+    components: {
+        Form,
+        FormCard,
+    },
+    props: {
+        model: {
+            type: Object,
+            required: true,
+        },
+    },
+    data() {
+        return {};
+    },
+    created() {
+        console.log(this.model);
+    },
+    computed: {
+        isSimpleInput() {
+            return (
+                this.model.step_type.startsWith("data_input") ||
+                this.model.step_type.startsWith("data_collection_input")
+            );
+        },
+        inputs() {
+            this.model.inputs.forEach((input) => {
+                input.flavor = "module";
+                input.hide_label = this.isSimpleInput;
+            });
+            if (this.model.inputs && this.model.inputs.length > 0) {
+                return this.model.inputs;
+            } else {
+                return [
+                    {
+                        type: "hidden",
+                        name: "No options available.",
+                        ignore: null,
+                    },
+                ];
+            }
+        },
+    },
+    methods: {},
+};
+</script>
diff --git a/client/src/mvc/tool/tool-form-composite.js b/client/src/mvc/tool/tool-form-composite.js
index 57043fcfb8d..aee5bede902 100644
--- a/client/src/mvc/tool/tool-form-composite.js
+++ b/client/src/mvc/tool/tool-form-composite.js
@@ -37,13 +37,13 @@ var View = Backbone.View.extend({
 
     render: function () {
         this.deferred.reset();
-        this._renderParameters();
+        /*this._renderParameters();
         this._renderHistory();
         this._renderUseCachedJob();
         this._renderResourceParameters();
         _.each(this.steps, (step) => {
             this._renderStep(step);
-        });
+        });*/
     },
 
     /** Render workflow parameters */
@@ -62,7 +62,7 @@ var View = Backbone.View.extend({
                     });
                 },
             });
-            //this._append(this.$el.empty(), this.wp_form.$el);
+            this._append(this.$el.empty(), this.wp_form.$el);
         }
     },
 
@@ -98,7 +98,7 @@ var View = Backbone.View.extend({
                 },
             ],
         });
-        //this._append(this.$el, this.history_form.$el);
+        this._append(this.$el, this.history_form.$el);
     },
 
     /** Render Workflow Options */

From 62a1475d6a4e0e30c874b681ab5c9543617612ee Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 03:26:07 -0400
Subject: [PATCH 167/452] Add correct tool form title to workflow steps

---
 client/src/components/Tool/ToolCard.vue       |  6 +---
 .../Workflow/Run/WorkflowRunToolStep.vue      | 28 ++++---------------
 2 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index 850cfd2050a..8c276fb16cd 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -143,10 +143,6 @@ export default {
             type: String,
             required: false,
         },
-        sustainVersion: {
-            type: Boolean,
-            default: false,
-        },
         options: {
             type: Object,
             required: true,
@@ -179,7 +175,7 @@ export default {
             return !!this.user.email && !this.isFavorite;
         },
         showVersions() {
-            return !this.sustainVersion && this.versions && this.versions.length > 1;
+            return this.versions && this.versions.length > 1;
         },
         availableVersions() {
             const versions = this.versions.slice();
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 09195fa97ac..1d5d7448120 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,8 +1,8 @@
 <template>
     <div>
-        <FormCard title="model.label">
+        <FormCard :title="title">
             <template v-slot:body>
-                <Form :inputs="inputs" />
+                <Form :inputs="model.inputs" />
             </template>
         </FormCard>
     </div>
@@ -31,27 +31,11 @@ export default {
         console.log(this.model);
     },
     computed: {
-        isSimpleInput() {
-            return (
-                this.model.step_type.startsWith("data_input") ||
-                this.model.step_type.startsWith("data_collection_input")
-            );
-        },
-        inputs() {
-            this.model.inputs.forEach((input) => {
-                input.flavor = "module";
-                input.hide_label = this.isSimpleInput;
-            });
-            if (this.model.inputs && this.model.inputs.length > 0) {
-                return this.model.inputs;
+        title() {
+            if (this.model.fixed_title) {
+                return this.model.fixed_title;
             } else {
-                return [
-                    {
-                        type: "hidden",
-                        name: "No options available.",
-                        ignore: null,
-                    },
-                ];
+                return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
             }
         },
     },

From 3b002cd15e9712202b066e2bfbd2137ed9210c3a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 03:29:04 -0400
Subject: [PATCH 168/452] Fix step type selector in workflow run component

---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index bd04fc61cb5..964f917a238 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -22,12 +22,8 @@
         </FormCard>
         <div ref="form" />
         <div v-for="step in model.steps">
-            <div v-if="step.step_type.startsWith('data_input') || step.step_type.startsWith('data_collection_input')">
-                <WorkflowRunDefaultStep :model="step" />
-            </div>
-            <div v-else>
-                <WorkflowRunToolStep :model="step" />
-            </div>
+            <WorkflowRunToolStep v-if="step.step_type == 'tool'" :model="step" />
+            <WorkflowRunDefaultStep v-else :model="step" />
         </div>
     </div>
 </template>

From 8cc89b1159ddd72e8ad5760d818edcc05558ce73 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 03:42:53 -0400
Subject: [PATCH 169/452] Add corresponding icons to workflow steps

---
 .../src/components/Workflow/Run/WorkflowRunDefaultStep.vue  | 6 +++++-
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 0d878cafb7a..224f0eaf702 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="model.fixed_title">
+        <FormCard :title="model.fixed_title" :icon="icon">
             <template v-slot:body>
                 <Form :inputs="inputs" />
             </template>
@@ -10,6 +10,7 @@
 
 <script>
 import { getGalaxyInstance } from "app";
+import WorkflowIcons from "components/Workflow/icons";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
 
@@ -28,6 +29,9 @@ export default {
         return {};
     },
     computed: {
+        icon() {
+            return WorkflowIcons[this.model.step_type];
+        },
         isSimpleInput() {
             return (
                 this.model.step_type.startsWith("data_input") ||
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 1d5d7448120..76da6aae14f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="title">
+        <FormCard :title="title" icon="fa-wrench">
             <template v-slot:body>
                 <Form :inputs="model.inputs" />
             </template>

From 8f3d4f18f3a80324fad0127fc34b71bb4dd256d7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 11:09:27 -0400
Subject: [PATCH 170/452] Remove unused fixed title condition for tool steps

---
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 76da6aae14f..ee2200bf77b 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -32,11 +32,7 @@ export default {
     },
     computed: {
         title() {
-            if (this.model.fixed_title) {
-                return this.model.fixed_title;
-            } else {
-                return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
-            }
+            return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
         },
     },
     methods: {},

From 5c67e7e6c71bdd4d1a47c6066f1c4c51a8abcfe0 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 12:35:08 -0400
Subject: [PATCH 171/452] Allow workflow steps to be collapsed

---
 client/src/components/Form/FormCard.vue       | 38 +++++++++++++++++--
 .../Workflow/Run/WorkflowRunDefaultStep.vue   |  2 +-
 .../Workflow/Run/WorkflowRunToolStep.vue      |  2 +-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 69399b90a83..95a88fdc53a 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -3,15 +3,28 @@
         <div class="portlet-header">
             <div class="portlet-operations">
                 <slot name="operations" />
+                <b-button
+                    v-if="collapsible"
+                    role="button"
+                    title="Collapse/Expand"
+                    variant="link"
+                    size="sm"
+                    class="float-right py-0"
+                    v-b-tooltip.hover
+                    @click="onCollapse"
+                >
+                    <span v-if="collapsed" class="fa fa-eye" />
+                    <span v-else class="fa fa-eye-slash" />
+                </b-button>
             </div>
-            <div class="portlet-title">
+            <a class="portlet-title" @click="onCollapse" :href="href">
                 <i :class="['portlet-title-icon fa mr-1', icon]" style="display: inline"></i>
                 <span class="portlet-title-text">
                     <b itemprop="name">{{ title }}</b> <span itemprop="description">{{ description }}</span>
                 </span>
-            </div>
+            </a>
         </div>
-        <div class="portlet-content">
+        <div v-if="!collapsed" class="portlet-content">
             <slot name="body" />
         </div>
     </div>
@@ -31,6 +44,25 @@ export default {
             type: String,
             default: "",
         },
+        collapsible: {
+            type: Boolean,
+            default: false,
+        },
+    },
+    data() {
+        return {
+            collapsed: this.collapsible,
+        };
+    },
+    computed: {
+        href() {
+            return this.collapsible ? "#" : null;
+        },
+    },
+    methods: {
+        onCollapse() {
+            this.collapsed = this.collapsible && !this.collapsed;
+        },
     },
 };
 </script>
diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 224f0eaf702..9176bbf68e4 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="model.fixed_title" :icon="icon">
+        <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true">
             <template v-slot:body>
                 <Form :inputs="inputs" />
             </template>
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index ee2200bf77b..a1a835ed8b1 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="title" icon="fa-wrench">
+        <FormCard :title="title" icon="fa-wrench" :collapsible="true">
             <template v-slot:body>
                 <Form :inputs="model.inputs" />
             </template>

From 788d9c49a4dac06dce8c4ece618c44abd69aa483 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 14:32:44 -0400
Subject: [PATCH 172/452] Remove legacy run workflow form view from run
 component

---
 .../components/Workflow/Run/WorkflowRunForm.vue    | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 964f917a238..8582f29de7a 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -20,7 +20,6 @@
                 <Form :inputs="resourceInputs" />
             </template>
         </FormCard>
-        <div ref="form" />
         <div v-for="step in model.steps">
             <WorkflowRunToolStep v-if="step.step_type == 'tool'" :model="step" />
             <WorkflowRunDefaultStep v-else :model="step" />
@@ -30,7 +29,6 @@
 
 <script>
 import { getGalaxyInstance } from "app";
-import ToolFormComposite from "mvc/tool/tool-form-composite";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
@@ -123,18 +121,6 @@ export default {
             return this.toArray(this.model.wpInputs);
         },
     },
-    mounted() {
-        this.$nextTick(() => {
-            const el = this.$refs["form"];
-            const formProps = {
-                el,
-                model: this.model,
-                setRunButtonStatus: this.setRunButtonStatus,
-                handleInvocations: this.handleInvocations,
-            };
-            this.runForm = new ToolFormComposite.View(formProps);
-        });
-    },
     methods: {
         execute() {
             this.runForm.execute();

From 3cad3daf11eddbdd2fc171eb131d36da03bfb136 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 17 Jun 2021 14:34:53 -0400
Subject: [PATCH 173/452] Remove legacy tool form base and tool composite form
 views

---
 client/src/mvc/tool/tool-form-base.js      | 289 -------------
 client/src/mvc/tool/tool-form-composite.js | 475 ---------------------
 2 files changed, 764 deletions(-)
 delete mode 100644 client/src/mvc/tool/tool-form-base.js
 delete mode 100644 client/src/mvc/tool/tool-form-composite.js

diff --git a/client/src/mvc/tool/tool-form-base.js b/client/src/mvc/tool/tool-form-base.js
deleted file mode 100644
index 5648e664f78..00000000000
--- a/client/src/mvc/tool/tool-form-base.js
+++ /dev/null
@@ -1,289 +0,0 @@
-/**
-    This is the base class of the tool form plugin. This class is e.g. inherited by the regular and the workflow tool form.
-*/
-import $ from "jquery";
-import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
-import _l from "utils/localization";
-import ariaAlert from "utils/ariaAlert";
-import Deferred from "utils/deferred";
-import Ui from "mvc/ui/ui-misc";
-import FormBase from "mvc/form/form-view";
-import Webhooks from "mvc/webhooks";
-import ToolFooter from "components/Tool/ToolFooter.vue";
-import Vue from "vue";
-import axios from "axios";
-import { copy } from "utils/clipboard";
-
-export default FormBase.extend({
-    initialize: function (options) {
-        const Galaxy = getGalaxyInstance();
-        var self = this;
-        this.deferred = new Deferred();
-        FormBase.prototype.initialize.call(this, options);
-
-        // optional model update
-        this._update();
-
-        // listen to history panel
-        if (this.model.get("listen_to_history") && Galaxy.currHistoryPanel) {
-            this.listenTo(Galaxy.currHistoryPanel.collection, "change", () => {
-                self.model.get("onchange")();
-            });
-        }
-        // destroy dom elements
-        this.$el.on("remove", () => {
-            self._destroy();
-        });
-    },
-
-    /** Allows tool form variation to update tool model */
-    _update: function () {
-        var self = this;
-        var callback = this.model.get("buildmodel");
-        if (callback) {
-            this.deferred.reset();
-            this.deferred.execute((process) => {
-                callback(process, self);
-                process.then(() => {
-                    self._render();
-                });
-            });
-        } else {
-            this._render();
-        }
-    },
-
-    /** Wait for deferred build processes before removal */
-    _destroy: function () {
-        var self = this;
-        this.$el.off().hide();
-        this.deferred.execute(() => {
-            FormBase.prototype.remove.call(self);
-            const Galaxy = getGalaxyInstance();
-            Galaxy.emit.debug("tool-form-base::_destroy()", "Destroy view.");
-        });
-    },
-
-    /** Build form */
-    _render: function () {
-        var self = this;
-        var options = this.model.attributes;
-        if (options.setupToolMicrodata) {
-            this.$el.attr("itemscope", "itemscope");
-            this.$el.attr("itemtype", "https://schema.org/CreativeWork");
-        }
-        this.model.set({
-            title:
-                options.fixed_title ||
-                `<b itemprop="name">${options.name}</b> <span itemprop="description">${options.description}</span> (Galaxy Version ${options.version})`,
-            operations: !options.hide_operations && this._operations(),
-            onchange: function () {
-                self.deferred.reset();
-                self.deferred.execute((process) => {
-                    self.model.get("postchange")(process, self);
-                });
-            },
-        });
-        this.render();
-        if (!this.model.get("collapsible")) {
-            this.$el.append($("<div/>").addClass("mt-2").append(this._footer()));
-        }
-        options.tool_errors &&
-            this.message.update({
-                status: "danger",
-                message: options.tool_errors,
-                persistent: true,
-            });
-        this.show_message &&
-            this.message.update({
-                status: "success",
-                message: `Now you are using '${options.name}' version ${options.version}, id '${options.id}'.`,
-                persistent: false,
-            });
-        this.show_message = true;
-    },
-
-    /** Create tool operation menu */
-    _operations: function () {
-        var self = this;
-        var options = this.model.attributes;
-        const Galaxy = getGalaxyInstance();
-
-        // Buttons for adding and removing favorite.
-        const in_favorites = Galaxy.user.getFavorites().tools.indexOf(options.id) >= 0;
-        var favorite_button = new Ui.Button({
-            icon: "fa-star-o",
-            title: options.narrow ? null : "Favorite",
-            tooltip: "Add to favorites",
-            visible: !Galaxy.user.isAnonymous() && !in_favorites,
-            onclick: () => {
-                axios
-                    .put(`${Galaxy.root}api/users/${Galaxy.user.id}/favorites/tools`, { object_id: options.id })
-                    .then((response) => {
-                        favorite_button.hide();
-                        remove_favorite_button.show();
-                        Galaxy.user.updateFavorites("tools", response.data);
-                        ariaAlert("added to favorites");
-                    });
-            },
-        });
-
-        var remove_favorite_button = new Ui.Button({
-            icon: "fa-star",
-            title: options.narrow ? null : "Added",
-            tooltip: "Remove from favorites",
-            visible: !Galaxy.user.isAnonymous() && in_favorites,
-            onclick: () => {
-                axios
-                    .delete(
-                        `${Galaxy.root}api/users/${Galaxy.user.id}/favorites/tools/${encodeURIComponent(options.id)}`
-                    )
-                    .then((response) => {
-                        remove_favorite_button.hide();
-                        favorite_button.show();
-                        Galaxy.user.updateFavorites("tools", response.data);
-                        ariaAlert("removed from favorites");
-                    });
-            },
-        });
-
-        // button for version selection
-        var versions_button = new Ui.ButtonMenu({
-            icon: "fa-cubes",
-            cls: "btn btn-secondary float-right tool-versions",
-            title: options.narrow ? null : "Versions",
-            tooltip: "Select another tool version",
-        });
-
-        if (!options.sustain_version && options.versions && options.versions.length > 1) {
-            for (var i in options.versions.reverse()) {
-                var version = options.versions[i];
-                if (version != options.version) {
-                    versions_button.addMenu({
-                        title: `Switch to ${version}`,
-                        version: version,
-                        icon: "fa-cube",
-                        onclick: function () {
-                            // here we update the tool version (some tools encode the version also in the id)
-                            self.model.set("id", options.id.replace(options.version, this.version));
-                            self.model.set("version", this.version);
-                            self.model.get("onchange")();
-                            self._update();
-                        },
-                    });
-                }
-            }
-        } else {
-            versions_button.$el.hide();
-        }
-
-        // button for options e.g. search, help
-        var menu_button = new Ui.ButtonMenu({
-            id: "options",
-            icon: "fa-caret-down",
-            title: options.narrow ? null : "Options",
-            tooltip: "View available options",
-        });
-        menu_button.addMenu({
-            icon: "fa-chain",
-            title: _l("Copy link"),
-            onclick: function () {
-                copy(
-                    `${window.location.origin + getAppRoot()}root?tool_id=${options.id}`,
-                    "Link was copied to your clipboard"
-                );
-            },
-        });
-        menu_button.addMenu({
-            icon: "fa-files-o",
-            title: _l("Copy Tool ID"),
-            onclick: function () {
-                copy(`${options.id}`, "Tool ID was copied to your clipboard");
-            },
-        });
-
-        // add admin operations
-        if (Galaxy.user && Galaxy.user.get("is_admin")) {
-            menu_button.addMenu({
-                icon: "fa-download",
-                title: _l("Download"),
-                onclick: function () {
-                    window.location.href = `${getAppRoot()}api/tools/${options.id}/download`;
-                },
-            });
-        }
-
-        // add toolshed url
-        if (options.sharable_url) {
-            menu_button.addMenu({
-                icon: "fa-external-link",
-                title: _l("See in Tool Shed"),
-                onclick: function () {
-                    window.open(options.sharable_url);
-                },
-            });
-        }
-
-        // add tool menu webhooks
-        Webhooks.load({
-            type: "tool-menu",
-            callback: function (webhooks) {
-                webhooks.each((model) => {
-                    var webhook = model.toJSON();
-                    if (webhook.activate && webhook.config.function) {
-                        menu_button.addMenu({
-                            icon: webhook.config.icon,
-                            title: webhook.config.title,
-                            onclick: function () {
-                                var func = new Function("options", webhook.config.function);
-                                func(options);
-                            },
-                        });
-                    }
-                });
-            },
-        });
-
-        return {
-            menu: menu_button,
-            versions: versions_button,
-            favorite: favorite_button,
-            remove_favorite: remove_favorite_button,
-        };
-    },
-
-    /** Create footer */
-    _footer: function () {
-        const options = this.model.attributes;
-        const $el = $("<div/>").append(this._templateHelp(options));
-        const toolFooterInstance = Vue.extend(ToolFooter);
-        const vm = document.createElement("div");
-        $el.append(vm);
-        const propsData = {
-            id: options.id,
-            hasCitations: options.citations,
-            xrefs: options.xrefs,
-            license: options.license,
-            creators: options.creator,
-            requirements: options.requirements,
-        };
-        new toolFooterInstance({
-            propsData,
-        }).$mount(vm);
-        return $el;
-    },
-
-    /** Templates */
-    _templateHelp: function (options) {
-        var $tmpl = $("<div/>").addClass("form-help form-text mt-4").append(options.help);
-        $tmpl.find("a").attr("target", "_blank");
-        $tmpl.find("img").each(function () {
-            var img_src = $(this).attr("src");
-            if (img_src.indexOf("admin_toolshed") !== -1) {
-                $(this).attr("src", getAppRoot() + img_src);
-            }
-        });
-        return $tmpl;
-    },
-});
diff --git a/client/src/mvc/tool/tool-form-composite.js b/client/src/mvc/tool/tool-form-composite.js
deleted file mode 100644
index aee5bede902..00000000000
--- a/client/src/mvc/tool/tool-form-composite.js
+++ /dev/null
@@ -1,475 +0,0 @@
-/** This is the run workflow tool form view. */
-import _ from "underscore";
-import $ from "jquery";
-import Backbone from "backbone";
-import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
-import _l from "utils/localization";
-import Utils from "utils/utils";
-import Deferred from "utils/deferred";
-import Form from "mvc/form/form-view";
-import { isDataStep, invokeWorkflow } from "components/Workflow/Run";
-
-import ToolFormBase from "mvc/tool/tool-form-base";
-import Modal from "mvc/ui/ui-modal";
-
-var View = Backbone.View.extend({
-    initialize: function (options) {
-        const Galaxy = getGalaxyInstance();
-        this.modal = Galaxy.modal || new Modal.View();
-        this.runWorkflowModel = options.model;
-        this.deferred = new Deferred();
-        this.setRunButtonStatus = options.setRunButtonStatus;
-        this.handleInvocations = options.handleInvocations;
-        this.setElement(options.el);
-        this._configure();
-        this.render();
-    },
-
-    /** Configures form/step options for each workflow step */
-    _configure: function () {
-        this.forms = [];
-        this.steps = this.runWorkflowModel.steps;
-        this.links = this.runWorkflowModel.links;
-        this.parms = this.runWorkflowModel.parms;
-        this.wp_inputs = this.runWorkflowModel.wpInputs;
-    },
-
-    render: function () {
-        this.deferred.reset();
-        /*this._renderParameters();
-        this._renderHistory();
-        this._renderUseCachedJob();
-        this._renderResourceParameters();
-        _.each(this.steps, (step) => {
-            this._renderStep(step);
-        });*/
-    },
-
-    /** Render workflow parameters */
-    _renderParameters: function () {
-        this.wp_form = null;
-        if (!_.isEmpty(this.wp_inputs)) {
-            this.wp_form = new Form({
-                title: "<b>Workflow Parameters</b>",
-                inputs: this.wp_inputs,
-                cls: "ui-portlet-section",
-                onchange: () => {
-                    _.each(this.wp_form.input_list, (input_def, i) => {
-                        _.each(input_def.links, (step) => {
-                            this._refreshStep(step);
-                        });
-                    });
-                },
-            });
-            this._append(this.$el.empty(), this.wp_form.$el);
-        }
-    },
-
-    /** Render workflow parameters */
-    _renderHistory: function () {
-        this.history_form = new Form({
-            cls: "ui-portlet-section",
-            title: "<b>History Options</b>",
-            inputs: [
-                {
-                    type: "conditional",
-                    name: "new_history",
-                    test_param: {
-                        name: "check",
-                        label: "Send results to a new history",
-                        type: "boolean",
-                        value: "false",
-                        help: "",
-                    },
-                    cases: [
-                        {
-                            value: "true",
-                            inputs: [
-                                {
-                                    name: "name",
-                                    label: "History name",
-                                    type: "text",
-                                    value: this.runWorkflowModel.name,
-                                },
-                            ],
-                        },
-                    ],
-                },
-            ],
-        });
-        this._append(this.$el, this.history_form.$el);
-    },
-
-    /** Render Workflow Options */
-    _renderResourceParameters: function () {
-        this.workflow_resource_parameters_form = null;
-        if (!_.isEmpty(this.runWorkflowModel.workflowResourceParameters)) {
-            this.workflow_resource_parameters_form = new Form({
-                cls: "ui-portlet-section",
-                title: "<b>Workflow Resource Options</b>",
-                inputs: this.runWorkflowModel.workflowResourceParameters,
-            });
-            this._append(this.$el, this.workflow_resource_parameters_form.$el);
-        }
-    },
-
-    /** Render job caching option */
-    _renderUseCachedJob: function () {
-        const Galaxy = getGalaxyInstance();
-        var extra_user_preferences = {};
-        if (Galaxy.user.attributes.preferences && "extra_user_preferences" in Galaxy.user.attributes.preferences) {
-            extra_user_preferences = JSON.parse(Galaxy.user.attributes.preferences.extra_user_preferences);
-        }
-        var display_use_cached_job_checkbox =
-            "use_cached_job|use_cached_job_checkbox" in extra_user_preferences
-                ? extra_user_preferences["use_cached_job|use_cached_job_checkbox"]
-                : false;
-        this.display_use_cached_job_checkbox = display_use_cached_job_checkbox === "true";
-        if (this.display_use_cached_job_checkbox) {
-            this.job_options_form = new Form({
-                cls: "ui-portlet-section",
-                title: "<b>Job re-use Options</b>",
-                inputs: [
-                    {
-                        type: "conditional",
-                        name: "use_cached_job",
-                        test_param: {
-                            name: "check",
-                            label: "BETA: Attempt to reuse jobs with identical parameters?",
-                            type: "boolean",
-                            value: "false",
-                            help: "This may skip executing jobs that you have already run.",
-                        },
-                    },
-                ],
-            });
-            this._append(this.$el, this.job_options_form.$el);
-        }
-    },
-
-    /** Render step */
-    _renderStep: function (step) {
-        const Galaxy = getGalaxyInstance();
-        var form = null;
-        this.deferred.execute((promise) => {
-            this.$el.addClass("ui-steps");
-            if (step.step_type == "tool") {
-                step.postchange = function (process, form) {
-                    var current_state = {
-                        tool_id: step.id,
-                        tool_version: step.version,
-                        inputs: $.extend(true, {}, form.data.create()),
-                    };
-                    form.wait(true);
-                    Galaxy.emit.debug("tool-form-composite::postchange()", "Sending current state.", current_state);
-                    Utils.request({
-                        type: "POST",
-                        url: `${getAppRoot()}api/tools/${step.id}/build`,
-                        data: current_state,
-                        success: function (data) {
-                            form.update(data);
-                            form.wait(false);
-                            Galaxy.emit.debug("tool-form-composite::postchange()", "Received new model.", data);
-                            process.resolve();
-                        },
-                        error: function (response) {
-                            Galaxy.emit.debug("tool-form-composite::postchange()", "Refresh request failed.", response);
-                            process.reject();
-                        },
-                    });
-                };
-                form = new ToolFormBase(step);
-                if (step.post_job_actions && step.post_job_actions.length) {
-                    form.portlet.append(
-                        $("<div/>")
-                            .addClass("ui-form-element-disabled")
-                            .append($("<div/>").addClass("ui-form-title").html("<b>Job Post Actions</b>"))
-                            .append(
-                                $("<div/>")
-                                    .addClass("ui-form-preview")
-                                    .html(
-                                        _.reduce(
-                                            step.post_job_actions,
-                                            (memo, value) => `${memo} ${value.short_str}`,
-                                            ""
-                                        )
-                                    )
-                            )
-                    );
-                }
-            } else {
-                var is_simple_input = ["data_input", "data_collection_input"].indexOf(step.step_type) != -1;
-                _.each(step.inputs, (input) => {
-                    input.flavor = "module";
-                    input.hide_label = is_simple_input;
-                });
-                form = new Form(
-                    Utils.merge(
-                        {
-                            title: step.fixed_title,
-                            onchange: () => {
-                                _.each(this.links[step.index], (link) => {
-                                    this._refreshStep(link);
-                                });
-                            },
-                            inputs:
-                                step.inputs && step.inputs.length > 0
-                                    ? step.inputs
-                                    : [
-                                          {
-                                              type: "hidden",
-                                              name: "No options available.",
-                                              ignore: null,
-                                          },
-                                      ],
-                        },
-                        step
-                    )
-                );
-                if (step.step_label) {
-                    form.$el.attr("step-label", step.step_label);
-                }
-            }
-            this.forms[step.index] = form;
-            this._append(this.$el, form.$el);
-            if (step.needs_refresh) {
-                this._refreshStep(step);
-            }
-            form.portlet[!this.show_progress ? "enable" : "disable"]();
-            if (this.show_progress) {
-                const percentage = ((step.index + 1) * 100.0) / this.steps.length;
-                this.setRunButtonStatus(false, "Preparing...", percentage);
-            }
-            Galaxy.emit.debug("tool-form-composite::initialize()", `${step.index} : Workflow step state ready.`, step);
-            window.setTimeout(() => {
-                promise.resolve();
-            }, 0);
-        });
-    },
-
-    /** Refreshes step values from source step values */
-    _refreshStep: function (step) {
-        var form = this.forms[step.index];
-        if (form) {
-            _.each(this.parms[step.index], (input, name) => {
-                if (input.step_linked || input.wp_linked) {
-                    var field = form.field_list[form.data.match(name)];
-                    if (field) {
-                        var new_value;
-                        if (input.step_linked) {
-                            new_value = { values: [] };
-                            _.each(input.step_linked, (source_step) => {
-                                if (isDataStep(source_step)) {
-                                    var value = this.forms[source_step.index].data.create().input;
-                                    if (value) {
-                                        _.each(value.values, (v) => {
-                                            new_value.values.push(v);
-                                        });
-                                    }
-                                }
-                            });
-                            if (!input.multiple && new_value.values.length > 0) {
-                                new_value = {
-                                    values: [new_value.values[0]],
-                                };
-                            }
-                        } else if (input.wp_linked) {
-                            new_value = input.value;
-                            var re = /\$\{(.+?)\}/g;
-                            var match;
-                            while ((match = re.exec(input.value))) {
-                                var wp_field = this.wp_form.field_list[this.wp_form.data.match(match[1])];
-                                var wp_value = wp_field && wp_field.value();
-                                if (wp_value) {
-                                    new_value = new_value.split(match[0]).join(wp_value);
-                                }
-                            }
-                        }
-                        if (new_value !== undefined) {
-                            field.value(new_value);
-                        }
-                    }
-                }
-            });
-            form.trigger("change");
-        } else {
-            step.needs_refresh = true;
-        }
-    },
-
-    /** Build remaining steps */
-    execute: function () {
-        this.show_progress = true;
-        this._enabled(false);
-        this.deferred.execute((promise) => {
-            window.setTimeout(() => {
-                promise.resolve();
-                this._submit();
-            }, 0);
-        });
-    },
-
-    /** Validate and submit workflow */
-    _submit: function () {
-        const Galaxy = getGalaxyInstance();
-        var history_form_data = this.history_form.data.create();
-        var job_def = {
-            new_history_name: history_form_data["new_history|name"] ? history_form_data["new_history|name"] : null,
-            history_id: !history_form_data["new_history|name"] ? this.runWorkflowModel.historyId : null,
-            resource_params: this.workflow_resource_parameters_form
-                ? this.workflow_resource_parameters_form.data.create()
-                : {},
-            replacement_params: this.wp_form ? this.wp_form.data.create() : {},
-            parameters: {},
-            // Tool form will submit flat maps for each parameter
-            // (e.g. "repeat_0|cond|param": "foo" instead of nested
-            // data structures).
-            parameters_normalized: true,
-            // Tool form always wants a list of invocations back
-            // so that inputs can be batched.
-            batch: true,
-        };
-        if (this.display_use_cached_job_checkbox) {
-            job_def.use_cached_job = this.job_options_form.data.create()["use_cached_job|check"] === "true";
-        }
-        var validated = true;
-        for (var i in this.forms) {
-            var form = this.forms[i];
-            var job_inputs = form.data.create();
-            var step = this.steps[i];
-            var step_index = step.step_index;
-            form.trigger("reset");
-            for (var job_input_id in job_inputs) {
-                var input_value = job_inputs[job_input_id];
-                var input_id = form.data.match(job_input_id);
-                var input_def = form.input_list[input_id];
-                if (!input_def.step_linked) {
-                    if (isDataStep(step)) {
-                        validated = input_value && input_value.values && input_value.values.length > 0;
-                        if (!validated && input_def.optional) {
-                            validated = true;
-                        }
-                    } else {
-                        validated =
-                            input_def.optional ||
-                            (input_def.is_workflow && input_value !== "") ||
-                            (!input_def.is_workflow && input_value !== null);
-                    }
-                    if (!validated) {
-                        if (input_def.type == "hidden") {
-                            this.modal.show({
-                                title: _l("Workflow submission failed"),
-                                body: `Step ${step_index}: ${
-                                    step.step_label || step.step_name
-                                } is in an invalid state. Please remove and re-add this step in the Workflow Editor.`,
-                                buttons: {
-                                    Close: () => {
-                                        this.modal.hide();
-                                    },
-                                },
-                            });
-                        }
-                        form.highlight(input_id);
-                        break;
-                    }
-                    job_def.parameters[step_index] = job_def.parameters[step_index] || {};
-                    job_def.parameters[step_index][job_input_id] = job_inputs[job_input_id];
-                }
-            }
-            if (!validated) {
-                break;
-            }
-        }
-        if (!validated) {
-            this._enabled(true);
-            Galaxy.emit.debug("tool-form-composite::submit()", "Validation failed.", job_def);
-        } else {
-            Galaxy.emit.debug("tool-form-composite::submit()", "Validation complete.", job_def);
-            invokeWorkflow(this.runWorkflowModel.workflowId, job_def)
-                .then((invocations) => {
-                    Galaxy.emit.debug("tool-form-composite::submit", "Submission successful.", invocations);
-                    if (Array.isArray(invocations) && invocations.length > 0) {
-                        this.handleInvocations(invocations);
-                    } else {
-                        this.submissionErrorModal(job_def, invocations);
-                    }
-                    this._enabled(true);
-                })
-                .catch((response) => {
-                    // TODO: Is this the same response as the Utils post would
-                    // have had?
-                    Galaxy.emit.debug("tool-form-composite::submit", "Submission failed.", response);
-                    var input_found = false;
-                    if (response && response.err_data) {
-                        for (var i in this.forms) {
-                            var form = this.forms[i];
-                            var step_related_errors = response.err_data[form.model.get("step_index")];
-                            if (step_related_errors) {
-                                var error_messages = form.data.matchResponse(step_related_errors);
-                                for (var input_id in error_messages) {
-                                    form.highlight(input_id, error_messages[input_id]);
-                                    input_found = true;
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                    if (!input_found) {
-                        this.submissionErrorModal(job_def, response);
-                    }
-                    this._enabled(true);
-                });
-        }
-    },
-
-    /** Append new dom to body */
-    _append: function ($container, $el) {
-        $container.append("<p/>").append($el);
-    },
-
-    /** Set enabled/disabled state */
-    _enabled: function (enabled) {
-        this.setRunButtonStatus(enabled, "Sending...", -1);
-        if (this.wp_form) {
-            this.wp_form.portlet[enabled ? "enable" : "disable"]();
-        }
-        if (this.history_form) {
-            this.history_form.portlet[enabled ? "enable" : "disable"]();
-        }
-        _.each(this.forms, (form) => {
-            if (form) {
-                form.portlet[enabled ? "enable" : "disable"]();
-            }
-        });
-    },
-
-    submissionErrorModal: function (job_def, response) {
-        this.modal.show({
-            title: _l("Workflow submission failed"),
-            body: this._templateError(job_def, response && response.err_msg),
-            buttons: {
-                Close: () => {
-                    this.modal.hide();
-                },
-            },
-        });
-    },
-
-    /** Templates */
-    _templateError: function (response, err_msg) {
-        return $("<div/>")
-            .addClass("errormessagelarge")
-            .append(
-                $("<p/>").text(
-                    `The server could not complete the request. Please contact the Galaxy Team if this error persists. ${
-                        JSON.stringify(err_msg) || ""
-                    }`
-                )
-            )
-            .append($("<pre/>").text(JSON.stringify(response, null, 4)));
-    },
-});
-export default {
-    View: View,
-};

From d1c4a69fe87f4ac56682d544724d5904a5859e41 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 21 Jun 2021 15:55:12 -0400
Subject: [PATCH 174/452] Update workflow parameters in form

---
 client/src/components/Form/FormDisplay.vue    | 35 +++++++++++++++++++
 .../Workflow/Run/WorkflowRunForm.vue          |  8 +++--
 .../Workflow/Run/WorkflowRunToolStep.vue      | 13 ++++---
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 09d42bd6f16..af63a90c0df 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -4,6 +4,7 @@
 
 <script>
 import Form from "mvc/form/form-view";
+import FormData from "mvc/form/form-data";
 
 export default {
     props: {
@@ -39,6 +40,10 @@ export default {
             type: Object,
             default: null,
         },
+        placeholderParams: {
+            type: Object,
+            default: null,
+        },
     },
     data() {
         return {
@@ -70,6 +75,9 @@ export default {
                 }
             });
         },
+        placeholderParams() {
+            this.onPlaceholderParams();
+        },
     },
     mounted() {
         this.onRender();
@@ -125,6 +133,32 @@ export default {
         },
     },
     methods: {
+        onPlaceholderParams() {
+            this.params = {};
+            FormData.visitInputs(this.inputs, (input, name) => {
+                this.params[name] = input;
+            });
+            _.each(this.params, (input, name) => {
+                if (input.wp_linked) {
+                    var field = this.form.field_list[this.form.data.match(name)];
+                    if (field) {
+                        var new_value;
+                        new_value = input.value;
+                        var re = /\$\{(.+?)\}/g;
+                        var match;
+                        while ((match = re.exec(input.value))) {
+                            var wp_value = this.placeholderParams[match[1]];
+                            if (wp_value) {
+                                new_value = new_value.split(match[0]).join(wp_value);
+                            }
+                        }
+                    }
+                    if (new_value !== undefined) {
+                        field.value(new_value);
+                    }
+                }
+            });
+        },
         onChange() {
             this.formData = this.form.data.create();
             this.$emit("onChange", this.formData);
@@ -142,6 +176,7 @@ export default {
                         this.onChange();
                     },
                 });
+                this.onPlaceholderParams();
                 this.onChange();
             });
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 8582f29de7a..133fcbf25cc 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -2,7 +2,7 @@
     <div>
         <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
-                <Form :inputs="wpInputs" />
+                <Form :inputs="wpInputs" @onChange="onWpInputs" />
             </template>
         </FormCard>
         <FormCard title="History Options">
@@ -21,7 +21,7 @@
             </template>
         </FormCard>
         <div v-for="step in model.steps">
-            <WorkflowRunToolStep v-if="step.step_type == 'tool'" :model="step" />
+            <WorkflowRunToolStep v-if="step.step_type == 'tool'" :model="step" :placeholderParams="wpData" />
             <WorkflowRunDefaultStep v-else :model="step" />
         </div>
     </div>
@@ -53,6 +53,7 @@ export default {
     },
     data() {
         return {
+            wpData: {},
             historyInputs: [
                 {
                     type: "conditional",
@@ -122,6 +123,9 @@ export default {
         },
     },
     methods: {
+        onWpInputs(data) {
+            this.wpData = data;
+        },
         execute() {
             this.runForm.execute();
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index a1a835ed8b1..6efab0c1f58 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,15 +1,14 @@
 <template>
     <div>
-        <FormCard :title="title" icon="fa-wrench" :collapsible="true">
+        <FormCard :title="title" icon="fa-wrench" :collapsible="false">
             <template v-slot:body>
-                <Form :inputs="model.inputs" />
+                <Form :inputs="model.inputs" :placeholderParams="placeholderParams" />
             </template>
         </FormCard>
     </div>
 </template>
 
 <script>
-import { getGalaxyInstance } from "app";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
 
@@ -23,18 +22,18 @@ export default {
             type: Object,
             required: true,
         },
+        placeholderParams: {
+            type: Object,
+            default: null,
+        },
     },
     data() {
         return {};
     },
-    created() {
-        console.log(this.model);
-    },
     computed: {
         title() {
             return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
         },
     },
-    methods: {},
 };
 </script>

From adc3e6e821666c9a6df2f63e4b57f917ab34a57f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 21 Jun 2021 15:57:38 -0400
Subject: [PATCH 175/452] Trigger change when workflow parameters change

---
 client/src/components/Form/FormDisplay.vue | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index af63a90c0df..f3e6ef26015 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -77,6 +77,7 @@ export default {
         },
         placeholderParams() {
             this.onPlaceholderParams();
+            this.onChange();
         },
     },
     mounted() {

From 454037ba5fc9221f1bfd0ff7d5dc76accf5b0107 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 21 Jun 2021 16:39:30 -0400
Subject: [PATCH 176/452] Preliminary update mechanism for tool steps

---
 client/src/components/Form/FormCard.vue       |  2 +-
 client/src/components/Form/FormDisplay.vue    | 58 ++++++++++++++-----
 .../Workflow/Run/WorkflowRunDefaultStep.vue   |  8 ++-
 .../Workflow/Run/WorkflowRunForm.vue          | 15 ++++-
 .../Workflow/Run/WorkflowRunToolStep.vue      | 13 ++++-
 5 files changed, 76 insertions(+), 20 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 95a88fdc53a..3cd4a54a44b 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -24,7 +24,7 @@
                 </span>
             </a>
         </div>
-        <div v-if="!collapsed" class="portlet-content">
+        <div v-show="!collapsed" class="portlet-content">
             <slot name="body" />
         </div>
     </div>
diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index f3e6ef26015..aa54f1d9d4e 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -5,6 +5,7 @@
 <script>
 import Form from "mvc/form/form-view";
 import FormData from "mvc/form/form-data";
+import { isDataStep } from "components/Workflow/Run/model";
 
 export default {
     props: {
@@ -40,7 +41,11 @@ export default {
             type: Object,
             default: null,
         },
-        placeholderParams: {
+        replaceParams: {
+            type: Object,
+            default: null,
+        },
+        replaceData: {
             type: Object,
             default: null,
         },
@@ -75,8 +80,12 @@ export default {
                 }
             });
         },
-        placeholderParams() {
-            this.onPlaceholderParams();
+        replaceParams() {
+            this.onReplaceParams();
+            this.onChange();
+        },
+        replaceData() {
+            this.onReplaceParams();
             this.onChange();
         },
     },
@@ -134,23 +143,46 @@ export default {
         },
     },
     methods: {
-        onPlaceholderParams() {
+        onReplaceParams() {
             this.params = {};
             FormData.visitInputs(this.inputs, (input, name) => {
                 this.params[name] = input;
             });
             _.each(this.params, (input, name) => {
-                if (input.wp_linked) {
+                if (input.wp_linked || input.step_linked) {
                     var field = this.form.field_list[this.form.data.match(name)];
                     if (field) {
                         var new_value;
-                        new_value = input.value;
-                        var re = /\$\{(.+?)\}/g;
-                        var match;
-                        while ((match = re.exec(input.value))) {
-                            var wp_value = this.placeholderParams[match[1]];
-                            if (wp_value) {
-                                new_value = new_value.split(match[0]).join(wp_value);
+                        if (input.step_linked) {
+                            new_value = { values: [] };
+                            _.each(input.step_linked, (source_step) => {
+                                if (isDataStep(source_step)) {
+                                    console.log(this.replaceData);
+                                    console.log(source_step.index);
+                                    var value = this.replaceData[source_step.index].input;
+                                    console.log(value);
+                                    if (value) {
+                                        _.each(value.values, (v) => {
+                                            new_value.values.push(v);
+                                        });
+                                    }
+                                }
+                            });
+                            if (!input.multiple && new_value.values.length > 0) {
+                                new_value = {
+                                    values: [new_value.values[0]],
+                                };
+                            }
+                        }
+                        if (input.wp_linked) {
+                            new_value = input.value;
+                            var re = /\$\{(.+?)\}/g;
+                            var match;
+                            while ((match = re.exec(input.value))) {
+                                var wp_value = this.replaceParams[match[1]];
+                                if (wp_value) {
+                                    new_value = new_value.split(match[0]).join(wp_value);
+                                }
                             }
                         }
                     }
@@ -177,7 +209,7 @@ export default {
                         this.onChange();
                     },
                 });
-                this.onPlaceholderParams();
+                this.onReplaceParams();
                 this.onChange();
             });
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 9176bbf68e4..2d18e9ba279 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -2,7 +2,7 @@
     <div>
         <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true">
             <template v-slot:body>
-                <Form :inputs="inputs" />
+                <Form :inputs="inputs" @onChange="onChange" />
             </template>
         </FormCard>
     </div>
@@ -56,6 +56,10 @@ export default {
             }
         },
     },
-    methods: {},
+    methods: {
+        onChange(data) {
+            this.$emit("onChange", this.model.index, data);
+        },
+    },
 };
 </script>
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 133fcbf25cc..c34516453c7 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -21,13 +21,20 @@
             </template>
         </FormCard>
         <div v-for="step in model.steps">
-            <WorkflowRunToolStep v-if="step.step_type == 'tool'" :model="step" :placeholderParams="wpData" />
-            <WorkflowRunDefaultStep v-else :model="step" />
+            <WorkflowRunToolStep
+                v-if="step.step_type == 'tool'"
+                :model="step"
+                :replaceData="stepData"
+                :replaceParams="wpData"
+                @onChange="onStepInputs"
+            />
+            <WorkflowRunDefaultStep v-else :model="step" @onChange="onStepInputs" />
         </div>
     </div>
 </template>
 
 <script>
+import Vue from "vue";
 import { getGalaxyInstance } from "app";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
@@ -53,6 +60,7 @@ export default {
     },
     data() {
         return {
+            stepData: {},
             wpData: {},
             historyInputs: [
                 {
@@ -123,6 +131,9 @@ export default {
         },
     },
     methods: {
+        onStepInputs(stepId, data) {
+            Vue.set(this.stepData, stepId, data);
+        },
         onWpInputs(data) {
             this.wpData = data;
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 6efab0c1f58..f294e5f83db 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -2,7 +2,7 @@
     <div>
         <FormCard :title="title" icon="fa-wrench" :collapsible="false">
             <template v-slot:body>
-                <Form :inputs="model.inputs" :placeholderParams="placeholderParams" />
+                <Form :inputs="model.inputs" :replaceParams="replaceParams" :replaceData="replaceData" @onChange="onChange" />
             </template>
         </FormCard>
     </div>
@@ -22,7 +22,11 @@ export default {
             type: Object,
             required: true,
         },
-        placeholderParams: {
+        replaceParams: {
+            type: Object,
+            default: null,
+        },
+        replaceData: {
             type: Object,
             default: null,
         },
@@ -35,5 +39,10 @@ export default {
             return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
         },
     },
+    methods: {
+        onChange(data) {
+            this.$emit("onChange", this.model.index, data);
+        },
+    },
 };
 </script>

From 9ebe3925fef802b44e8dee49c5b834859f30e6c2 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 22 Jun 2021 20:33:24 -0400
Subject: [PATCH 177/452] Handle connected input change triggers to tool step
 states

---
 client/src/components/Form/FormDisplay.vue    |  3 ---
 .../Workflow/Run/WorkflowRunForm.vue          | 10 ++++++---
 .../Workflow/Run/WorkflowRunToolStep.vue      | 22 +++++++++++++++++--
 .../src/components/Workflow/Run/services.js   | 19 ++++++++++++++++
 4 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index aa54f1d9d4e..bbb5b056cef 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -157,10 +157,7 @@ export default {
                             new_value = { values: [] };
                             _.each(input.step_linked, (source_step) => {
                                 if (isDataStep(source_step)) {
-                                    console.log(this.replaceData);
-                                    console.log(source_step.index);
                                     var value = this.replaceData[source_step.index].input;
-                                    console.log(value);
                                     if (value) {
                                         _.each(value.values, (v) => {
                                             new_value.values.push(v);
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index c34516453c7..e3ec4a3a42f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -26,9 +26,9 @@
                 :model="step"
                 :replaceData="stepData"
                 :replaceParams="wpData"
-                @onChange="onStepInputs"
+                @onChange="onToolStepInputs"
             />
-            <WorkflowRunDefaultStep v-else :model="step" @onChange="onStepInputs" />
+            <WorkflowRunDefaultStep v-else :model="step" @onChange="onDefaultStepInputs" />
         </div>
     </div>
 </template>
@@ -131,8 +131,12 @@ export default {
         },
     },
     methods: {
-        onStepInputs(stepId, data) {
+        onDefaultStepInputs(stepId, data) {
             Vue.set(this.stepData, stepId, data);
+            this.stepData = Object.assign({}, this.stepData);
+        },
+        onToolStepInputs(stepId, data) {
+            this.stepData[stepId] = data;
         },
         onWpInputs(data) {
             this.wpData = data;
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index f294e5f83db..817d2a1967a 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -2,7 +2,13 @@
     <div>
         <FormCard :title="title" icon="fa-wrench" :collapsible="false">
             <template v-slot:body>
-                <Form :inputs="model.inputs" :replaceParams="replaceParams" :replaceData="replaceData" @onChange="onChange" />
+                <Form
+                    :inputs="model.inputs"
+                    :form-config="formConfig"
+                    :replaceParams="replaceParams"
+                    :replaceData="replaceData"
+                    @onChange="onChange"
+                />
             </template>
         </FormCard>
     </div>
@@ -11,6 +17,7 @@
 <script>
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
+import { getTool } from "./services";
 
 export default {
     components: {
@@ -32,7 +39,9 @@ export default {
         },
     },
     data() {
-        return {};
+        return {
+            formConfig: {},
+        };
     },
     computed: {
         title() {
@@ -41,6 +50,15 @@ export default {
     },
     methods: {
         onChange(data) {
+            getTool(this.model.id, this.model.version, data).then(
+                (formConfig) => {
+                    this.formConfig = formConfig;
+                },
+                (response) => {
+                    //Galaxy.emit.debug("tool-form-composite::postchange()", "Refresh request failed.", response);
+                    //process.reject();
+                }
+            );
             this.$emit("onChange", this.model.index, data);
         },
     },
diff --git a/client/src/components/Workflow/Run/services.js b/client/src/components/Workflow/Run/services.js
index 58122884002..a70a1c342ec 100644
--- a/client/src/components/Workflow/Run/services.js
+++ b/client/src/components/Workflow/Run/services.js
@@ -31,3 +31,22 @@ export async function invokeWorkflow(workflowId, invocationData) {
     const response = await axios.post(url, invocationData);
     return response.data;
 }
+
+/**
+ * Request tool step data.
+ *
+ * @param {String} workflowId - (Stored?) Workflow ID to fetch data for.
+ */
+export async function getTool(toolId, toolVersion, toolInputs) {
+    const requestData = {
+        tool_id: toolId,
+        tool_version: toolVersion,
+        inputs: $.extend(true, {}, toolInputs),
+    };
+    try {
+        const { data } = await axios.post(`${getAppRoot()}api/tools/${toolId}/build`, requestData);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}

From 24bd44a737b791b8866e9dfd5df85369dc2d7ba2 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 22 Jun 2021 20:42:11 -0400
Subject: [PATCH 178/452] Add error handling, adjust form message style

---
 client/src/components/Form/FormMessage.vue               | 2 +-
 client/src/components/Tool/ToolCard.vue                  | 4 ++--
 .../src/components/Workflow/Editor/Forms/FormDefault.vue | 2 +-
 .../src/components/Workflow/Run/WorkflowRunToolStep.vue  | 9 ++++++---
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/client/src/components/Form/FormMessage.vue b/client/src/components/Form/FormMessage.vue
index 3d8b65ac7ed..2cffe77fc4f 100644
--- a/client/src/components/Form/FormMessage.vue
+++ b/client/src/components/Form/FormMessage.vue
@@ -1,5 +1,5 @@
 <template>
-    <b-alert :variant="variant" :show="showAlert">
+    <b-alert class="mt-2" :variant="variant" :show="showAlert">
         {{ message | l }}
     </b-alert>
 </template>
diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index 8c276fb16cd..1f06b2b9e85 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -88,8 +88,8 @@
                 </div>
             </div>
             <div class="portlet-content">
-                <FormMessage class="mt-2" :message="errorText" variant="danger" :persistent="true" />
-                <FormMessage class="mt-2" :message="messageText" :variant="messageVariant" />
+                <FormMessage :message="errorText" variant="danger" :persistent="true" />
+                <FormMessage :message="messageText" :variant="messageVariant" />
                 <slot name="body" />
             </div>
         </div>
diff --git a/client/src/components/Workflow/Editor/Forms/FormDefault.vue b/client/src/components/Workflow/Editor/Forms/FormDefault.vue
index 226b2a11a82..f4637d50bbd 100644
--- a/client/src/components/Workflow/Editor/Forms/FormDefault.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormDefault.vue
@@ -27,7 +27,7 @@
             </b-button>
         </template>
         <template v-slot:body>
-            <FormMessage class="mt-2" :message="errorText" variant="danger" :persistent="true" />
+            <FormMessage :message="errorText" variant="danger" :persistent="true" />
             <FormElement
                 id="__label"
                 :value="node.label"
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 817d2a1967a..7c4e1f227c2 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -2,6 +2,7 @@
     <div>
         <FormCard :title="title" icon="fa-wrench" :collapsible="false">
             <template v-slot:body>
+                <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <Form
                     :inputs="model.inputs"
                     :form-config="formConfig"
@@ -16,6 +17,7 @@
 
 <script>
 import Form from "components/Form/Form";
+import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
 import { getTool } from "./services";
 
@@ -23,6 +25,7 @@ export default {
     components: {
         Form,
         FormCard,
+        FormMessage,
     },
     props: {
         model: {
@@ -41,6 +44,7 @@ export default {
     data() {
         return {
             formConfig: {},
+            errorText: null,
         };
     },
     computed: {
@@ -54,9 +58,8 @@ export default {
                 (formConfig) => {
                     this.formConfig = formConfig;
                 },
-                (response) => {
-                    //Galaxy.emit.debug("tool-form-composite::postchange()", "Refresh request failed.", response);
-                    //process.reject();
+                (errorText) => {
+                    this.errorText = errorText;
                 }
             );
             this.$emit("onChange", this.model.index, data);

From dbba34b53df105bf97fd3ef9686ab7d89f9f56cc Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 22 Jun 2021 20:45:10 -0400
Subject: [PATCH 179/452] Allow collapsing tool steps

---
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 7c4e1f227c2..97886cff31a 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="title" icon="fa-wrench" :collapsible="false">
+        <FormCard :title="title" icon="fa-wrench" :collapsible="true">
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <Form

From ae7e7ab4975167a419524b76442e92a24ccb13d1 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 10:29:29 -0400
Subject: [PATCH 180/452] Refactor parameter replacement handling

---
 client/src/components/Form/FormDisplay.vue    | 57 +++++--------------
 .../Workflow/Run/WorkflowRunForm.vue          |  2 +-
 .../Workflow/Run/WorkflowRunToolStep.vue      | 39 ++++++++++++-
 3 files changed, 52 insertions(+), 46 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index bbb5b056cef..42f4476f56d 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -85,7 +85,7 @@ export default {
             this.onChange();
         },
         replaceData() {
-            this.onReplaceParams();
+            //this.onReplaceParams();
             this.onChange();
         },
     },
@@ -144,50 +144,19 @@ export default {
     },
     methods: {
         onReplaceParams() {
-            this.params = {};
-            FormData.visitInputs(this.inputs, (input, name) => {
-                this.params[name] = input;
-            });
-            _.each(this.params, (input, name) => {
-                if (input.wp_linked || input.step_linked) {
-                    var field = this.form.field_list[this.form.data.match(name)];
-                    if (field) {
-                        var new_value;
-                        if (input.step_linked) {
-                            new_value = { values: [] };
-                            _.each(input.step_linked, (source_step) => {
-                                if (isDataStep(source_step)) {
-                                    var value = this.replaceData[source_step.index].input;
-                                    if (value) {
-                                        _.each(value.values, (v) => {
-                                            new_value.values.push(v);
-                                        });
-                                    }
-                                }
-                            });
-                            if (!input.multiple && new_value.values.length > 0) {
-                                new_value = {
-                                    values: [new_value.values[0]],
-                                };
-                            }
-                        }
-                        if (input.wp_linked) {
-                            new_value = input.value;
-                            var re = /\$\{(.+?)\}/g;
-                            var match;
-                            while ((match = re.exec(input.value))) {
-                                var wp_value = this.replaceParams[match[1]];
-                                if (wp_value) {
-                                    new_value = new_value.split(match[0]).join(wp_value);
-                                }
-                            }
-                        }
+            if (this.replaceParams) {
+                this.params = {};
+                FormData.visitInputs(this.inputs, (input, name) => {
+                    this.params[name] = input;
+                });
+                _.each(this.params, (input, name) => {
+                    const newValue = this.replaceParams[name];
+                    if (newValue) {
+                        const field = this.form.field_list[this.form.data.match(name)];
+                        field.value(newValue);
                     }
-                    if (new_value !== undefined) {
-                        field.value(new_value);
-                    }
-                }
-            });
+                });
+            }
         },
         onChange() {
             this.formData = this.form.data.create();
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index e3ec4a3a42f..68ce88851cd 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -25,7 +25,7 @@
                 v-if="step.step_type == 'tool'"
                 :model="step"
                 :replaceData="stepData"
-                :replaceParams="wpData"
+                :wpData="wpData"
                 @onChange="onToolStepInputs"
             />
             <WorkflowRunDefaultStep v-else :model="step" @onChange="onDefaultStepInputs" />
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 97886cff31a..5d479c665d9 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -19,6 +19,7 @@
 import Form from "components/Form/Form";
 import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
+import FormData from "mvc/form/form-data";
 import { getTool } from "./services";
 
 export default {
@@ -32,7 +33,7 @@ export default {
             type: Object,
             required: true,
         },
-        replaceParams: {
+        wpData: {
             type: Object,
             default: null,
         },
@@ -44,15 +45,50 @@ export default {
     data() {
         return {
             formConfig: {},
+            formData: {},
+            replaceParams: {},
             errorText: null,
         };
     },
+    watch: {
+        wpData() {
+            this.onReplaceParams();
+        },
+    },
     computed: {
         title() {
             return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
         },
     },
     methods: {
+        onReplaceParams() {
+            const params = {};
+            FormData.visitInputs(this.model.inputs, (input, name) => {
+                params[name] = input;
+            });
+            const values = this.formData;
+            this.replaceParams = {};
+            _.each(params, (input, name) => {
+                if (input.wp_linked) {
+                    let new_value;
+                    if (input.wp_linked) {
+                        new_value = input.value;
+                        var re = /\$\{(.+?)\}/g;
+                        var match;
+                        while ((match = re.exec(input.value))) {
+                            var wp_value = this.wpData[match[1]];
+                            if (wp_value) {
+                                new_value = new_value.split(match[0]).join(wp_value);
+                            }
+                        }
+                    }
+                    if (new_value !== undefined) {
+                        this.replaceParams[name] = new_value;
+                    }
+                }
+            });
+            console.log(this.replaceParams);
+        },
         onChange(data) {
             getTool(this.model.id, this.model.version, data).then(
                 (formConfig) => {
@@ -62,6 +98,7 @@ export default {
                     this.errorText = errorText;
                 }
             );
+            this.formData = data;
             this.$emit("onChange", this.model.index, data);
         },
     },

From 1697f1ecd09e5422cea23a7d09cc215175b9ec2a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 10:43:55 -0400
Subject: [PATCH 181/452] Add linked dataset to revised parameter replacement
 handling

---
 client/src/components/Form/FormDisplay.vue    |  9 -----
 .../Workflow/Run/WorkflowRunForm.vue          |  4 +-
 .../Workflow/Run/WorkflowRunToolStep.vue      | 39 ++++++++++++++-----
 3 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 42f4476f56d..2cda9dd8409 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -5,7 +5,6 @@
 <script>
 import Form from "mvc/form/form-view";
 import FormData from "mvc/form/form-data";
-import { isDataStep } from "components/Workflow/Run/model";
 
 export default {
     props: {
@@ -45,10 +44,6 @@ export default {
             type: Object,
             default: null,
         },
-        replaceData: {
-            type: Object,
-            default: null,
-        },
     },
     data() {
         return {
@@ -84,10 +79,6 @@ export default {
             this.onReplaceParams();
             this.onChange();
         },
-        replaceData() {
-            //this.onReplaceParams();
-            this.onChange();
-        },
     },
     mounted() {
         this.onRender();
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 68ce88851cd..763caa0b7c9 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -24,8 +24,8 @@
             <WorkflowRunToolStep
                 v-if="step.step_type == 'tool'"
                 :model="step"
-                :replaceData="stepData"
-                :wpData="wpData"
+                :step-data="stepData"
+                :wp-data="wpData"
                 @onChange="onToolStepInputs"
             />
             <WorkflowRunDefaultStep v-else :model="step" @onChange="onDefaultStepInputs" />
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 5d479c665d9..dcd93bbecd2 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -6,8 +6,7 @@
                 <Form
                     :inputs="model.inputs"
                     :form-config="formConfig"
-                    :replaceParams="replaceParams"
-                    :replaceData="replaceData"
+                    :replace-params="replaceParams"
                     @onChange="onChange"
                 />
             </template>
@@ -20,6 +19,7 @@ import Form from "components/Form/Form";
 import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
 import FormData from "mvc/form/form-data";
+import { isDataStep } from "components/Workflow/Run/model";
 import { getTool } from "./services";
 
 export default {
@@ -37,7 +37,7 @@ export default {
             type: Object,
             default: null,
         },
-        replaceData: {
+        stepData: {
             type: Object,
             default: null,
         },
@@ -51,6 +51,9 @@ export default {
         };
     },
     watch: {
+        stepData() {
+            this.onReplaceParams();
+        },
         wpData() {
             this.onReplaceParams();
         },
@@ -69,21 +72,39 @@ export default {
             const values = this.formData;
             this.replaceParams = {};
             _.each(params, (input, name) => {
-                if (input.wp_linked) {
-                    let new_value;
+                if (input.wp_linked || input.step_linked) {
+                    let newValue;
+                    if (input.step_linked) {
+                        newValue = { values: [] };
+                        _.each(input.step_linked, (source_step) => {
+                            if (isDataStep(source_step)) {
+                                var value = this.stepData[source_step.index].input;
+                                if (value) {
+                                    _.each(value.values, (v) => {
+                                        newValue.values.push(v);
+                                    });
+                                }
+                            }
+                        });
+                        if (!input.multiple && newValue.values.length > 0) {
+                            newValue = {
+                                values: [newValue.values[0]],
+                            };
+                        }
+                    }
                     if (input.wp_linked) {
-                        new_value = input.value;
+                        newValue = input.value;
                         var re = /\$\{(.+?)\}/g;
                         var match;
                         while ((match = re.exec(input.value))) {
                             var wp_value = this.wpData[match[1]];
                             if (wp_value) {
-                                new_value = new_value.split(match[0]).join(wp_value);
+                                newValue = newValue.split(match[0]).join(wp_value);
                             }
                         }
                     }
-                    if (new_value !== undefined) {
-                        this.replaceParams[name] = new_value;
+                    if (newValue !== undefined) {
+                        this.replaceParams[name] = newValue;
                     }
                 }
             });

From 3fbf3ff929bbf406e62710c26527c3dafa0d8f19 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 10:47:35 -0400
Subject: [PATCH 182/452] Remove debug statements, trigger change event for
 replacement values conditionally

---
 client/src/components/Form/FormDisplay.vue                 | 3 +--
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 2cda9dd8409..620da634c24 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -77,7 +77,6 @@ export default {
         },
         replaceParams() {
             this.onReplaceParams();
-            this.onChange();
         },
     },
     mounted() {
@@ -147,6 +146,7 @@ export default {
                         field.value(newValue);
                     }
                 });
+                this.onChange();
             }
         },
         onChange() {
@@ -166,7 +166,6 @@ export default {
                         this.onChange();
                     },
                 });
-                this.onReplaceParams();
                 this.onChange();
             });
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index dcd93bbecd2..3466e0bbc83 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -108,7 +108,6 @@ export default {
                     }
                 }
             });
-            console.log(this.replaceParams);
         },
         onChange(data) {
             getTool(this.model.id, this.model.version, data).then(

From 86001732d7df73598b1d9e7f6f2daf9b1036658b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 10:55:00 -0400
Subject: [PATCH 183/452] Move input visitor helper to Form component utilities

---
 client/src/components/Form/FormDisplay.vue    |  4 +-
 client/src/components/Form/utilities.js       | 42 +++++++++++++++++
 .../Workflow/Run/WorkflowRunToolStep.vue      |  4 +-
 client/src/mvc/form/form-data.js              | 46 +------------------
 4 files changed, 47 insertions(+), 49 deletions(-)
 create mode 100644 client/src/components/Form/utilities.js

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 620da634c24..85e2f754a2c 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -4,7 +4,7 @@
 
 <script>
 import Form from "mvc/form/form-view";
-import FormData from "mvc/form/form-data";
+import { visitInputs } from "components/Form/utilities";
 
 export default {
     props: {
@@ -136,7 +136,7 @@ export default {
         onReplaceParams() {
             if (this.replaceParams) {
                 this.params = {};
-                FormData.visitInputs(this.inputs, (input, name) => {
+                visitInputs(this.inputs, (input, name) => {
                     this.params[name] = input;
                 });
                 _.each(this.params, (input, name) => {
diff --git a/client/src/components/Form/utilities.js b/client/src/components/Form/utilities.js
new file mode 100644
index 00000000000..669c07f18db
--- /dev/null
+++ b/client/src/components/Form/utilities.js
@@ -0,0 +1,42 @@
+/** Visits tool inputs
+ * @param{dict}   inputs    - Nested dictionary of input elements
+ * @param{dict}   callback  - Called with the mapped dictionary object and corresponding model node
+ */
+ export var visitInputs = (inputs, callback, prefix, context) => {
+    context = $.extend({}, context);
+    _.each(inputs, (input) => {
+        if (input && input.type && input.name) {
+            context[input.name] = input;
+        }
+    });
+    for (var key in inputs) {
+        var node = inputs[key];
+        node.name = node.name || key;
+        var name = prefix ? `${prefix}|${node.name}` : node.name;
+        switch (node.type) {
+            case "repeat":
+                _.each(node.cache, (cache, j) => {
+                    visitInputs(cache, callback, `${name}_${j}`, context);
+                });
+                break;
+            case "conditional":
+                if (node.test_param) {
+                    callback(node.test_param, `${name}|${node.test_param.name}`, context);
+                    var selectedCase = matchCase(node, node.test_param.value);
+                    if (selectedCase != -1) {
+                        visitInputs(node.cases[selectedCase].inputs, callback, name, context);
+                    } else {
+                        console.debug(`Form.utilities::visitInputs() - Invalid case for ${name}.`);
+                    }
+                } else {
+                    console.debug(`Form.utilities::visitInputs() - Conditional test parameter missing for ${name}.`);
+                }
+                break;
+            case "section":
+                visitInputs(node.inputs, callback, name, context);
+                break;
+            default:
+                callback(node, name, context);
+        }
+    }
+};
\ No newline at end of file
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 3466e0bbc83..374754d4688 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -18,7 +18,7 @@
 import Form from "components/Form/Form";
 import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
-import FormData from "mvc/form/form-data";
+import { visitInputs } from "components/Form/utilities";
 import { isDataStep } from "components/Workflow/Run/model";
 import { getTool } from "./services";
 
@@ -66,7 +66,7 @@ export default {
     methods: {
         onReplaceParams() {
             const params = {};
-            FormData.visitInputs(this.model.inputs, (input, name) => {
+            visitInputs(this.model.inputs, (input, name) => {
                 params[name] = input;
             });
             const values = this.formData;
diff --git a/client/src/mvc/form/form-data.js b/client/src/mvc/form/form-data.js
index 61f0778b8f8..63c68919716 100644
--- a/client/src/mvc/form/form-data.js
+++ b/client/src/mvc/form/form-data.js
@@ -1,7 +1,7 @@
 import _ from "underscore";
 import $ from "jquery";
 import Backbone from "backbone";
-import { getGalaxyInstance } from "app";
+import { visitInputs } from "components/Form/utilities";
 
 /* This class maps the form dom to an api compatible javascript dictionary. */
 export var Manager = Backbone.Model.extend({
@@ -202,50 +202,6 @@ export var matchCase = (input, value) => {
     return -1;
 };
 
-/** Visits tool inputs
- * @param{dict}   inputs    - Nested dictionary of input elements
- * @param{dict}   callback  - Called with the mapped dictionary object and corresponding model node
- */
-export var visitInputs = (inputs, callback, prefix, context) => {
-    context = $.extend({}, context);
-    _.each(inputs, (input) => {
-        if (input && input.type && input.name) {
-            context[input.name] = input;
-        }
-    });
-    for (var key in inputs) {
-        const Galaxy = getGalaxyInstance();
-        var node = inputs[key];
-        node.name = node.name || key;
-        var name = prefix ? `${prefix}|${node.name}` : node.name;
-        switch (node.type) {
-            case "repeat":
-                _.each(node.cache, (cache, j) => {
-                    visitInputs(cache, callback, `${name}_${j}`, context);
-                });
-                break;
-            case "conditional":
-                if (node.test_param) {
-                    callback(node.test_param, `${name}|${node.test_param.name}`, context);
-                    var selectedCase = matchCase(node, node.test_param.value);
-                    if (selectedCase != -1) {
-                        visitInputs(node.cases[selectedCase].inputs, callback, name, context);
-                    } else {
-                        Galaxy.emit.debug(`form-data::visitInputs() - Invalid case for ${name}.`);
-                    }
-                } else {
-                    Galaxy.emit.debug(`form-data::visitInputs() - Conditional test parameter missing for ${name}.`);
-                }
-                break;
-            case "section":
-                visitInputs(node.inputs, callback, name, context);
-                break;
-            default:
-                callback(node, name, context);
-        }
-    }
-};
-
 export default {
     Manager: Manager,
     visitInputs: visitInputs,

From 1dcac7dea1eed709594af38fe083f9bcd47c0a64 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 12:23:47 -0400
Subject: [PATCH 184/452] Restore initial collapsible state as in the current
 run form

---
 client/src/components/Form/FormCard.vue       |  6 +++++-
 client/src/components/Form/FormDisplay.vue    |  2 +-
 client/src/components/Form/utilities.js       |  4 ++--
 .../Workflow/Run/WorkflowRunDefaultStep.vue   |  2 +-
 .../Workflow/Run/WorkflowRunForm.vue          |  3 +--
 .../Workflow/Run/WorkflowRunToolStep.vue      | 20 ++++++++++---------
 6 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 3cd4a54a44b..c24d65cffcf 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -48,10 +48,14 @@ export default {
             type: Boolean,
             default: false,
         },
+        initialCollapse: {
+            type: Boolean,
+            default: false,
+        },
     },
     data() {
         return {
-            collapsed: this.collapsible,
+            collapsed: this.initialCollapse,
         };
     },
     computed: {
diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 85e2f754a2c..b4415e5a4df 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -146,7 +146,7 @@ export default {
                         field.value(newValue);
                     }
                 });
-                this.onChange();
+                this.form.trigger("change");
             }
         },
         onChange() {
diff --git a/client/src/components/Form/utilities.js b/client/src/components/Form/utilities.js
index 669c07f18db..64f90c463c4 100644
--- a/client/src/components/Form/utilities.js
+++ b/client/src/components/Form/utilities.js
@@ -2,7 +2,7 @@
  * @param{dict}   inputs    - Nested dictionary of input elements
  * @param{dict}   callback  - Called with the mapped dictionary object and corresponding model node
  */
- export var visitInputs = (inputs, callback, prefix, context) => {
+export var visitInputs = (inputs, callback, prefix, context) => {
     context = $.extend({}, context);
     _.each(inputs, (input) => {
         if (input && input.type && input.name) {
@@ -39,4 +39,4 @@
                 callback(node, name, context);
         }
     }
-};
\ No newline at end of file
+};
diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 2d18e9ba279..eb7f368c8d1 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true">
+        <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :initial-collapse="false">
             <template v-slot:body>
                 <Form :inputs="inputs" @onChange="onChange" />
             </template>
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 763caa0b7c9..147a73baf6f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -34,7 +34,6 @@
 </template>
 
 <script>
-import Vue from "vue";
 import { getGalaxyInstance } from "app";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
@@ -132,7 +131,7 @@ export default {
     },
     methods: {
         onDefaultStepInputs(stepId, data) {
-            Vue.set(this.stepData, stepId, data);
+            this.stepData[stepId] = data;
             this.stepData = Object.assign({}, this.stepData);
         },
         onToolStepInputs(stepId, data) {
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 374754d4688..ede56f55bc8 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="title" icon="fa-wrench" :collapsible="true">
+        <FormCard :title="title" icon="fa-wrench" :collapsible="true" :initial-collapse="model.collapsed">
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <Form
@@ -52,6 +52,7 @@ export default {
     },
     watch: {
         stepData() {
+            console.log(this.model.id);
             this.onReplaceParams();
         },
         wpData() {
@@ -76,9 +77,10 @@ export default {
                     let newValue;
                     if (input.step_linked) {
                         newValue = { values: [] };
-                        _.each(input.step_linked, (source_step) => {
-                            if (isDataStep(source_step)) {
-                                var value = this.stepData[source_step.index].input;
+                        _.each(input.step_linked, (sourceStep) => {
+                            if (isDataStep(sourceStep)) {
+                                const sourceData = this.stepData[sourceStep.index];
+                                const value = sourceData && sourceData.input;
                                 if (value) {
                                     _.each(value.values, (v) => {
                                         newValue.values.push(v);
@@ -94,12 +96,12 @@ export default {
                     }
                     if (input.wp_linked) {
                         newValue = input.value;
-                        var re = /\$\{(.+?)\}/g;
-                        var match;
+                        const re = /\$\{(.+?)\}/g;
+                        let match;
                         while ((match = re.exec(input.value))) {
-                            var wp_value = this.wpData[match[1]];
-                            if (wp_value) {
-                                newValue = newValue.split(match[0]).join(wp_value);
+                            const wpValue = this.wpData[match[1]];
+                            if (wpValue) {
+                                newValue = newValue.split(match[0]).join(wpValue);
                             }
                         }
                     }

From 370f8a0b0a91b4890e1a0338e21972297745c40d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 12:59:40 -0400
Subject: [PATCH 185/452] Consolidate redundant run button component, refactor
 without using references

---
 client/src/components/WaitButton.vue          | 67 -------------------
 .../components/Workflow/Run/WorkflowRun.vue   | 60 ++---------------
 .../Workflow/Run/WorkflowRunForm.vue          | 14 ++--
 .../Workflow/Run/WorkflowRunFormSimple.vue    | 15 +++--
 client/src/style/scss/base.scss               |  5 --
 5 files changed, 24 insertions(+), 137 deletions(-)
 delete mode 100644 client/src/components/WaitButton.vue

diff --git a/client/src/components/WaitButton.vue b/client/src/components/WaitButton.vue
deleted file mode 100644
index dd19137cedd..00000000000
--- a/client/src/components/WaitButton.vue
+++ /dev/null
@@ -1,67 +0,0 @@
-/** Button variant with waiting and progress tracking. */
-<template>
-    <button class="ui-button-default" :class="buttonClass" type="button" @click="$emit('click')">
-        <i class="icon fa" :class="iconClass"> </i>
-        <span class="title"> {{ currentText }} </span>
-        <div class="progress" v-if="percentage != -1">
-            <div class="progress-bar" :style="{ width: progressWidth }"></div>
-        </div>
-    </button>
-</template>
-
-<script>
-export default {
-    props: {
-        title: {
-            type: String,
-            default: "",
-        },
-        icon: {
-            type: String,
-            default: "fa-play",
-        },
-        percentage: {
-            type: Number,
-            default: -1,
-        },
-        waiting: {
-            type: Boolean,
-            default: false,
-        },
-        waitText: {
-            type: String,
-            default: "Sending...",
-        },
-        variant: {
-            type: String,
-            default: "info",
-        },
-    },
-    computed: {
-        progressWidth() {
-            return `${this.percentage}%`;
-        },
-        buttonClass() {
-            if (this.waiting) {
-                return "disabled";
-            } else {
-                return `btn btn-${this.variant}`;
-            }
-        },
-        iconClass() {
-            if (this.waiting) {
-                return "fa-spinner fa-spin mr-1";
-            } else {
-                return this.icon + " mr-1";
-            }
-        },
-        currentText() {
-            if (this.waiting) {
-                return this.waitText;
-            } else {
-                return this.title;
-            }
-        },
-    },
-};
-</script>
diff --git a/client/src/components/Workflow/Run/WorkflowRun.vue b/client/src/components/Workflow/Run/WorkflowRun.vue
index c7d1ee31425..d90fb2242e9 100644
--- a/client/src/components/Workflow/Run/WorkflowRun.vue
+++ b/client/src/components/Workflow/Run/WorkflowRun.vue
@@ -7,11 +7,7 @@
             <b-alert v-if="loading" variant="info" show>
                 <loading-span message="Loading workflow run data" />
             </b-alert>
-            <workflow-run-success
-                v-else-if="invocations != null"
-                :invocations="invocations"
-                :workflow-name="workflowName"
-            />
+            <workflow-run-success v-else-if="!!invocations" :invocations="invocations" :workflow-name="workflowName" />
             <div v-else class="ui-form-composite">
                 <div class="ui-form-composite-messages mb-4">
                     <b-alert v-if="hasUpgradeMessages" variant="warning" show>
@@ -29,41 +25,15 @@
                         Workflow submission failed: {{ submissionError }}
                     </b-alert>
                 </div>
-                <!-- h4 as a class here looks odd but it was in the Backbone -->
-                <div class="ui-form-composite-header h4">
-                    <b>Workflow: {{ workflowName }}</b>
-                    <wait-button
-                        title="Run Workflow"
-                        id="run-workflow"
-                        variant="primary"
-                        :disabled="!runButtonEnabled"
-                        :waiting="!runButtonEnabled"
-                        :wait-text="runButtonWaitText"
-                        :percentage="runButtonPercentage"
-                        @click="execute"
-                    >
-                    </wait-button>
-                </div>
-                <workflow-run-form
-                    ref="runform"
+                <workflow-run-form-simple
+                    v-if="simpleForm"
                     :model="model"
-                    v-if="!simpleForm"
-                    :set-run-button-status="setRunButtonStatus"
+                    :target-history="simpleFormTargetHistory"
+                    :use-job-cache="simpleFormUseJobCache"
                     @submissionSuccess="handleInvocations"
+                    @showAdvanced="showAdvanced"
                 />
-                <div v-else>
-                    <workflow-run-form-simple
-                        ref="runform"
-                        :model="model"
-                        :set-run-button-status="setRunButtonStatus"
-                        :target-history="simpleFormTargetHistory"
-                        :use-job-cache="simpleFormUseJobCache"
-                        @submissionSuccess="handleInvocations"
-                        @submissionError="handleSubmissionError"
-                    />
-                    <!-- Options to default one way or the other, disable if admins want, etc.. -->
-                    <a href="#" @click="showAdvanced">Expand to full workflow form.</a>
-                </div>
+                <workflow-run-form v-else :model="model" @submissionSuccess="handleInvocations" />
             </div>
         </span>
     </span>
@@ -71,7 +41,6 @@
 
 <script>
 import { getRunData } from "./services";
-import WaitButton from "components/WaitButton";
 import LoadingSpan from "components/LoadingSpan";
 import WorkflowRunSuccess from "./WorkflowRunSuccess";
 import WorkflowRunForm from "./WorkflowRunForm";
@@ -82,7 +51,6 @@ import { errorMessageAsString } from "utils/simple-error";
 export default {
     components: {
         LoadingSpan,
-        WaitButton,
         WorkflowRunSuccess,
         WorkflowRunForm,
         WorkflowRunFormSimple,
@@ -109,10 +77,6 @@ export default {
             hasUpgradeMessages: false,
             hasStepVersionChanges: false,
             workflowName: "",
-            runForm: null,
-            runButtonEnabled: true,
-            runButtonWaitText: "",
-            runButtonPercentage: -1,
             invocations: null,
             simpleForm: null,
             submissionError: null,
@@ -158,20 +122,10 @@ export default {
                 this.loading = false;
             })
             .catch((response) => {
-                console.log(response);
                 this.error = errorMessageAsString(response);
             });
     },
     methods: {
-        execute() {
-            this.$refs.runform.execute();
-            this.submissionError = null;
-        },
-        setRunButtonStatus(enabled, waitText, percentage) {
-            this.runButtonEnabled = enabled;
-            this.runButtonWaitText = waitText;
-            this.runButtonPercentage = percentage;
-        },
         handleInvocations(invocations) {
             this.invocations = invocations;
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 147a73baf6f..dbc52458623 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,5 +1,9 @@
 <template>
     <div>
+        <div class="h4 clearfix mb-3">
+            <b>Workflow: {{ model.name }}</b>
+            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @click="onExecute" />
+        </div>
         <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
                 <Form :inputs="wpInputs" @onChange="onWpInputs" />
@@ -37,6 +41,7 @@
 import { getGalaxyInstance } from "app";
 import Form from "components/Form/Form";
 import FormCard from "components/Form/FormCard";
+import ButtonSpinner from "components/Common/ButtonSpinner";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
 import WorkflowRunToolStep from "./WorkflowRunToolStep";
 
@@ -44,6 +49,7 @@ export default {
     components: {
         Form,
         FormCard,
+        ButtonSpinner,
         WorkflowRunDefaultStep,
         WorkflowRunToolStep,
     },
@@ -52,10 +58,6 @@ export default {
             type: Object,
             required: true,
         },
-        setRunButtonStatus: {
-            type: Function,
-            required: true,
-        },
     },
     data() {
         return {
@@ -140,9 +142,7 @@ export default {
         onWpInputs(data) {
             this.wpData = data;
         },
-        execute() {
-            this.runForm.execute();
-        },
+        onExecute() {},
         handleInvocations(invocations) {
             this.$emit("submissionSuccess", invocations);
         },
diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
index b3809a92104..738da40c544 100644
--- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
@@ -1,22 +1,28 @@
 <template>
+    <div class="h4 clearfix mb-3">
+        <b>Workflow: {{ model.name }}</b>
+        <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @click="execute" />
+    </div>
     <div ref="form"></div>
+    <!-- Options to default one way or the other, disable if admins want, etc.. -->
+    <a href="#" @click="$emit('showAdvanced')">Expand to full workflow form.</a>
 </template>
 
 <script>
 import Form from "mvc/form/form-view";
+import ButtonSpinner from "components/Common/ButtonSpinner";
 import { invokeWorkflow } from "./services";
 import { isWorkflowInput } from "components/Workflow/constants";
 
 export default {
+    components: {
+        ButtonSpinner,
+    },
     props: {
         model: {
             type: Object,
             required: true,
         },
-        setRunButtonStatus: {
-            type: Function,
-            required: true,
-        },
         targetHistory: {
             type: String,
             default: "current",
@@ -32,7 +38,6 @@ export default {
             inputTypes: {},
         };
     },
-    computed: {},
     created() {
         this.form = new Form(this.formDefinition());
         this.$nextTick(() => {
diff --git a/client/src/style/scss/base.scss b/client/src/style/scss/base.scss
index f7b71f82d12..d516c1c8523 100644
--- a/client/src/style/scss/base.scss
+++ b/client/src/style/scss/base.scss
@@ -587,11 +587,6 @@ div.form-row-input label {
     margin-top: 5px;
 }
 
-#run-workflow {
-    @extend .float-right;
-    @extend .mt-3;
-}
-
 .form-row {
     select,
     textarea,

From e781bb4f32887217be1eee6ab40b0b25bacf9ba6 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 13:35:09 -0400
Subject: [PATCH 186/452] Use vue components in refactored simplified workflow
 run form, restore config parsing

---
 .../Workflow/Run/WorkflowRunFormSimple.vue    | 50 ++++++++-----------
 lib/galaxy/managers/configuration.py          |  1 +
 2 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
index 738da40c544..557da05e287 100644
--- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
@@ -1,15 +1,17 @@
 <template>
-    <div class="h4 clearfix mb-3">
-        <b>Workflow: {{ model.name }}</b>
-        <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @click="execute" />
+    <div>
+        <div class="h4 clearfix mb-3">
+            <b>Workflow: {{ model.name }}</b>
+            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @click="onExecute" />
+        </div>
+        <Form :inputs="formInputs" @onChange="onChange" />
+        <!-- Options to default one way or the other, disable if admins want, etc.. -->
+        <a href="#" @click="$emit('showAdvanced')">Expand to full workflow form.</a>
     </div>
-    <div ref="form"></div>
-    <!-- Options to default one way or the other, disable if admins want, etc.. -->
-    <a href="#" @click="$emit('showAdvanced')">Expand to full workflow form.</a>
 </template>
 
 <script>
-import Form from "mvc/form/form-view";
+import Form from "components/Form/Form";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import { invokeWorkflow } from "./services";
 import { isWorkflowInput } from "components/Workflow/constants";
@@ -17,6 +19,7 @@ import { isWorkflowInput } from "components/Workflow/constants";
 export default {
     components: {
         ButtonSpinner,
+        Form,
     },
     props: {
         model: {
@@ -34,21 +37,13 @@ export default {
     },
     data() {
         return {
-            form: null,
+            formData: {},
             inputTypes: {},
         };
     },
-    created() {
-        this.form = new Form(this.formDefinition());
-        this.$nextTick(() => {
-            const el = this.$refs["form"];
-            el.appendChild(this.form.$el[0]);
-        });
-    },
-    methods: {
-        formDefinition() {
+    computed: {
+        formInputs() {
             const inputs = [];
-
             // Add workflow parameters.
             Object.values(this.model.wpInputs).forEach((input) => {
                 const inputCopy = Object.assign({}, input);
@@ -57,7 +52,6 @@ export default {
                 inputs.push(inputCopy);
                 this.inputTypes[inputCopy.name] = "replacement_parameter";
             });
-
             // Add actual input modules.
             this.model.steps.forEach((step, i) => {
                 if (!isWorkflowInput(step.step_type)) {
@@ -73,18 +67,18 @@ export default {
                 inputs.push(stepAsInput);
                 this.inputTypes[stepName] = step.step_type;
             });
-
-            const def = {
-                inputs: inputs,
-            };
-            return def;
+            return inputs;
         },
-        execute() {
+    },
+    methods: {
+        onChange(data) {
+            this.formData = data;
+        },
+        onExecute() {
             const replacementParams = {};
             const inputs = {};
-            const formData = this.form.data.create();
-            for (const inputName in formData) {
-                const value = formData[inputName];
+            for (const inputName in this.formData) {
+                const value = this.formData[inputName];
                 const inputType = this.inputTypes[inputName];
                 if (inputType == "replacement_parameter") {
                     replacementParams[inputName] = value;
diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py
index 38aca4485cb..3e43f4320c4 100644
--- a/lib/galaxy/managers/configuration.py
+++ b/lib/galaxy/managers/configuration.py
@@ -172,6 +172,7 @@ class ConfigSerializer(base.ModelSerializer):
             'simplified_workflow_run_ui': _use_config,
             'simplified_workflow_run_ui_target_history': _use_config,
             'simplified_workflow_run_ui_job_cache': _use_config,
+            'simplified_workflow_run_ui': _use_config,
             'has_user_tool_filters': _defaults_to(False),
             # TODO: is there no 'correct' way to get an api url? controller='api', action='tools' is a hack
             # at any rate: the following works with path_prefix but is still brittle

From c9a496f5459460995538ee80be96ffde169b575c Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 13:59:30 -0400
Subject: [PATCH 187/452] Add error handling to simplified workflow form

---
 client/src/components/Workflow/Run/WorkflowRun.vue     |  2 +-
 .../components/Workflow/Run/WorkflowRunFormSimple.vue  | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRun.vue b/client/src/components/Workflow/Run/WorkflowRun.vue
index d90fb2242e9..c5410e918ea 100644
--- a/client/src/components/Workflow/Run/WorkflowRun.vue
+++ b/client/src/components/Workflow/Run/WorkflowRun.vue
@@ -86,6 +86,7 @@ export default {
     created() {
         getRunData(this.workflowId)
             .then((runData) => {
+                this.loading = false;
                 const model = new WorkflowRunModel(runData);
                 let simpleForm = this.preferSimpleForm;
                 if (simpleForm) {
@@ -119,7 +120,6 @@ export default {
                 this.hasUpgradeMessages = model.hasUpgradeMessages;
                 this.hasStepVersionChanges = model.hasStepVersionChanges;
                 this.workflowName = this.model.name;
-                this.loading = false;
             })
             .catch((response) => {
                 this.error = errorMessageAsString(response);
diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
index 557da05e287..19ff3de2d78 100644
--- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
@@ -2,8 +2,11 @@
     <div>
         <div class="h4 clearfix mb-3">
             <b>Workflow: {{ model.name }}</b>
-            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @click="onExecute" />
+            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @onClick="onExecute" />
         </div>
+        <b-alert v-if="errorText" class="mt-2" variant="danger" show>
+            {{ errorText | l }}
+        </b-alert>
         <Form :inputs="formInputs" @onChange="onChange" />
         <!-- Options to default one way or the other, disable if admins want, etc.. -->
         <a href="#" @click="$emit('showAdvanced')">Expand to full workflow form.</a>
@@ -15,6 +18,7 @@ import Form from "components/Form/Form";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import { invokeWorkflow } from "./services";
 import { isWorkflowInput } from "components/Workflow/constants";
+import { errorMessageAsString } from "utils/simple-error";
 
 export default {
     components: {
@@ -39,6 +43,7 @@ export default {
         return {
             formData: {},
             inputTypes: {},
+            errorText: null,
         };
     },
     computed: {
@@ -103,8 +108,7 @@ export default {
                     this.$emit("submissionSuccess", invocations);
                 })
                 .catch((error) => {
-                    console.log(error);
-                    this.$emit("submissionError", error);
+                    this.errorText = errorMessageAsString(error);
                 });
         },
     },

From 2865171fe4a205e3ca6c258df138d19c53fe12f9 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 23 Jun 2021 15:04:24 -0400
Subject: [PATCH 188/452] Add execution handler for regular run workflow form

---
 .../Workflow/Run/WorkflowRunForm.vue          | 125 +++++++++++++++++-
 1 file changed, 120 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index dbc52458623..9f789c0eafc 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -2,7 +2,7 @@
     <div>
         <div class="h4 clearfix mb-3">
             <b>Workflow: {{ model.name }}</b>
-            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @click="onExecute" />
+            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @onClick="onExecute" />
         </div>
         <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
@@ -11,17 +11,17 @@
         </FormCard>
         <FormCard title="History Options">
             <template v-slot:body>
-                <Form :inputs="historyInputs" />
+                <Form :inputs="historyInputs" @onChange="onHistoryInputs" />
             </template>
         </FormCard>
         <FormCard v-if="cacheInputsAvailable" title="Job re-use Options">
             <template v-slot:body>
-                <Form :inputs="cacheInputs" />
+                <Form :inputs="cacheInputs" @onChange="onCacheInputs" />
             </template>
         </FormCard>
         <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
             <template v-slot:body>
-                <Form :inputs="resourceInputs" />
+                <Form :inputs="resourceInputs" @onChange="onResourceInputs" />
             </template>
         </FormCard>
         <div v-for="step in model.steps">
@@ -63,6 +63,8 @@ export default {
         return {
             stepData: {},
             wpData: {},
+            historyData: {},
+            cacheData: {},
             historyInputs: [
                 {
                     type: "conditional",
@@ -139,10 +141,123 @@ export default {
         onToolStepInputs(stepId, data) {
             this.stepData[stepId] = data;
         },
+        onHistoryInputs(data) {
+            this.historyData = data;
+        },
+        onResourceInputs(data) {
+            this.resourceData = data;
+        },
         onWpInputs(data) {
             this.wpData = data;
         },
-        onExecute() {},
+        onCacheInputs(data) {
+            this.cacheData = data;
+        },
+        onExecute() {
+            const Galaxy = getGalaxyInstance();
+            const job_def = {
+                new_history_name: this.historyData["new_history|name"] ? this.historyData["new_history|name"] : null,
+                history_id: !this.historyData["new_history|name"] ? this.model.historyId : null,
+                resource_params: this.resourceData,
+                replacement_params: this.wpData,
+                use_cached_job: this.cacheData["use_cached_job|check"] === "true",
+                parameters: {},
+                // Tool form will submit flat maps for each parameter
+                // (e.g. "repeat_0|cond|param": "foo" instead of nested
+                // data structures).
+                parameters_normalized: true,
+                // Tool form always wants a list of invocations back
+                // so that inputs can be batched.
+                batch: true,
+            };
+            var validated = true;
+            /*for (var i in this.forms) {
+                var form = this.forms[i];
+                var job_inputs = form.data.create();
+                var step = this.steps[i];
+                var step_index = step.step_index;
+                form.trigger("reset");
+                for (var job_input_id in job_inputs) {
+                    var input_value = job_inputs[job_input_id];
+                    var input_id = form.data.match(job_input_id);
+                    var input_def = form.input_list[input_id];
+                    if (!input_def.step_linked) {
+                        if (isDataStep(step)) {
+                            validated = input_value && input_value.values && input_value.values.length > 0;
+                            if (!validated && input_def.optional) {
+                                validated = true;
+                            }
+                        } else {
+                            validated =
+                                input_def.optional ||
+                                (input_def.is_workflow && input_value !== "") ||
+                                (!input_def.is_workflow && input_value !== null);
+                        }
+                        if (!validated) {
+                            if (input_def.type == "hidden") {
+                                this.modal.show({
+                                    title: _l("Workflow submission failed"),
+                                    body: `Step ${step_index}: ${
+                                        step.step_label || step.step_name
+                                    } is in an invalid state. Please remove and re-add this step in the Workflow Editor.`,
+                                    buttons: {
+                                        Close: () => {
+                                            this.modal.hide();
+                                        },
+                                    },
+                                });
+                            }
+                            form.highlight(input_id);
+                            break;
+                        }
+                        job_def.parameters[step_index] = job_def.parameters[step_index] || {};
+                        job_def.parameters[step_index][job_input_id] = job_inputs[job_input_id];
+                    }
+                }
+                if (!validated) {
+                    break;
+                }
+            }
+            if (!validated) {
+                this._enabled(true);
+                Galaxy.emit.debug("tool-form-composite::submit()", "Validation failed.", job_def);
+            } else {
+                Galaxy.emit.debug("tool-form-composite::submit()", "Validation complete.", job_def);
+                invokeWorkflow(this.runWorkflowModel.workflowId, job_def)
+                    .then((invocations) => {
+                        Galaxy.emit.debug("tool-form-composite::submit", "Submission successful.", invocations);
+                        if (Array.isArray(invocations) && invocations.length > 0) {
+                            this.handleInvocations(invocations);
+                        } else {
+                            this.submissionErrorModal(job_def, invocations);
+                        }
+                        this._enabled(true);
+                    })
+                    .catch((response) => {
+                        // TODO: Is this the same response as the Utils post would
+                        // have had?
+                        Galaxy.emit.debug("tool-form-composite::submit", "Submission failed.", response);
+                        var input_found = false;
+                        if (response && response.err_data) {
+                            for (var i in this.forms) {
+                                var form = this.forms[i];
+                                var step_related_errors = response.err_data[form.model.get("step_index")];
+                                if (step_related_errors) {
+                                    var error_messages = form.data.matchResponse(step_related_errors);
+                                    for (var input_id in error_messages) {
+                                        form.highlight(input_id, error_messages[input_id]);
+                                        input_found = true;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        if (!input_found) {
+                            this.submissionErrorModal(job_def, response);
+                        }
+                        this._enabled(true);
+                    });*/
+        },
         handleInvocations(invocations) {
             this.$emit("submissionSuccess", invocations);
         },

From 61268ed38d872b9bc7c927ebe3154c9288337b78 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 8 Jul 2021 14:45:20 -0400
Subject: [PATCH 189/452] Adjust to renaming Form to FormDisplay component

---
 .../Workflow/Run/WorkflowRunDefaultStep.vue          |  6 +++---
 .../src/components/Workflow/Run/WorkflowRunForm.vue  | 12 ++++++------
 .../Workflow/Run/WorkflowRunFormSimple.vue           |  6 +++---
 .../components/Workflow/Run/WorkflowRunToolStep.vue  |  7 +++----
 4 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index eb7f368c8d1..9acbf2aa660 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -2,7 +2,7 @@
     <div>
         <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :initial-collapse="false">
             <template v-slot:body>
-                <Form :inputs="inputs" @onChange="onChange" />
+                <FormDisplay :inputs="inputs" @onChange="onChange" />
             </template>
         </FormCard>
     </div>
@@ -11,12 +11,12 @@
 <script>
 import { getGalaxyInstance } from "app";
 import WorkflowIcons from "components/Workflow/icons";
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";
 
 export default {
     components: {
-        Form,
+        FormDisplay,
         FormCard,
     },
     props: {
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 9f789c0eafc..0acdaaf1962 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -6,22 +6,22 @@
         </div>
         <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
-                <Form :inputs="wpInputs" @onChange="onWpInputs" />
+                <FormDisplay :inputs="wpInputs" @onChange="onWpInputs" />
             </template>
         </FormCard>
         <FormCard title="History Options">
             <template v-slot:body>
-                <Form :inputs="historyInputs" @onChange="onHistoryInputs" />
+                <FormDisplay :inputs="historyInputs" @onChange="onHistoryInputs" />
             </template>
         </FormCard>
         <FormCard v-if="cacheInputsAvailable" title="Job re-use Options">
             <template v-slot:body>
-                <Form :inputs="cacheInputs" @onChange="onCacheInputs" />
+                <FormDisplay :inputs="cacheInputs" @onChange="onCacheInputs" />
             </template>
         </FormCard>
         <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
             <template v-slot:body>
-                <Form :inputs="resourceInputs" @onChange="onResourceInputs" />
+                <FormDisplay :inputs="resourceInputs" @onChange="onResourceInputs" />
             </template>
         </FormCard>
         <div v-for="step in model.steps">
@@ -39,7 +39,7 @@
 
 <script>
 import { getGalaxyInstance } from "app";
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
@@ -47,7 +47,7 @@ import WorkflowRunToolStep from "./WorkflowRunToolStep";
 
 export default {
     components: {
-        Form,
+        FormDisplay,
         FormCard,
         ButtonSpinner,
         WorkflowRunDefaultStep,
diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
index 19ff3de2d78..82b373f08c9 100644
--- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
@@ -7,14 +7,14 @@
         <b-alert v-if="errorText" class="mt-2" variant="danger" show>
             {{ errorText | l }}
         </b-alert>
-        <Form :inputs="formInputs" @onChange="onChange" />
+        <FormDisplay :inputs="formInputs" @onChange="onChange" />
         <!-- Options to default one way or the other, disable if admins want, etc.. -->
         <a href="#" @click="$emit('showAdvanced')">Expand to full workflow form.</a>
     </div>
 </template>
 
 <script>
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import { invokeWorkflow } from "./services";
 import { isWorkflowInput } from "components/Workflow/constants";
@@ -23,7 +23,7 @@ import { errorMessageAsString } from "utils/simple-error";
 export default {
     components: {
         ButtonSpinner,
-        Form,
+        FormDisplay,
     },
     props: {
         model: {
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index ede56f55bc8..26fc1cc45af 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -3,7 +3,7 @@
         <FormCard :title="title" icon="fa-wrench" :collapsible="true" :initial-collapse="model.collapsed">
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
-                <Form
+                <FormDisplay
                     :inputs="model.inputs"
                     :form-config="formConfig"
                     :replace-params="replaceParams"
@@ -15,7 +15,7 @@
 </template>
 
 <script>
-import Form from "components/Form/Form";
+import FormDisplay from "components/Form/FormDisplay";
 import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
 import { visitInputs } from "components/Form/utilities";
@@ -24,7 +24,7 @@ import { getTool } from "./services";
 
 export default {
     components: {
-        Form,
+        FormDisplay,
         FormCard,
         FormMessage,
     },
@@ -52,7 +52,6 @@ export default {
     },
     watch: {
         stepData() {
-            console.log(this.model.id);
             this.onReplaceParams();
         },
         wpData() {

From 2cc675c218de67daf8bcda684bd638eb1ebc1112 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 8 Jul 2021 15:31:50 -0400
Subject: [PATCH 190/452] Use fortawesome to load collapsible icons in tool
 card component

---
 client/src/components/Form/FormCard.vue | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index c24d65cffcf..8966a29c6c8 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -13,8 +13,8 @@
                     v-b-tooltip.hover
                     @click="onCollapse"
                 >
-                    <span v-if="collapsed" class="fa fa-eye" />
-                    <span v-else class="fa fa-eye-slash" />
+                    <font-awesome-icon v-if="collapsed" icon="eye-slash" class="fa-fw" />
+                    <font-awesome-icon v-else icon="eye" class="fa-fw" />
                 </b-button>
             </div>
             <a class="portlet-title" @click="onCollapse" :href="href">
@@ -30,7 +30,17 @@
     </div>
 </template>
 <script>
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faEye, faEyeSlash } from "@fortawesome/free-solid-svg-icons";
+
+library.add(faEye);
+library.add(faEyeSlash);
+
 export default {
+    components: {
+        FontAwesomeIcon,
+    },
     props: {
         title: {
             type: String,

From 255bf69e9b4854eb51d8b56a204a73146a301b26 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 8 Jul 2021 15:42:10 -0400
Subject: [PATCH 191/452] Fix tooltip orientation

---
 client/src/components/Form/FormCard.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 8966a29c6c8..736ab9ef34e 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -10,7 +10,7 @@
                     variant="link"
                     size="sm"
                     class="float-right py-0"
-                    v-b-tooltip.hover
+                    v-b-tooltip.hover.bottom
                     @click="onCollapse"
                 >
                     <font-awesome-icon v-if="collapsed" icon="eye-slash" class="fa-fw" />

From 386502e7971e98b23ab48029bbc17abbad5a9b9e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 12 Jul 2021 11:23:05 -0400
Subject: [PATCH 192/452] Migrate form conditional case matching helper from
 backbone utilities

---
 client/src/components/Form/utilities.js | 20 ++++++++++++++++++++
 client/src/mvc/form/form-data.js        | 23 ++---------------------
 2 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/client/src/components/Form/utilities.js b/client/src/components/Form/utilities.js
index 64f90c463c4..5c863cda8aa 100644
--- a/client/src/components/Form/utilities.js
+++ b/client/src/components/Form/utilities.js
@@ -40,3 +40,23 @@ export var visitInputs = (inputs, callback, prefix, context) => {
         }
     }
 };
+
+/** Match conditional values to selected cases
+ * @param{dict}   input     - Definition of conditional input parameter
+ * @param{dict}   value     - Current value
+ */
+export var matchCase = (input, value) => {
+    if (input.test_param.type == "boolean") {
+        if (value == "true") {
+            value = input.test_param.truevalue || "true";
+        } else {
+            value = input.test_param.falsevalue || "false";
+        }
+    }
+    for (var i in input.cases) {
+        if (input.cases[i].value == value) {
+            return i;
+        }
+    }
+    return -1;
+};
diff --git a/client/src/mvc/form/form-data.js b/client/src/mvc/form/form-data.js
index 63c68919716..12c7848fb7c 100644
--- a/client/src/mvc/form/form-data.js
+++ b/client/src/mvc/form/form-data.js
@@ -1,7 +1,7 @@
 import _ from "underscore";
 import $ from "jquery";
 import Backbone from "backbone";
-import { visitInputs } from "components/Form/utilities";
+import { matchCase, visitInputs } from "components/Form/utilities";
 
 /* This class maps the form dom to an api compatible javascript dictionary. */
 export var Manager = Backbone.Model.extend({
@@ -182,27 +182,8 @@ export var Manager = Backbone.Model.extend({
     },
 });
 
-/** Match conditional values to selected cases
- * @param{dict}   input     - Definition of conditional input parameter
- * @param{dict}   value     - Current value
- */
-export var matchCase = (input, value) => {
-    if (input.test_param.type == "boolean") {
-        if (value == "true") {
-            value = input.test_param.truevalue || "true";
-        } else {
-            value = input.test_param.falsevalue || "false";
-        }
-    }
-    for (var i in input.cases) {
-        if (input.cases[i].value == value) {
-            return i;
-        }
-    }
-    return -1;
-};
-
 export default {
     Manager: Manager,
+    matchCase: matchCase,
     visitInputs: visitInputs,
 };

From 45e571d7dc4b6fd5a29855bcb5579a48aafeb28c Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 13 Jul 2021 15:26:55 -0400
Subject: [PATCH 193/452] Fix submission error handling

---
 client/src/components/Workflow/Run/WorkflowRun.vue        | 8 +++++++-
 .../src/components/Workflow/Run/WorkflowRunFormSimple.vue | 6 +-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRun.vue b/client/src/components/Workflow/Run/WorkflowRun.vue
index c5410e918ea..18d4e8f5e54 100644
--- a/client/src/components/Workflow/Run/WorkflowRun.vue
+++ b/client/src/components/Workflow/Run/WorkflowRun.vue
@@ -31,9 +31,15 @@
                     :target-history="simpleFormTargetHistory"
                     :use-job-cache="simpleFormUseJobCache"
                     @submissionSuccess="handleInvocations"
+                    @submissionError="handleSubmissionError"
                     @showAdvanced="showAdvanced"
                 />
-                <workflow-run-form v-else :model="model" @submissionSuccess="handleInvocations" />
+                <workflow-run-form
+                    v-else
+                    :model="model"
+                    @submissionSuccess="handleInvocations"
+                    @submissionError="handleSubmissionError"
+                />
             </div>
         </span>
     </span>
diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
index 82b373f08c9..1683ee3e8cf 100644
--- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue
@@ -4,9 +4,6 @@
             <b>Workflow: {{ model.name }}</b>
             <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @onClick="onExecute" />
         </div>
-        <b-alert v-if="errorText" class="mt-2" variant="danger" show>
-            {{ errorText | l }}
-        </b-alert>
         <FormDisplay :inputs="formInputs" @onChange="onChange" />
         <!-- Options to default one way or the other, disable if admins want, etc.. -->
         <a href="#" @click="$emit('showAdvanced')">Expand to full workflow form.</a>
@@ -43,7 +40,6 @@ export default {
         return {
             formData: {},
             inputTypes: {},
-            errorText: null,
         };
     },
     computed: {
@@ -108,7 +104,7 @@ export default {
                     this.$emit("submissionSuccess", invocations);
                 })
                 .catch((error) => {
-                    this.errorText = errorMessageAsString(error);
+                    this.$emit("submissionError", errorMessageAsString(error));
                 });
         },
     },

From 60fb69ce9f760c99f9cda6cac5657b694353e7e4 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 19 Jul 2021 15:46:02 -0400
Subject: [PATCH 194/452] Preliminary handling of validation errors and
 highlighting in workflow steps

---
 client/src/components/Form/FormDisplay.vue    |  7 +++--
 .../Workflow/Run/WorkflowRunDefaultStep.vue   | 25 ++++++++++++++++--
 .../Workflow/Run/WorkflowRunForm.vue          | 26 ++++++++++++++++++-
 .../Workflow/Run/WorkflowRunToolStep.vue      | 15 +++++++++++
 client/src/style/scss/ui.scss                 |  1 +
 5 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index b4415e5a4df..d1137ec2049 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -94,10 +94,13 @@ export default {
                 const input_id = this.form.data.match(job_input_id);
                 const input_field = this.form.field_list[input_id];
                 const input_def = this.form.input_list[input_id];
-                if (!input_id || !input_def || !input_field) {
+                if (!input_id || !input_def || !input_field || input_def.step_linked) {
                     continue;
                 }
-                if (!input_def.optional && input_value == null && input_def.type != "hidden") {
+                if (input_value && input_value.values && input_value.values.length == 0 && !input_def.optional) {
+                    return [input_id, "Please provide data for this input."];
+                }
+                if (!input_value && !input_def.optional && input_def.type != "hidden") {
                     return [input_id, "Please provide a value for this option."];
                 }
                 if (input_field.validate) {
diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 9acbf2aa660..3c1293d1a00 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -2,7 +2,12 @@
     <div>
         <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :initial-collapse="false">
             <template v-slot:body>
-                <FormDisplay :inputs="inputs" @onChange="onChange" />
+                <FormDisplay
+                    :inputs="inputs"
+                    :validation-scroll-to="validationScrollTo"
+                    @onChange="onChange"
+                    @onValidation="onValidation"
+                />
             </template>
         </FormCard>
     </div>
@@ -24,9 +29,22 @@ export default {
             type: Object,
             required: true,
         },
+        stepScrollTo: {
+            type: Object,
+            required: true,
+        },
     },
     data() {
-        return {};
+        return {
+            validationScrollTo: [],
+        };
+    },
+    watch: {
+        stepScrollTo() {
+            if (this.stepScrollTo && this.stepScrollTo.stepId == this.model.index) {
+                this.validationScrollTo = this.stepScrollTo.validation;
+            }
+        },
     },
     computed: {
         icon() {
@@ -60,6 +78,9 @@ export default {
         onChange(data) {
             this.$emit("onChange", this.model.index, data);
         },
+        onValidation(validation) {
+            this.$emit("onValidation", this.model.index, validation);
+        },
     },
 };
 </script>
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 0acdaaf1962..d58f78f4a3f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -29,10 +29,18 @@
                 v-if="step.step_type == 'tool'"
                 :model="step"
                 :step-data="stepData"
+                :step-scroll-to="stepScrollTo"
                 :wp-data="wpData"
                 @onChange="onToolStepInputs"
+                @onValidation="onValidation"
+            />
+            <WorkflowRunDefaultStep
+                v-else
+                :model="step"
+                :step-scroll-to="stepScrollTo"
+                @onChange="onDefaultStepInputs"
+                @onValidation="onValidation"
             />
-            <WorkflowRunDefaultStep v-else :model="step" @onChange="onDefaultStepInputs" />
         </div>
     </div>
 </template>
@@ -62,6 +70,8 @@ export default {
     data() {
         return {
             stepData: {},
+            stepValidation: {},
+            stepScrollTo: {},
             wpData: {},
             historyData: {},
             cacheData: {},
@@ -153,7 +163,20 @@ export default {
         onCacheInputs(data) {
             this.cacheData = data;
         },
+        onValidation(stepId, validation) {
+            this.stepValidation[stepId] = validation;
+        },
         onExecute() {
+            for (const [stepId, stepValidation] of Object.entries(this.stepValidation)) {
+                if (stepValidation) {
+                    const validation = stepValidation.slice();
+                    this.stepScrollTo = {
+                        stepId,
+                        validation,
+                    };
+                    return;
+                }
+            }
             const Galaxy = getGalaxyInstance();
             const job_def = {
                 new_history_name: this.historyData["new_history|name"] ? this.historyData["new_history|name"] : null,
@@ -170,6 +193,7 @@ export default {
                 // so that inputs can be batched.
                 batch: true,
             };
+
             var validated = true;
             /*for (var i in this.forms) {
                 var form = this.forms[i];
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 26fc1cc45af..1b023e50446 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -7,7 +7,9 @@
                     :inputs="model.inputs"
                     :form-config="formConfig"
                     :replace-params="replaceParams"
+                    :validation-scroll-to="validationScrollTo"
                     @onChange="onChange"
+                    @onValidation="onValidation"
                 />
             </template>
         </FormCard>
@@ -41,6 +43,10 @@ export default {
             type: Object,
             default: null,
         },
+        stepScrollTo: {
+            type: Object,
+            required: true,
+        },
     },
     data() {
         return {
@@ -48,9 +54,15 @@ export default {
             formData: {},
             replaceParams: {},
             errorText: null,
+            validationScrollTo: [],
         };
     },
     watch: {
+        stepScrollTo() {
+            if (this.stepScrollTo && this.stepScrollTo.stepId == this.model.index) {
+                this.validationScrollTo = this.stepScrollTo.validation;
+            }
+        },
         stepData() {
             this.onReplaceParams();
         },
@@ -122,6 +134,9 @@ export default {
             this.formData = data;
             this.$emit("onChange", this.model.index, data);
         },
+        onValidation(validation) {
+            this.$emit("onValidation", this.model.index, validation);
+        },
     },
 };
 </script>
diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss
index a752574eb50..3099d91631e 100644
--- a/client/src/style/scss/ui.scss
+++ b/client/src/style/scss/ui.scss
@@ -195,6 +195,7 @@ $ui-margin-horizontal-large: $margin-v * 2;
         background: $portlet-bg-color;
         border-color: transparent;
         .portlet-title {
+            @extend .my-1;
             .portlet-title-text.collapsible {
                 cursor: pointer;
                 text-decoration: underline;

From a63874b870d471cfd022555fbff06324e074ba79 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 19 Jul 2021 16:02:33 -0400
Subject: [PATCH 195/452] Add workflow run submission protocol to workflow
 component

---
 .../Workflow/Run/WorkflowRunForm.vue          | 133 ++++++------------
 1 file changed, 42 insertions(+), 91 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index d58f78f4a3f..2ca72019c74 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -2,7 +2,13 @@
     <div>
         <div class="h4 clearfix mb-3">
             <b>Workflow: {{ model.name }}</b>
-            <ButtonSpinner class="float-right" title="Run Workflow" id="run-workflow" @onClick="onExecute" />
+            <ButtonSpinner
+                id="run-workflow"
+                class="float-right"
+                title="Run Workflow"
+                :wait="disableExecution"
+                @onClick="onExecute"
+            />
         </div>
         <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
             <template v-slot:body>
@@ -52,6 +58,7 @@ import FormCard from "components/Form/FormCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
 import WorkflowRunToolStep from "./WorkflowRunToolStep";
+import { invokeWorkflow } from "./services";
 
 export default {
     components: {
@@ -69,6 +76,7 @@ export default {
     },
     data() {
         return {
+            disableExecution: false,
             stepData: {},
             stepValidation: {},
             stepScrollTo: {},
@@ -177,14 +185,14 @@ export default {
                     return;
                 }
             }
-            const Galaxy = getGalaxyInstance();
-            const job_def = {
+
+            const jobDef = {
                 new_history_name: this.historyData["new_history|name"] ? this.historyData["new_history|name"] : null,
                 history_id: !this.historyData["new_history|name"] ? this.model.historyId : null,
                 resource_params: this.resourceData,
                 replacement_params: this.wpData,
                 use_cached_job: this.cacheData["use_cached_job|check"] === "true",
-                parameters: {},
+                parameters: this.stepData,
                 // Tool form will submit flat maps for each parameter
                 // (e.g. "repeat_0|cond|param": "foo" instead of nested
                 // data structures).
@@ -194,96 +202,39 @@ export default {
                 batch: true,
             };
 
-            var validated = true;
-            /*for (var i in this.forms) {
-                var form = this.forms[i];
-                var job_inputs = form.data.create();
-                var step = this.steps[i];
-                var step_index = step.step_index;
-                form.trigger("reset");
-                for (var job_input_id in job_inputs) {
-                    var input_value = job_inputs[job_input_id];
-                    var input_id = form.data.match(job_input_id);
-                    var input_def = form.input_list[input_id];
-                    if (!input_def.step_linked) {
-                        if (isDataStep(step)) {
-                            validated = input_value && input_value.values && input_value.values.length > 0;
-                            if (!validated && input_def.optional) {
-                                validated = true;
-                            }
-                        } else {
-                            validated =
-                                input_def.optional ||
-                                (input_def.is_workflow && input_value !== "") ||
-                                (!input_def.is_workflow && input_value !== null);
-                        }
-                        if (!validated) {
-                            if (input_def.type == "hidden") {
-                                this.modal.show({
-                                    title: _l("Workflow submission failed"),
-                                    body: `Step ${step_index}: ${
-                                        step.step_label || step.step_name
-                                    } is in an invalid state. Please remove and re-add this step in the Workflow Editor.`,
-                                    buttons: {
-                                        Close: () => {
-                                            this.modal.hide();
-                                        },
-                                    },
-                                });
-                            }
-                            form.highlight(input_id);
-                            break;
-                        }
-                        job_def.parameters[step_index] = job_def.parameters[step_index] || {};
-                        job_def.parameters[step_index][job_input_id] = job_inputs[job_input_id];
-                    }
-                }
-                if (!validated) {
-                    break;
-                }
-            }
-            if (!validated) {
-                this._enabled(true);
-                Galaxy.emit.debug("tool-form-composite::submit()", "Validation failed.", job_def);
-            } else {
-                Galaxy.emit.debug("tool-form-composite::submit()", "Validation complete.", job_def);
-                invokeWorkflow(this.runWorkflowModel.workflowId, job_def)
-                    .then((invocations) => {
-                        Galaxy.emit.debug("tool-form-composite::submit", "Submission successful.", invocations);
-                        if (Array.isArray(invocations) && invocations.length > 0) {
-                            this.handleInvocations(invocations);
-                        } else {
-                            this.submissionErrorModal(job_def, invocations);
-                        }
-                        this._enabled(true);
-                    })
-                    .catch((response) => {
-                        // TODO: Is this the same response as the Utils post would
-                        // have had?
-                        Galaxy.emit.debug("tool-form-composite::submit", "Submission failed.", response);
-                        var input_found = false;
-                        if (response && response.err_data) {
-                            for (var i in this.forms) {
-                                var form = this.forms[i];
-                                var step_related_errors = response.err_data[form.model.get("step_index")];
-                                if (step_related_errors) {
-                                    var error_messages = form.data.matchResponse(step_related_errors);
-                                    for (var input_id in error_messages) {
-                                        form.highlight(input_id, error_messages[input_id]);
-                                        input_found = true;
-                                        break;
-                                    }
+            console.debug("WorkflowRunForm::onExecute()", "Ready for submission.", jobDef);
+            this.disableExecution = true;
+            invokeWorkflow(this.model.workflowId, jobDef)
+                .then((invocations) => {
+                    console.debug("WorkflowRunForm::onExecute()", "Submission successful.", invocations);
+                    this.disableExecution = false;
+                    this.$emit("submissionSuccess", invocations);
+                })
+                .catch((response) => {
+                    // TODO: Is this the same response as the Utils post would
+                    // have had?
+                    console.debug("WorkflowRunForm::onExecute()", "Submission failed.", response);
+                    /*var input_found = false;
+                    if (response && response.err_data) {
+                        for (var i in this.forms) {
+                            var form = this.forms[i];
+                            var step_related_errors = response.err_data[form.model.get("step_index")];
+                            if (step_related_errors) {
+                                var error_messages = form.data.matchResponse(step_related_errors);
+                                for (var input_id in error_messages) {
+                                    form.highlight(input_id, error_messages[input_id]);
+                                    input_found = true;
+                                    break;
                                 }
                             }
                         }
-                        if (!input_found) {
-                            this.submissionErrorModal(job_def, response);
-                        }
-                        this._enabled(true);
-                    });*/
-        },
-        handleInvocations(invocations) {
-            this.$emit("submissionSuccess", invocations);
+                    }
+                    if (!input_found) {
+                        this.submissionErrorModal(jobDef, response);
+                    }*/
+                    this.$emit("submissionError", response);
+                    this.disableExecution = false;
+                });
         },
         toArray(obj) {
             return obj ? Object.keys(obj).map((k) => obj[k]) : [];

From 803f02083b49d284ceb710b2a2c70659abc37120 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 19 Jul 2021 22:41:56 -0400
Subject: [PATCH 196/452] Handle validation highlighting and workflow step
 collapse handling

---
 client/src/components/Form/FormCard.vue       |  8 +++-----
 .../Workflow/Run/WorkflowRunDefaultStep.vue   | 19 ++++++++++++-------
 .../Workflow/Run/WorkflowRunForm.vue          | 15 ++++++++++-----
 .../Workflow/Run/WorkflowRunToolStep.vue      | 18 +++++++++---------
 4 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index 736ab9ef34e..ae104920783 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -58,15 +58,13 @@ export default {
             type: Boolean,
             default: false,
         },
-        initialCollapse: {
+        collapsed: {
             type: Boolean,
             default: false,
         },
     },
     data() {
-        return {
-            collapsed: this.initialCollapse,
-        };
+        return {};
     },
     computed: {
         href() {
@@ -75,7 +73,7 @@ export default {
     },
     methods: {
         onCollapse() {
-            this.collapsed = this.collapsible && !this.collapsed;
+            this.$emit("update:collapsed", !this.collapsed);
         },
     },
 };
diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 3c1293d1a00..9b1628e2004 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -1,6 +1,11 @@
 <template>
     <div>
-        <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :initial-collapse="false">
+        <FormCard
+            :title="model.fixed_title"
+            :icon="icon"
+            :collapsible="true"
+            :collapsed.sync="collapsed"
+        >
             <template v-slot:body>
                 <FormDisplay
                     :inputs="inputs"
@@ -29,20 +34,20 @@ export default {
             type: Object,
             required: true,
         },
-        stepScrollTo: {
-            type: Object,
+        validationScrollTo: {
+            type: Array,
             required: true,
         },
     },
     data() {
         return {
-            validationScrollTo: [],
+            collapsed: this.model.collapsed,
         };
     },
     watch: {
-        stepScrollTo() {
-            if (this.stepScrollTo && this.stepScrollTo.stepId == this.model.index) {
-                this.validationScrollTo = this.stepScrollTo.validation;
+        validationScrollTo() {
+            if (this.validationScrollTo.length > 0) {
+                this.collapsed = false;
             }
         },
     },
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 2ca72019c74..945f5803157 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -35,7 +35,7 @@
                 v-if="step.step_type == 'tool'"
                 :model="step"
                 :step-data="stepData"
-                :step-scroll-to="stepScrollTo"
+                :validation-scroll-to="getValidationScrollTo(step.index)"
                 :wp-data="wpData"
                 @onChange="onToolStepInputs"
                 @onValidation="onValidation"
@@ -43,7 +43,7 @@
             <WorkflowRunDefaultStep
                 v-else
                 :model="step"
-                :step-scroll-to="stepScrollTo"
+                :validation-scroll-to="getValidationScrollTo(step.index)"
                 @onChange="onDefaultStepInputs"
                 @onValidation="onValidation"
             />
@@ -152,6 +152,12 @@ export default {
         },
     },
     methods: {
+        getValidationScrollTo(stepId) {
+            if (this.stepScrollTo.stepId == stepId) {
+                return this.stepScrollTo.stepValidation;
+            }
+            return [];
+        },
         onDefaultStepInputs(stepId, data) {
             this.stepData[stepId] = data;
             this.stepData = Object.assign({}, this.stepData);
@@ -177,10 +183,9 @@ export default {
         onExecute() {
             for (const [stepId, stepValidation] of Object.entries(this.stepValidation)) {
                 if (stepValidation) {
-                    const validation = stepValidation.slice();
                     this.stepScrollTo = {
-                        stepId,
-                        validation,
+                        stepId: stepId,
+                        stepValidation: stepValidation.slice(),
                     };
                     return;
                 }
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 1b023e50446..2f92939a3ed 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div>
-        <FormCard :title="title" icon="fa-wrench" :collapsible="true" :initial-collapse="model.collapsed">
+        <FormCard :title="title" icon="fa-wrench" :collapsible="true" :collapsed.sync="collapsed">
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <FormDisplay
@@ -43,32 +43,32 @@ export default {
             type: Object,
             default: null,
         },
-        stepScrollTo: {
-            type: Object,
+        validationScrollTo: {
+            type: Array,
             required: true,
         },
     },
     data() {
         return {
+            collapsed: this.model.collapsed,
             formConfig: {},
             formData: {},
             replaceParams: {},
             errorText: null,
-            validationScrollTo: [],
         };
     },
     watch: {
-        stepScrollTo() {
-            if (this.stepScrollTo && this.stepScrollTo.stepId == this.model.index) {
-                this.validationScrollTo = this.stepScrollTo.validation;
-            }
-        },
         stepData() {
             this.onReplaceParams();
         },
         wpData() {
             this.onReplaceParams();
         },
+        validationScrollTo() {
+            if (this.validationScrollTo.length > 0) {
+                this.collapsed = false;
+            }
+        },
     },
     computed: {
         title() {

From 2b7bdd60a4d108a1819a6f65f4876590f75c6c7b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 00:50:44 -0400
Subject: [PATCH 197/452] Fix workflow parameter handling and validation

---
 client/src/components/Form/FormDisplay.vue                 | 3 +++
 client/src/components/Tool/ToolCard.vue                    | 2 +-
 .../src/components/Workflow/Run/WorkflowRunDefaultStep.vue | 7 +------
 client/src/components/Workflow/Run/model.js                | 1 +
 client/src/style/scss/ui.scss                              | 1 -
 5 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index d1137ec2049..24e240f59a1 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -103,6 +103,9 @@ export default {
                 if (!input_value && !input_def.optional && input_def.type != "hidden") {
                     return [input_id, "Please provide a value for this option."];
                 }
+                if (input_def.wp_linked && input_def.text_value == input_value) {
+                    return [input_id, "Please provide a value for this workflow parameter."];
+                }
                 if (input_field.validate) {
                     const message = input_field.validate();
                     if (message) {
diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index 1f06b2b9e85..68fa050ee22 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -79,7 +79,7 @@
                         <span class="fa fa-star" />
                     </b-button>
                 </div>
-                <div class="portlet-title">
+                <div class="portlet-title my-1">
                     <i class="portlet-title-icon fa mr-1 fa-wrench" style="display: inline"></i>
                     <span class="portlet-title-text">
                         <b itemprop="name">{{ title }}</b> <span itemprop="description">{{ description }}</span> (Galaxy
diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 9b1628e2004..d478e40290c 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -1,11 +1,6 @@
 <template>
     <div>
-        <FormCard
-            :title="model.fixed_title"
-            :icon="icon"
-            :collapsible="true"
-            :collapsed.sync="collapsed"
-        >
+        <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :collapsed.sync="collapsed">
             <template v-slot:body>
                 <FormDisplay
                     :inputs="inputs"
diff --git a/client/src/components/Workflow/Run/model.js b/client/src/components/Workflow/Run/model.js
index a35ba4eeccb..c0b66ec5ab5 100644
--- a/client/src/components/Workflow/Run/model.js
+++ b/client/src/components/Workflow/Run/model.js
@@ -123,6 +123,7 @@ export class WorkflowRunModel {
                 color: `hsl( ${++wp_count * 100}, 70%, 30% )`,
                 style: "ui-form-wp-source",
                 links: [],
+                optional: true,
             });
         };
 
diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss
index 3099d91631e..a752574eb50 100644
--- a/client/src/style/scss/ui.scss
+++ b/client/src/style/scss/ui.scss
@@ -195,7 +195,6 @@ $ui-margin-horizontal-large: $margin-v * 2;
         background: $portlet-bg-color;
         border-color: transparent;
         .portlet-title {
-            @extend .my-1;
             .portlet-title-text.collapsible {
                 cursor: pointer;
                 text-decoration: underline;

From 275a7b92fadd35eb8970b0d51b69a563bdf312f8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 11:16:05 -0400
Subject: [PATCH 198/452] Refine client-side input validation of forms

---
 client/src/components/Form/FormDisplay.vue | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 24e240f59a1..cb813858ad4 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -97,10 +97,15 @@ export default {
                 if (!input_id || !input_def || !input_field || input_def.step_linked) {
                     continue;
                 }
-                if (input_value && input_value.values && input_value.values.length == 0 && !input_def.optional) {
+                if (
+                    input_value &&
+                    Array.isArray(input_value.values) &&
+                    input_value.values.length == 0 &&
+                    !input_def.optional
+                ) {
                     return [input_id, "Please provide data for this input."];
                 }
-                if (!input_value && !input_def.optional && input_def.type != "hidden") {
+                if (input_value == null && !input_def.optional && input_def.type != "hidden") {
                     return [input_id, "Please provide a value for this option."];
                 }
                 if (input_def.wp_linked && input_def.text_value == input_value) {

From 9ceb33e696e8729f7e574163cb047e19e92bf1b5 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 11:50:36 -0400
Subject: [PATCH 199/452] Add server side validation error handling to run form
 client component

---
 .../Workflow/Run/WorkflowRunForm.vue          | 62 ++++++++++---------
 .../Workflow/Run/WorkflowRunToolStep.vue      |  5 ++
 2 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 945f5803157..420be9fb3f7 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -6,7 +6,7 @@
                 id="run-workflow"
                 class="float-right"
                 title="Run Workflow"
-                :wait="disableExecution"
+                :wait="showExecuting"
                 @onClick="onExecute"
             />
         </div>
@@ -36,6 +36,7 @@
                 :model="step"
                 :step-data="stepData"
                 :validation-scroll-to="getValidationScrollTo(step.index)"
+                :validation-errors="getValidationErrors(step.index)"
                 :wp-data="wpData"
                 @onChange="onToolStepInputs"
                 @onValidation="onValidation"
@@ -76,10 +77,11 @@ export default {
     },
     data() {
         return {
-            disableExecution: false,
+            showExecuting: false,
             stepData: {},
-            stepValidation: {},
+            stepValidations: {},
             stepScrollTo: {},
+            stepErrors: {},
             wpData: {},
             historyData: {},
             cacheData: {},
@@ -154,10 +156,16 @@ export default {
     methods: {
         getValidationScrollTo(stepId) {
             if (this.stepScrollTo.stepId == stepId) {
-                return this.stepScrollTo.stepValidation;
+                return this.stepScrollTo.details;
             }
             return [];
         },
+        getValidationErrors(stepId) {
+            if (this.stepErrors.stepId == stepId) {
+                return this.stepErrors.details;
+            }
+            return {};
+        },
         onDefaultStepInputs(stepId, data) {
             this.stepData[stepId] = data;
             this.stepData = Object.assign({}, this.stepData);
@@ -178,14 +186,14 @@ export default {
             this.cacheData = data;
         },
         onValidation(stepId, validation) {
-            this.stepValidation[stepId] = validation;
+            this.stepValidations[stepId] = validation;
         },
         onExecute() {
-            for (const [stepId, stepValidation] of Object.entries(this.stepValidation)) {
+            for (const [stepId, stepValidation] of Object.entries(this.stepValidations)) {
                 if (stepValidation) {
                     this.stepScrollTo = {
                         stepId: stepId,
-                        stepValidation: stepValidation.slice(),
+                        details: stepValidation.slice(),
                     };
                     return;
                 }
@@ -208,37 +216,31 @@ export default {
             };
 
             console.debug("WorkflowRunForm::onExecute()", "Ready for submission.", jobDef);
-            this.disableExecution = true;
+            this.showExecuting = true;
             invokeWorkflow(this.model.workflowId, jobDef)
                 .then((invocations) => {
                     console.debug("WorkflowRunForm::onExecute()", "Submission successful.", invocations);
-                    this.disableExecution = false;
+                    this.showExecuting = false;
                     this.$emit("submissionSuccess", invocations);
                 })
-                .catch((response) => {
-                    // TODO: Is this the same response as the Utils post would
-                    // have had?
-                    console.debug("WorkflowRunForm::onExecute()", "Submission failed.", response);
-                    /*var input_found = false;
-                    if (response && response.err_data) {
-                        for (var i in this.forms) {
-                            var form = this.forms[i];
-                            var step_related_errors = response.err_data[form.model.get("step_index")];
-                            if (step_related_errors) {
-                                var error_messages = form.data.matchResponse(step_related_errors);
-                                for (var input_id in error_messages) {
-                                    form.highlight(input_id, error_messages[input_id]);
-                                    input_found = true;
-                                    break;
-                                }
+                .catch((e) => {
+                    console.debug("WorkflowRunForm::onExecute()", "Submission failed.", e);
+                    this.showExecuting = false;
+                    const errorData = e && e.response && e.response.data && e.response.data.err_data;
+                    if (errorData) {
+                        for (const stepId in errorData) {
+                            const stepError = errorData[stepId];
+                            if (stepError) {
+                                this.stepError = {
+                                    stepId: stepId,
+                                    details: stepError,
+                                };
+                                return;
                             }
                         }
+                    } else {
+                        this.$emit("submissionError", response);
                     }
-                    if (!input_found) {
-                        this.submissionErrorModal(jobDef, response);
-                    }*/
-                    this.$emit("submissionError", response);
-                    this.disableExecution = false;
                 });
         },
         toArray(obj) {
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 2f92939a3ed..a2e0f7f25b8 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -8,6 +8,7 @@
                     :form-config="formConfig"
                     :replace-params="replaceParams"
                     :validation-scroll-to="validationScrollTo"
+                    :validation-errors="validationErrors"
                     @onChange="onChange"
                     @onValidation="onValidation"
                 />
@@ -47,6 +48,10 @@ export default {
             type: Array,
             required: true,
         },
+        validationErrors: {
+            type: Object,
+            default: null,
+        },
     },
     data() {
         return {

From 28a374f8b9360b8640b50a3f2722b42294185763 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 12:23:06 -0400
Subject: [PATCH 200/452] Unify handling of client and server side validation

---
 client/src/components/Form/FormDisplay.vue    | 25 ++++++-------------
 client/src/components/Tool/ToolForm.vue       |  4 +--
 .../Workflow/Run/WorkflowRunForm.vue          | 16 +++---------
 .../Workflow/Run/WorkflowRunToolStep.vue      |  5 ----
 4 files changed, 14 insertions(+), 36 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index cb813858ad4..4708451b6fc 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -32,10 +32,6 @@ export default {
             type: Array,
             default: null,
         },
-        validationErrors: {
-            type: Object,
-            default: null,
-        },
         formConfig: {
             type: Object,
             default: null,
@@ -61,12 +57,6 @@ export default {
             this.onHighlight(this.validation, true);
             this.$emit("onValidation", this.validation);
         },
-        validationErrors() {
-            const matchedErrors = Object.entries(this.form.data.matchResponse(this.validationErrors));
-            if (matchedErrors.length > 0) {
-                this.onHighlight(matchedErrors[0]);
-            }
-        },
         formConfig() {
             this.$nextTick(() => {
                 this.form.update(this.formConfig);
@@ -103,18 +93,18 @@ export default {
                     input_value.values.length == 0 &&
                     !input_def.optional
                 ) {
-                    return [input_id, "Please provide data for this input."];
+                    return [job_input_id, "Please provide data for this input."];
                 }
                 if (input_value == null && !input_def.optional && input_def.type != "hidden") {
-                    return [input_id, "Please provide a value for this option."];
+                    return [job_input_id, "Please provide a value for this option."];
                 }
                 if (input_def.wp_linked && input_def.text_value == input_value) {
-                    return [input_id, "Please provide a value for this workflow parameter."];
+                    return [job_input_id, "Please provide a value for this workflow parameter."];
                 }
                 if (input_field.validate) {
                     const message = input_field.validate();
                     if (message) {
-                        return [input_id, message];
+                        return [job_input_id, message];
                     }
                 }
                 if (input_value && input_value.batch) {
@@ -125,7 +115,7 @@ export default {
                             batch_src = src;
                         } else if (batch_src !== src) {
                             return [
-                                input_id,
+                                job_input_id,
                                 "Please select either dataset or dataset list fields for all batch mode fields.",
                             ];
                         }
@@ -134,7 +124,7 @@ export default {
                         batch_n = n;
                     } else if (batch_n !== n) {
                         return [
-                            input_id,
+                            job_input_id,
                             `Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>${n}</b> selection(s) while a previous field contains <b>${batch_n}</b>.`,
                         ];
                     }
@@ -183,7 +173,8 @@ export default {
         onHighlight(validation, silent = false) {
             this.form.trigger("reset");
             if (validation && validation.length == 2) {
-                this.form.highlight(validation[0], validation[1], silent);
+                const input_id = this.form.data.match(validation[0]);
+                this.form.highlight(input_id, validation[1], silent);
             }
         },
     },
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 29f5de66e28..47865f9cca7 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -42,7 +42,6 @@
                             <FormDisplay
                                 :id="formConfig.id"
                                 :inputs="inputs"
-                                :validation-errors="validationErrors"
                                 :validation-scroll-to="validationScrollTo"
                                 :form-config="formConfig"
                                 @onChange="onChange"
@@ -301,7 +300,8 @@ export default {
                     this.showExecuting = false;
                     const errorData = e && e.response && e.response.data && e.response.data.err_data;
                     if (errorData) {
-                        this.validationErrors = errorData;
+                        const errorEntries = Object.entries(errorData);
+                        this.validationScrollTo = errorEntries[0];
                     } else {
                         this.showError = true;
                         this.showForm = false;
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 420be9fb3f7..75c04be4970 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -36,7 +36,6 @@
                 :model="step"
                 :step-data="stepData"
                 :validation-scroll-to="getValidationScrollTo(step.index)"
-                :validation-errors="getValidationErrors(step.index)"
                 :wp-data="wpData"
                 @onChange="onToolStepInputs"
                 @onValidation="onValidation"
@@ -81,7 +80,6 @@ export default {
             stepData: {},
             stepValidations: {},
             stepScrollTo: {},
-            stepErrors: {},
             wpData: {},
             historyData: {},
             cacheData: {},
@@ -160,12 +158,6 @@ export default {
             }
             return [];
         },
-        getValidationErrors(stepId) {
-            if (this.stepErrors.stepId == stepId) {
-                return this.stepErrors.details;
-            }
-            return {};
-        },
         onDefaultStepInputs(stepId, data) {
             this.stepData[stepId] = data;
             this.stepData = Object.assign({}, this.stepData);
@@ -228,12 +220,12 @@ export default {
                     this.showExecuting = false;
                     const errorData = e && e.response && e.response.data && e.response.data.err_data;
                     if (errorData) {
-                        for (const stepId in errorData) {
-                            const stepError = errorData[stepId];
+                        for (const [stepId, stepError] of Object.entries(errorData)) {
                             if (stepError) {
-                                this.stepError = {
+                                const errorEntries = Object.entries(stepError);
+                                this.stepScrollTo = {
                                     stepId: stepId,
-                                    details: stepError,
+                                    details: errorEntries[0],
                                 };
                                 return;
                             }
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index a2e0f7f25b8..2f92939a3ed 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -8,7 +8,6 @@
                     :form-config="formConfig"
                     :replace-params="replaceParams"
                     :validation-scroll-to="validationScrollTo"
-                    :validation-errors="validationErrors"
                     @onChange="onChange"
                     @onValidation="onValidation"
                 />
@@ -48,10 +47,6 @@ export default {
             type: Array,
             required: true,
         },
-        validationErrors: {
-            type: Object,
-            default: null,
-        },
     },
     data() {
         return {

From 676cae5d44360aa456a6010cd7472058c15a111d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 13:05:58 -0400
Subject: [PATCH 201/452] Catch potential errors in server response formatting

---
 client/src/components/Tool/ToolForm.vue       |  9 +++++--
 .../Workflow/Run/WorkflowRunForm.vue          | 27 +++++++++++--------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 47865f9cca7..558848a0fc4 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -298,11 +298,16 @@ export default {
                 },
                 (e) => {
                     this.showExecuting = false;
+                    let genericError = true;
                     const errorData = e && e.response && e.response.data && e.response.data.err_data;
                     if (errorData) {
                         const errorEntries = Object.entries(errorData);
-                        this.validationScrollTo = errorEntries[0];
-                    } else {
+                        if (errorEntries.length > 0) {
+                            this.validationScrollTo = errorEntries[0];
+                            genericError = false;
+                        }
+                    }
+                    if (genericError) {
                         this.showError = true;
                         this.showForm = false;
                         this.errorTitle = "Job submission failed";
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 75c04be4970..a057a6795a1 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -154,7 +154,7 @@ export default {
     methods: {
         getValidationScrollTo(stepId) {
             if (this.stepScrollTo.stepId == stepId) {
-                return this.stepScrollTo.details;
+                return this.stepScrollTo.stepError;
             }
             return [];
         },
@@ -185,7 +185,7 @@ export default {
                 if (stepValidation) {
                     this.stepScrollTo = {
                         stepId: stepId,
-                        details: stepValidation.slice(),
+                        stepError: stepValidation.slice(),
                     };
                     return;
                 }
@@ -220,15 +220,20 @@ export default {
                     this.showExecuting = false;
                     const errorData = e && e.response && e.response.data && e.response.data.err_data;
                     if (errorData) {
-                        for (const [stepId, stepError] of Object.entries(errorData)) {
-                            if (stepError) {
-                                const errorEntries = Object.entries(stepError);
-                                this.stepScrollTo = {
-                                    stepId: stepId,
-                                    details: errorEntries[0],
-                                };
-                                return;
-                            }
+                        try {
+                            const errorEntries = Object.entries(errorData);
+                            this.stepScrollTo = {
+                                stepId: errorEntries[0][0],
+                                stepError: Object.entries(errorEntries[0][1])[0],
+                            };
+                        } catch (e) {
+                            console.debug(
+                                e,
+                                "WorkflowRunForm::onExecute()",
+                                "Invalid server error response.",
+                                errorData
+                            );
+                            this.$emit("submissionError", response);
                         }
                     } else {
                         this.$emit("submissionError", response);

From d22de0cf9aa17bc58b33431d279e3c7ce3f9a1cc Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 13:22:21 -0400
Subject: [PATCH 202/452] Fix error parameter naming

---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index a057a6795a1..37885311f9c 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -226,17 +226,17 @@ export default {
                                 stepId: errorEntries[0][0],
                                 stepError: Object.entries(errorEntries[0][1])[0],
                             };
-                        } catch (e) {
+                        } catch (errorFormatting) {
                             console.debug(
-                                e,
+                                errorFormatting,
                                 "WorkflowRunForm::onExecute()",
-                                "Invalid server error response.",
+                                "Invalid server error response format.",
                                 errorData
                             );
-                            this.$emit("submissionError", response);
+                            this.$emit("submissionError", e);
                         }
                     } else {
-                        this.$emit("submissionError", response);
+                        this.$emit("submissionError", e);
                     }
                 });
         },

From 87b2c5db8282c9fcb55d9eb4b6d3e763c3e249d1 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 14:19:25 -0400
Subject: [PATCH 203/452] Remove unnecessary jquery usage in form utilities

---
 client/src/components/Form/utilities.js        | 4 +++-
 client/src/components/Workflow/Run/model.js    | 6 +++---
 client/src/components/Workflow/Run/services.js | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Form/utilities.js b/client/src/components/Form/utilities.js
index 5c863cda8aa..cac83e9ff82 100644
--- a/client/src/components/Form/utilities.js
+++ b/client/src/components/Form/utilities.js
@@ -1,9 +1,11 @@
+import _ from "underscore";
+
 /** Visits tool inputs
  * @param{dict}   inputs    - Nested dictionary of input elements
  * @param{dict}   callback  - Called with the mapped dictionary object and corresponding model node
  */
 export var visitInputs = (inputs, callback, prefix, context) => {
-    context = $.extend({}, context);
+    context = Object.assign({}, context);
     _.each(inputs, (input) => {
         if (input && input.type && input.name) {
             context[input.name] = input;
diff --git a/client/src/components/Workflow/Run/model.js b/client/src/components/Workflow/Run/model.js
index c0b66ec5ab5..a9c4bbc447b 100644
--- a/client/src/components/Workflow/Run/model.js
+++ b/client/src/components/Workflow/Run/model.js
@@ -1,6 +1,6 @@
 import _ from "underscore";
 
-import FormData from "mvc/form/form-data";
+import { visitInputs } from "components/Form/utilities";
 import WorkflowIcons from "components/Workflow/icons";
 import Utils from "utils/utils";
 
@@ -73,7 +73,7 @@ export class WorkflowRunModel {
 
         // build linear index of step input pairs
         _.each(this.steps, (step, i) => {
-            FormData.visitInputs(step.inputs, (input, name) => {
+            visitInputs(step.inputs, (input, name) => {
                 this.parms[i][name] = input;
             });
         });
@@ -159,7 +159,7 @@ export class WorkflowRunModel {
         _.each(this.steps, (step, i) => {
             if (step.step_type == "tool") {
                 var data_resolved = true;
-                FormData.visitInputs(step.inputs, (input, name, context) => {
+                visitInputs(step.inputs, (input, name, context) => {
                     var is_runtime_value = input.value && input.value.__class__ == "RuntimeValue";
                     var is_data_input = ["data", "data_collection"].indexOf(input.type) != -1;
                     var data_ref = context[input.data_ref];
diff --git a/client/src/components/Workflow/Run/services.js b/client/src/components/Workflow/Run/services.js
index a70a1c342ec..5ae80968a0a 100644
--- a/client/src/components/Workflow/Run/services.js
+++ b/client/src/components/Workflow/Run/services.js
@@ -41,7 +41,7 @@ export async function getTool(toolId, toolVersion, toolInputs) {
     const requestData = {
         tool_id: toolId,
         tool_version: toolVersion,
-        inputs: $.extend(true, {}, toolInputs),
+        inputs: JSON.parse(JSON.stringify(toolInputs)),
     };
     try {
         const { data } = await axios.post(`${getAppRoot()}api/tools/${toolId}/build`, requestData);

From 9c1046559e5b1b53141f81cd502396d8d0740061 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 16:42:40 -0400
Subject: [PATCH 204/452] Remove unused import in form component, add step
 labels for selenium tests

---
 client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue | 2 +-
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue    | 2 +-
 client/src/mvc/form/form-data.js                              | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index d478e40290c..5f08564525f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div :step-label="model.step_label">
         <FormCard :title="model.fixed_title" :icon="icon" :collapsible="true" :collapsed.sync="collapsed">
             <template v-slot:body>
                 <FormDisplay
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 2f92939a3ed..15b817a1396 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,5 +1,5 @@
 <template>
-    <div>
+    <div :step-label="model.step_label">
         <FormCard :title="title" icon="fa-wrench" :collapsible="true" :collapsed.sync="collapsed">
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
diff --git a/client/src/mvc/form/form-data.js b/client/src/mvc/form/form-data.js
index 12c7848fb7c..ae7f5732d6a 100644
--- a/client/src/mvc/form/form-data.js
+++ b/client/src/mvc/form/form-data.js
@@ -1,4 +1,3 @@
-import _ from "underscore";
 import $ from "jquery";
 import Backbone from "backbone";
 import { matchCase, visitInputs } from "components/Form/utilities";

From c9b2627b1d5227ae9b91eeeffa0a0cc3be415a17 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 16:48:57 -0400
Subject: [PATCH 205/452] Remove unused global galaxy object in default
 workflow step component

---
 client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
index 5f08564525f..215b05ee3ee 100644
--- a/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunDefaultStep.vue
@@ -14,7 +14,6 @@
 </template>
 
 <script>
-import { getGalaxyInstance } from "app";
 import WorkflowIcons from "components/Workflow/icons";
 import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";

From 497bcfdfc1d062ec4593e1f42d9a00f6266289b8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 17:20:26 -0400
Subject: [PATCH 206/452] Fix linting, place use job cache preference into
 utilities, use user provider

---
 client/src/components/Tool/ToolForm.vue       |  10 +-
 client/src/components/Tool/utilities.js       |  11 ++
 .../Workflow/Run/WorkflowRunForm.vue          | 118 +++++++++---------
 .../Workflow/Run/WorkflowRunToolStep.vue      |   2 +-
 4 files changed, 71 insertions(+), 70 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 558848a0fc4..79261cf89ec 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -92,6 +92,7 @@
 import Scroller from "vue-scrollto";
 import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
+import { reuseCachedJobs } from "./utilities";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
@@ -207,14 +208,7 @@ export default {
             return config.server_mail_configured && !user.isAnonymous;
         },
         reuseAllowed(user) {
-            if (user.preferences && "extra_user_preferences" in user.preferences) {
-                const extra_user_preferences = JSON.parse(user.preferences.extra_user_preferences);
-                const keyCached = "use_cached_job|use_cached_job_checkbox";
-                const hasCachedJobs = keyCached in extra_user_preferences;
-                return hasCachedJobs ? extra_user_preferences[keyCached] : false;
-            } else {
-                return false;
-            }
+            return reuseCachedJobs(user.preferences);
         },
         onValidation(validationInternal) {
             this.validationInternal = validationInternal;
diff --git a/client/src/components/Tool/utilities.js b/client/src/components/Tool/utilities.js
index d0d4e736c5b..5100338a473 100644
--- a/client/src/components/Tool/utilities.js
+++ b/client/src/components/Tool/utilities.js
@@ -12,3 +12,14 @@ export function downloadTool(toolId) {
 export function openLink(url) {
     window.open(url);
 }
+
+export function reuseCachedJobs(userPreferences) {
+    if (userPreferences && "extra_user_preferences" in userPreferences) {
+        const extra_user_preferences = JSON.parse(userPreferences.extra_user_preferences);
+        const keyCached = "use_cached_job|use_cached_job_checkbox";
+        const hasCachedJobs = keyCached in extra_user_preferences;
+        return hasCachedJobs ? ["true", true].includes(extra_user_preferences[keyCached]) : false;
+    } else {
+        return false;
+    }
+}
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 37885311f9c..6ffa3275d5a 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,67 +1,71 @@
 <template>
-    <div>
-        <div class="h4 clearfix mb-3">
-            <b>Workflow: {{ model.name }}</b>
-            <ButtonSpinner
-                id="run-workflow"
-                class="float-right"
-                title="Run Workflow"
-                :wait="showExecuting"
-                @onClick="onExecute"
-            />
+    <CurrentUser v-slot="{ user }">
+        <div>
+            <div class="h4 clearfix mb-3">
+                <b>Workflow: {{ model.name }}</b>
+                <ButtonSpinner
+                    id="run-workflow"
+                    class="float-right"
+                    title="Run Workflow"
+                    :wait="showExecuting"
+                    @onClick="onExecute"
+                />
+            </div>
+            <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
+                <template v-slot:body>
+                    <FormDisplay :inputs="wpInputs" @onChange="onWpInputs" />
+                </template>
+            </FormCard>
+            <FormCard title="History Options">
+                <template v-slot:body>
+                    <FormDisplay :inputs="historyInputs" @onChange="onHistoryInputs" />
+                </template>
+            </FormCard>
+            <FormCard v-if="cacheInputsAvailable(user)" title="Job re-use Options">
+                <template v-slot:body>
+                    <FormDisplay :inputs="cacheInputs" @onChange="onCacheInputs" />
+                </template>
+            </FormCard>
+            <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
+                <template v-slot:body>
+                    <FormDisplay :inputs="resourceInputs" @onChange="onResourceInputs" />
+                </template>
+            </FormCard>
+            <div v-for="step in model.steps" :key="step.index">
+                <WorkflowRunToolStep
+                    v-if="step.step_type == 'tool'"
+                    :model="step"
+                    :step-data="stepData"
+                    :validation-scroll-to="getValidationScrollTo(step.index)"
+                    :wp-data="wpData"
+                    @onChange="onToolStepInputs"
+                    @onValidation="onValidation"
+                />
+                <WorkflowRunDefaultStep
+                    v-else
+                    :model="step"
+                    :validation-scroll-to="getValidationScrollTo(step.index)"
+                    @onChange="onDefaultStepInputs"
+                    @onValidation="onValidation"
+                />
+            </div>
         </div>
-        <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
-            <template v-slot:body>
-                <FormDisplay :inputs="wpInputs" @onChange="onWpInputs" />
-            </template>
-        </FormCard>
-        <FormCard title="History Options">
-            <template v-slot:body>
-                <FormDisplay :inputs="historyInputs" @onChange="onHistoryInputs" />
-            </template>
-        </FormCard>
-        <FormCard v-if="cacheInputsAvailable" title="Job re-use Options">
-            <template v-slot:body>
-                <FormDisplay :inputs="cacheInputs" @onChange="onCacheInputs" />
-            </template>
-        </FormCard>
-        <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
-            <template v-slot:body>
-                <FormDisplay :inputs="resourceInputs" @onChange="onResourceInputs" />
-            </template>
-        </FormCard>
-        <div v-for="step in model.steps">
-            <WorkflowRunToolStep
-                v-if="step.step_type == 'tool'"
-                :model="step"
-                :step-data="stepData"
-                :validation-scroll-to="getValidationScrollTo(step.index)"
-                :wp-data="wpData"
-                @onChange="onToolStepInputs"
-                @onValidation="onValidation"
-            />
-            <WorkflowRunDefaultStep
-                v-else
-                :model="step"
-                :validation-scroll-to="getValidationScrollTo(step.index)"
-                @onChange="onDefaultStepInputs"
-                @onValidation="onValidation"
-            />
-        </div>
-    </div>
+    </CurrentUser>
 </template>
 
 <script>
-import { getGalaxyInstance } from "app";
+import CurrentUser from "components/providers/CurrentUser";
 import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
 import WorkflowRunToolStep from "./WorkflowRunToolStep";
 import { invokeWorkflow } from "./services";
+import { reuseCachedJobs } from "components/Tool/utilities";
 
 export default {
     components: {
+        CurrentUser,
         FormDisplay,
         FormCard,
         ButtonSpinner,
@@ -112,17 +116,6 @@ export default {
         };
     },
     computed: {
-        cacheInputsAvailable() {
-            const Galaxy = getGalaxyInstance();
-            var extra_user_preferences = {};
-            if (Galaxy.user.attributes.preferences && "extra_user_preferences" in Galaxy.user.attributes.preferences) {
-                extra_user_preferences = JSON.parse(Galaxy.user.attributes.preferences.extra_user_preferences);
-            }
-            return;
-            "use_cached_job|use_cached_job_checkbox" in extra_user_preferences
-                ? extra_user_preferences["use_cached_job|use_cached_job_checkbox"] === "true"
-                : false;
-        },
         cacheInputs() {
             return [
                 {
@@ -152,6 +145,9 @@ export default {
         },
     },
     methods: {
+        cacheInputsAvailable(user) {
+            return reuseCachedJobs(user.preferences);
+        },
         getValidationScrollTo(stepId) {
             if (this.stepScrollTo.stepId == stepId) {
                 return this.stepScrollTo.stepError;
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 15b817a1396..41b440dc6d8 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -17,6 +17,7 @@
 </template>
 
 <script>
+import _ from "underscore";
 import FormDisplay from "components/Form/FormDisplay";
 import FormMessage from "components/Form/FormMessage";
 import FormCard from "components/Form/FormCard";
@@ -81,7 +82,6 @@ export default {
             visitInputs(this.model.inputs, (input, name) => {
                 params[name] = input;
             });
-            const values = this.formData;
             this.replaceParams = {};
             _.each(params, (input, name) => {
                 if (input.wp_linked || input.step_linked) {

From 545a1b5e27c898c3ea17ff4a85bc875c5dd407f4 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 17:32:10 -0400
Subject: [PATCH 207/452] Revise handling of using cached job option

---
 client/src/components/Tool/ToolForm.vue       | 10 ++---
 client/src/components/Tool/utilities.js       |  2 +-
 .../Workflow/Run/WorkflowRunForm.vue          | 39 +++++++------------
 3 files changed, 20 insertions(+), 31 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 79261cf89ec..a88ddf01d73 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -69,7 +69,7 @@
                                 title="Attempt to re-use jobs with identical parameters?"
                                 help="This may skip executing jobs that you have already run."
                                 type="boolean"
-                                v-model="reuse"
+                                v-model="useCachedJobs"
                             />
                         </template>
                         <template v-slot:buttons>
@@ -92,7 +92,7 @@
 import Scroller from "vue-scrollto";
 import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
-import { reuseCachedJobs } from "./utilities";
+import { allowCachedJobs } from "./utilities";
 import ToolCard from "./ToolCard";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
@@ -151,7 +151,7 @@ export default {
             errorContent: null,
             messageVariant: "",
             messageText: "",
-            reuse: false,
+            useCachedJobs: false,
             email: false,
             remap: false,
             entryPoints: [],
@@ -208,7 +208,7 @@ export default {
             return config.server_mail_configured && !user.isAnonymous;
         },
         reuseAllowed(user) {
-            return reuseCachedJobs(user.preferences);
+            return allowCachedJobs(user.preferences);
         },
         onValidation(validationInternal) {
             this.validationInternal = validationInternal;
@@ -261,7 +261,7 @@ export default {
             if (this.remap) {
                 jobDef.inputs["rerun_remap_job_id"] = this.job_id;
             }
-            if (this.reuse) {
+            if (this.useCachedJobs) {
                 jobDef.inputs["use_cached_job"] = true;
             }
             console.debug("toolForm::onExecute()", jobDef);
diff --git a/client/src/components/Tool/utilities.js b/client/src/components/Tool/utilities.js
index 5100338a473..c503f61141b 100644
--- a/client/src/components/Tool/utilities.js
+++ b/client/src/components/Tool/utilities.js
@@ -13,7 +13,7 @@ export function openLink(url) {
     window.open(url);
 }
 
-export function reuseCachedJobs(userPreferences) {
+export function allowCachedJobs(userPreferences) {
     if (userPreferences && "extra_user_preferences" in userPreferences) {
         const extra_user_preferences = JSON.parse(userPreferences.extra_user_preferences);
         const keyCached = "use_cached_job|use_cached_job_checkbox";
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 6ffa3275d5a..b2ec4ffdf40 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -21,9 +21,14 @@
                     <FormDisplay :inputs="historyInputs" @onChange="onHistoryInputs" />
                 </template>
             </FormCard>
-            <FormCard v-if="cacheInputsAvailable(user)" title="Job re-use Options">
+            <FormCard v-if="reuseAllowed(user)" title="Job re-use Options">
                 <template v-slot:body>
-                    <FormDisplay :inputs="cacheInputs" @onChange="onCacheInputs" />
+                    <FormElement
+                        title="Attempt to re-use jobs with identical parameters?"
+                        help="This may skip executing jobs that you have already run."
+                        type="boolean"
+                        v-model="useCachedJobs"
+                    />
                 </template>
             </FormCard>
             <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
@@ -57,17 +62,19 @@
 import CurrentUser from "components/providers/CurrentUser";
 import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";
+import FormElement from "components/Form/FormElement";
 import ButtonSpinner from "components/Common/ButtonSpinner";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
 import WorkflowRunToolStep from "./WorkflowRunToolStep";
 import { invokeWorkflow } from "./services";
-import { reuseCachedJobs } from "components/Tool/utilities";
+import { allowCachedJobs } from "components/Tool/utilities";
 
 export default {
     components: {
         CurrentUser,
         FormDisplay,
         FormCard,
+        FormElement,
         ButtonSpinner,
         WorkflowRunDefaultStep,
         WorkflowRunToolStep,
@@ -86,7 +93,7 @@ export default {
             stepScrollTo: {},
             wpData: {},
             historyData: {},
-            cacheData: {},
+            useCachedJobs: false,
             historyInputs: [
                 {
                     type: "conditional",
@@ -116,21 +123,6 @@ export default {
         };
     },
     computed: {
-        cacheInputs() {
-            return [
-                {
-                    type: "conditional",
-                    name: "use_cached_job",
-                    test_param: {
-                        name: "check",
-                        label: "Attempt to reuse jobs with identical parameters?",
-                        type: "boolean",
-                        value: "false",
-                        help: "This may skip executing jobs that you have already run.",
-                    },
-                },
-            ];
-        },
         resourceInputsAvailable() {
             return this.resourceInputs.length > 0;
         },
@@ -145,8 +137,8 @@ export default {
         },
     },
     methods: {
-        cacheInputsAvailable(user) {
-            return reuseCachedJobs(user.preferences);
+        reuseAllowed(user) {
+            return allowCachedJobs(user.preferences);
         },
         getValidationScrollTo(stepId) {
             if (this.stepScrollTo.stepId == stepId) {
@@ -170,9 +162,6 @@ export default {
         onWpInputs(data) {
             this.wpData = data;
         },
-        onCacheInputs(data) {
-            this.cacheData = data;
-        },
         onValidation(stepId, validation) {
             this.stepValidations[stepId] = validation;
         },
@@ -192,7 +181,7 @@ export default {
                 history_id: !this.historyData["new_history|name"] ? this.model.historyId : null,
                 resource_params: this.resourceData,
                 replacement_params: this.wpData,
-                use_cached_job: this.cacheData["use_cached_job|check"] === "true",
+                use_cached_job: useCachedJobs,
                 parameters: this.stepData,
                 // Tool form will submit flat maps for each parameter
                 // (e.g. "repeat_0|cond|param": "foo" instead of nested

From 5b845a263b6c767676c5c85a4de95046dbcbab3f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 17:52:46 -0400
Subject: [PATCH 208/452] Fix missing underscore import, fix option parsing

---
 client/src/components/Form/FormDisplay.vue             | 1 +
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 4708451b6fc..68f4a5cafdf 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -3,6 +3,7 @@
 </template>
 
 <script>
+import _ from "underscore";
 import Form from "mvc/form/form-view";
 import { visitInputs } from "components/Form/utilities";
 
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index b2ec4ffdf40..54a343355f9 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -181,7 +181,7 @@ export default {
                 history_id: !this.historyData["new_history|name"] ? this.model.historyId : null,
                 resource_params: this.resourceData,
                 replacement_params: this.wpData,
-                use_cached_job: useCachedJobs,
+                use_cached_job: this.useCachedJobs,
                 parameters: this.stepData,
                 // Tool form will submit flat maps for each parameter
                 // (e.g. "repeat_0|cond|param": "foo" instead of nested

From 43f15d3d853568aca85c9ca3c3a09eaec4de7f71 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 22:40:13 -0400
Subject: [PATCH 209/452] Filter connected inputs from workflow request payload

---
 .../src/components/Workflow/Run/WorkflowRunForm.vue | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 54a343355f9..d4bfc32bbe8 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -176,13 +176,24 @@ export default {
                 }
             }
 
+            const parameters = {};
+            Object.entries(this.stepData).forEach(([stepId, stepData]) => {
+                const stepDataFiltered = {};
+                Object.entries(stepData).forEach(([inputName, inputValue]) => {
+                    if (typeof inputValue !== "object" || inputValue.__class__ != "ConnectedValue") {
+                        stepDataFiltered[inputName] = inputValue;
+                    }
+                });
+                parameters[stepId] = stepDataFiltered;
+            });
+
             const jobDef = {
                 new_history_name: this.historyData["new_history|name"] ? this.historyData["new_history|name"] : null,
                 history_id: !this.historyData["new_history|name"] ? this.model.historyId : null,
                 resource_params: this.resourceData,
                 replacement_params: this.wpData,
                 use_cached_job: this.useCachedJobs,
-                parameters: this.stepData,
+                parameters: parameters,
                 // Tool form will submit flat maps for each parameter
                 // (e.g. "repeat_0|cond|param": "foo" instead of nested
                 // data structures).

From f4690f255dac13866d27c03109a7f193841b54d9 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 20 Jul 2021 23:02:32 -0400
Subject: [PATCH 210/452] Show input help for connected and collapsed inputs

---
 client/src/components/Workflow/Run/model.js |  2 +-
 client/src/mvc/form/form-input.js           | 10 +++-------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/client/src/components/Workflow/Run/model.js b/client/src/components/Workflow/Run/model.js
index a9c4bbc447b..c425c3f1cdc 100644
--- a/client/src/components/Workflow/Run/model.js
+++ b/client/src/components/Workflow/Run/model.js
@@ -104,7 +104,7 @@ export class WorkflowRunModel {
                     if (connection) {
                         input.type = "hidden";
                         input.help = input.step_linked ? `${input.help}, ` : "";
-                        input.help += `Output dataset '${connection.output_name}' from step ${parseInt(i) + 1}`;
+                        input.help += `Connected to '${connection.output_name}' from Step ${parseInt(i) + 1}`;
                         input.step_linked = input.step_linked || [];
                         input.step_linked.push({ index: step.index, step_type: step.step_type });
                     }
diff --git a/client/src/mvc/form/form-input.js b/client/src/mvc/form/form-input.js
index a4f84d75328..14de94a9db7 100644
--- a/client/src/mvc/form/form-input.js
+++ b/client/src/mvc/form/form-input.js
@@ -201,12 +201,8 @@ export default Backbone.View.extend({
                     )
                     .append($("<span/>").addClass("ui-form-title-text"))
             )
-            .append(
-                $("<div/>")
-                    .addClass("ui-form-field")
-                    .append($("<span/>").addClass("ui-form-info form-text text-muted mt-2"))
-                    .append($("<div/>").addClass("ui-form-backdrop"))
-            )
-            .append($("<div/>").addClass("ui-form-preview"));
+            .append($("<div/>").addClass("ui-form-field").append($("<div/>").addClass("ui-form-backdrop")))
+            .append($("<div/>").addClass("ui-form-preview"))
+            .append($("<span/>").addClass("ui-form-info form-text text-muted mb-2"));
     },
 });

From ed530564040d8a2ed4104b689e7b049652838eb8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 21 Jul 2021 00:06:11 -0400
Subject: [PATCH 211/452] Use connected value test as used in form-input

---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index d4bfc32bbe8..019078bee62 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -180,7 +180,7 @@ export default {
             Object.entries(this.stepData).forEach(([stepId, stepData]) => {
                 const stepDataFiltered = {};
                 Object.entries(stepData).forEach(([inputName, inputValue]) => {
-                    if (typeof inputValue !== "object" || inputValue.__class__ != "ConnectedValue") {
+                    if (JSON.stringify(inputValue) != JSON.stringify({ __class__: "ConnectedValue" })) {
                         stepDataFiltered[inputName] = inputValue;
                     }
                 });

From 7e9610f139be40d3d3fdf0355a25c4b6b009eea7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 21 Jul 2021 00:09:54 -0400
Subject: [PATCH 212/452] Test connected values without json parsing

---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 019078bee62..8d7f72de26c 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -180,7 +180,7 @@ export default {
             Object.entries(this.stepData).forEach(([stepId, stepData]) => {
                 const stepDataFiltered = {};
                 Object.entries(stepData).forEach(([inputName, inputValue]) => {
-                    if (JSON.stringify(inputValue) != JSON.stringify({ __class__: "ConnectedValue" })) {
+                    if (!inputValue || typeof inputValue !== "object" || inputValue.__class__ != "ConnectedValue") {
                         stepDataFiltered[inputName] = inputValue;
                     }
                 });

From 32e7f92867f506246c0f650b7397af9900234f8b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 21 Jul 2021 00:21:25 -0400
Subject: [PATCH 213/452] Fix job remap handling, parse history id on tool form
 data updates, use consistent option naming, fix appearance

---
 client/src/components/Form/FormCard.vue |  6 ++++--
 client/src/components/Tool/ToolCard.vue |  2 +-
 client/src/components/Tool/ToolForm.vue | 27 ++++++++++++-------------
 client/src/components/Tool/services.js  |  3 ++-
 client/src/mvc/form/form-view.js        |  1 -
 client/src/style/scss/ui.scss           | 11 +++-------
 6 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/client/src/components/Form/FormCard.vue b/client/src/components/Form/FormCard.vue
index ae104920783..88fca17d454 100644
--- a/client/src/components/Form/FormCard.vue
+++ b/client/src/components/Form/FormCard.vue
@@ -9,7 +9,7 @@
                     title="Collapse/Expand"
                     variant="link"
                     size="sm"
-                    class="float-right py-0"
+                    class="float-right"
                     v-b-tooltip.hover.bottom
                     @click="onCollapse"
                 >
@@ -73,7 +73,9 @@ export default {
     },
     methods: {
         onCollapse() {
-            this.$emit("update:collapsed", !this.collapsed);
+            if (this.collapsible) {
+                this.$emit("update:collapsed", !this.collapsed);
+            }
         },
     },
 };
diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue
index 68fa050ee22..1f06b2b9e85 100644
--- a/client/src/components/Tool/ToolCard.vue
+++ b/client/src/components/Tool/ToolCard.vue
@@ -79,7 +79,7 @@
                         <span class="fa fa-star" />
                     </b-button>
                 </div>
-                <div class="portlet-title my-1">
+                <div class="portlet-title">
                     <i class="portlet-title-icon fa mr-1 fa-wrench" style="display: inline"></i>
                     <span class="portlet-title-text">
                         <b itemprop="name">{{ title }}</b> <span itemprop="description">{{ description }}</span> (Galaxy
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index a88ddf01d73..b6b36b3cacb 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -53,7 +53,7 @@
                                 title="Email notification"
                                 help="Send an email notification when the job completes."
                                 type="boolean"
-                                v-model="email"
+                                v-model="useEmail"
                             />
                             <FormElement
                                 v-if="remapAllowed"
@@ -61,7 +61,7 @@
                                 :title="remapTitle"
                                 :help="remapHelp"
                                 type="boolean"
-                                v-model="remap"
+                                v-model="useJobRemapping"
                             />
                             <FormElement
                                 v-if="reuseAllowed(user)"
@@ -147,13 +147,14 @@ export default {
             error: null,
             formConfig: {},
             formData: {},
+            remapAllowed: false,
             errorTitle: null,
             errorContent: null,
             messageVariant: "",
             messageText: "",
             useCachedJobs: false,
-            email: false,
-            remap: false,
+            useEmail: false,
+            useJobRemapping: false,
             entryPoints: [],
             jobDef: {},
             jobResponse: {},
@@ -185,18 +186,15 @@ export default {
         inputs() {
             return this.formConfig.inputs;
         },
-        remapAllowed() {
-            return this.job_id && this.formConfig.job_remap;
-        },
         remapTitle() {
-            if (this.formConfig.job_remap === "job_produced_collection_elements") {
-                return "Replace elements in collection ?";
+            if (this.remapAllowed === "job_produced_collection_elements") {
+                return "Replace elements in collection?";
             } else {
-                return "Resume dependencies from this job ?";
+                return "Resume dependencies from this job?";
             }
         },
         remapHelp() {
-            if (this.formConfig.job_remap === "job_produced_collection_elements") {
+            if (this.remapAllowed === "job_produced_collection_elements") {
                 return "The previous run of this tool failed. Use this option to replace the failed element(s) in the dataset collection that were produced during the previous tool run.";
             } else {
                 return "The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.";
@@ -218,7 +216,7 @@ export default {
             this.onUpdate();
         },
         onUpdate() {
-            updateToolFormData(this.formConfig.id, this.currentVersion, this.formData).then((data) => {
+            updateToolFormData(this.formConfig.id, this.currentVersion, this.history_id, this.formData).then((data) => {
                 this.formConfig = data;
             });
         },
@@ -232,6 +230,7 @@ export default {
             this.currentVersion = newVersion || this.currentVersion;
             getToolFormData(this.id, this.currentVersion, this.job_id, this.history_id).then((data) => {
                 this.formConfig = data;
+                this.remapAllowed = this.job_id && data.job_remap;
                 this.showLoading = false;
                 this.showForm = true;
                 if (newVersion) {
@@ -255,10 +254,10 @@ export default {
                     ...this.formData,
                 },
             };
-            if (this.email) {
+            if (this.useEmail) {
                 jobDef.inputs["send_email_notification"] = true;
             }
-            if (this.remap) {
+            if (this.useJobRemapping) {
                 jobDef.inputs["rerun_remap_job_id"] = this.job_id;
             }
             if (this.useCachedJobs) {
diff --git a/client/src/components/Tool/services.js b/client/src/components/Tool/services.js
index f8df815e275..ad5a5720bc8 100644
--- a/client/src/components/Tool/services.js
+++ b/client/src/components/Tool/services.js
@@ -2,11 +2,12 @@ import axios from "axios";
 import { rethrowSimple } from "utils/simple-error";
 import { getAppRoot } from "onload/loadConfig";
 
-export async function updateToolFormData(tool_id, tool_version, inputs) {
+export async function updateToolFormData(tool_id, tool_version, history_id, inputs) {
     const current_state = {
         tool_id: tool_id,
         tool_version: tool_version,
         inputs: inputs,
+        history_id: history_id,
     };
     const url = `${getAppRoot()}api/tools/${tool_id}/build`;
     try {
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index fe0bc26a869..93ac9c7956d 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -32,7 +32,6 @@ export default Backbone.View.extend({
             if (field.update) {
                 field.update(node);
                 field.trigger("change");
-                console.debug("form-view::update()", `Updating input: ${input_id}`);
             }
         });
     },
diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss
index a752574eb50..afa56414df5 100644
--- a/client/src/style/scss/ui.scss
+++ b/client/src/style/scss/ui.scss
@@ -194,15 +194,10 @@ $ui-margin-horizontal-large: $margin-v * 2;
         @extend .rounded;
         background: $portlet-bg-color;
         border-color: transparent;
-        .portlet-title {
-            .portlet-title-text.collapsible {
-                cursor: pointer;
-                text-decoration: underline;
-            }
-        }
         .portlet-operations {
             button {
                 @extend .ml-1;
+                @extend .py-0;
             }
         }
     }
@@ -239,11 +234,11 @@ $ui-margin-horizontal-large: $margin-v * 2;
         background: $portlet-bg-color;
         @extend .rounded-0;
         border-top-right-radius: $border-radius-large !important;
-        padding: 5px;
+        @extend .py-1;
+        @extend .px-2;
     }
     > .portlet-content {
         @extend .pl-2;
-        @extend .pr-0;
     }
 }
 

From 20c83278ea2440c0078d07339867533f802e0d46 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 21 Jul 2021 03:25:42 -0400
Subject: [PATCH 214/452] Increase padding of form headers

---
 client/src/style/scss/ui.scss | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss
index afa56414df5..4ce0c217def 100644
--- a/client/src/style/scss/ui.scss
+++ b/client/src/style/scss/ui.scss
@@ -231,11 +231,10 @@ $ui-margin-horizontal-large: $margin-v * 2;
     border-left: solid 3px $portlet-bg-color;
     border-radius: $border-radius-large;
     .portlet-header:first-child {
-        background: $portlet-bg-color;
         @extend .rounded-0;
+        @extend .p-2;
+        background: $portlet-bg-color;
         border-top-right-radius: $border-radius-large !important;
-        @extend .py-1;
-        @extend .px-2;
     }
     > .portlet-content {
         @extend .pl-2;

From 6dad2344829e9143a8a6263c119156e7f91ec81b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 21 Jul 2021 03:41:59 -0400
Subject: [PATCH 215/452] Restore style for sub-sections

---
 client/src/style/scss/ui.scss | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/client/src/style/scss/ui.scss b/client/src/style/scss/ui.scss
index 4ce0c217def..4e347bba16a 100644
--- a/client/src/style/scss/ui.scss
+++ b/client/src/style/scss/ui.scss
@@ -194,6 +194,12 @@ $ui-margin-horizontal-large: $margin-v * 2;
         @extend .rounded;
         background: $portlet-bg-color;
         border-color: transparent;
+        .portlet-title {
+            .portlet-title-text.collapsible {
+                cursor: pointer;
+                text-decoration: underline;
+            }
+        }
         .portlet-operations {
             button {
                 @extend .ml-1;

From dc2112af3c08a380ead6d8211c65cc3460041f2f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 23 Jul 2021 08:01:08 -0400
Subject: [PATCH 216/452] Adjust selenium test to properly select control
 element in filter testing

---
 lib/galaxy/selenium/navigation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index c470af05574..11fc8a5c98c 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -84,7 +84,7 @@ toolbox_filters:
   selectors:
     input:
       type: xpath
-      selector: "//span[contains(. ,'${description}')]/../div/div[@class=\"custom-control\"]"
+      selector: "//span[contains(. ,'${description}')]/../div/div/div[@class=\"custom-control\"]"
     submit: '#submit'
 
 change_user_email:

From 8f975dcd4bed66f403618223ae69b1dd5ff98395 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 23 Jul 2021 10:11:48 -0400
Subject: [PATCH 217/452] Refactor streaming to util so toolbox.base doesn't
 depend on datatype code.

---
 lib/galaxy/datatypes/sniff.py    | 32 --------------------------------
 lib/galaxy/tools/toolbox/base.py |  4 ++--
 lib/galaxy/util/__init__.py      | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py
index 2e9323b3ee0..a899be82b5c 100644
--- a/lib/galaxy/datatypes/sniff.py
+++ b/lib/galaxy/datatypes/sniff.py
@@ -61,38 +61,6 @@ def stream_url_to_file(path, file_sources=None):
         return temp_name
 
 
-def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict'):
-    """Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor"""
-    # signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
-    CHUNK_SIZE = 1048576
-    try:
-        codecs.lookup(target_encoding)
-    except Exception:
-        target_encoding = util.DEFAULT_ENCODING  # utf-8
-    use_source_encoding = source_encoding is not None
-    while True:
-        chunk = stream.read(CHUNK_SIZE)
-        if not chunk:
-            break
-        if use_source_encoding:
-            # If a source encoding is given we use it to convert to the target encoding
-            try:
-                if not isinstance(chunk, str):
-                    chunk = chunk.decode(source_encoding, source_error)
-                os.write(fd, chunk.encode(target_encoding, target_error))
-            except UnicodeDecodeError:
-                use_source_encoding = False
-                os.write(fd, chunk)
-        else:
-            # Compressed files must be encoded after they are uncompressed in the upload utility,
-            # while binary files should not be encoded at all.
-            if isinstance(chunk, str):
-                chunk = chunk.encode(target_encoding, target_error)
-            os.write(fd, chunk)
-    os.close(fd)
-    return filename
-
-
 def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd):
     """Writes a stream to a temporary file, returns the temporary file's name"""
     fd, temp_name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index d99b43557d9..ce32b202369 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -10,7 +10,6 @@ from urllib.parse import urlparse
 
 from markupsafe import escape
 
-from galaxy.datatypes import sniff
 from galaxy.exceptions import (
     ConfigurationError,
     MessageException,
@@ -28,6 +27,7 @@ from galaxy.util import (
     listify,
     parse_xml,
     string_as_bool,
+    stream_to_open_named_file,
     unicodify,
 )
 from galaxy.util.bunch import Bunch
@@ -141,7 +141,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             log.debug('EDAM ontology file not present, downloading')
 
             page = urllib.request.urlopen('https://edamontology.org/EDAM.tsv')
-            sniff.stream_to_open_named_file(page, os.open(self.app.config.beta_edam_toolbox_ontology_path, os.O_WRONLY | os.O_CREAT), None, source_encoding=get_charset_from_http_headers(page.headers))
+            stream_to_open_named_file(page, os.open(self.app.config.beta_edam_toolbox_ontology_path, os.O_WRONLY | os.O_CREAT), None, source_encoding=get_charset_from_http_headers(page.headers))
 
         log.debug('Processing EDAM Terms')
         with open(self.app.config.beta_edam_toolbox_ontology_path, 'r') as handle:
diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index b4cb2253378..640ade67e7e 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -1674,6 +1674,38 @@ def download_to_file(url, dest_file_path, timeout=30, chunk_size=2 ** 20):
                 f.write(chunk)
 
 
+def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict'):
+    """Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor"""
+    # signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
+    CHUNK_SIZE = 1048576
+    try:
+        codecs.lookup(target_encoding)
+    except Exception:
+        target_encoding = util.DEFAULT_ENCODING  # utf-8
+    use_source_encoding = source_encoding is not None
+    while True:
+        chunk = stream.read(CHUNK_SIZE)
+        if not chunk:
+            break
+        if use_source_encoding:
+            # If a source encoding is given we use it to convert to the target encoding
+            try:
+                if not isinstance(chunk, str):
+                    chunk = chunk.decode(source_encoding, source_error)
+                os.write(fd, chunk.encode(target_encoding, target_error))
+            except UnicodeDecodeError:
+                use_source_encoding = False
+                os.write(fd, chunk)
+        else:
+            # Compressed files must be encoded after they are uncompressed in the upload utility,
+            # while binary files should not be encoded at all.
+            if isinstance(chunk, str):
+                chunk = chunk.encode(target_encoding, target_error)
+            os.write(fd, chunk)
+    os.close(fd)
+    return filename
+
+
 class classproperty:
 
     def __init__(self, f):

From 038045a600aad4d1b26eebd6711e9a73301e5f33 Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Fri, 23 Jul 2021 14:55:54 -0400
Subject: [PATCH 218/452] Add a GDPR-friendly user purge to pgcleanup.

---
 scripts/cleanup_datasets/pgcleanup.py | 125 +++++++++++++++++++++++++-
 1 file changed, 124 insertions(+), 1 deletion(-)

diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py
index 21cb488e5b1..a4498a5d438 100755
--- a/scripts/cleanup_datasets/pgcleanup.py
+++ b/scripts/cleanup_datasets/pgcleanup.py
@@ -12,6 +12,7 @@ import logging
 import os
 import string
 import sys
+import time
 from collections import namedtuple
 from functools import partial
 
@@ -105,6 +106,7 @@ class Action:
         self._debug = app.args.debug
         self._update_time = app.args.update_time
         self._force_retry = app.args.force_retry
+        self._epoch_time = str(int(time.time()))
         self._days = app.args.days
         self._config = app.config
         self._update = app._update
@@ -141,6 +143,7 @@ class Action:
         self.__log.propagate = False
         m = ('==== Log opened: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
         self.__log.info(m)
+        self.__log.info(f'Epoch time for this action: {self._epoch_time}')
 
     def __close_log(self):
         m = ('==== Log closed: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
@@ -188,6 +191,7 @@ class Action:
         return self._action_sql.format(
             update_time_sql=self._update_time_sql,
             force_retry_sql=self._force_retry_sql,
+            epoch_time=self._epoch_time,
         )
 
     @property
@@ -334,11 +338,13 @@ class PurgesHDAs:
         _purge_hda_dependencies_sql = self._purge_hda_dependencies_sql.format(
             update_time_sql=self._update_time_sql,
             force_retry_sql=self._force_retry_sql,
+            epoch_time=self._epoch_time,
         )
         return self._action_sql.format(
             purge_hda_dependencies_sql=_purge_hda_dependencies_sql,
             update_time_sql=self._update_time_sql,
             force_retry_sql=self._force_retry_sql,
+            epoch_time=self._epoch_time,
         )
 
 
@@ -525,6 +531,7 @@ class DeleteInactiveUsers(Action):
 class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
     """
     - Mark purged all users that are older than the specified number of days.
+    - Set User.disk_usage = 0 for all users purged in this step.
     - Mark purged all Histories whose user_ids are purged in this step.
     - Mark purged all HistoryDatasetAssociations whose history_ids are purged in this step.
     - Delete all UserGroupAssociations whose user_ids are purged in this step.
@@ -655,6 +662,121 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
         self.log.info('zero_disk_usage user_ids: %s', ' '.join(str(i) for i in user_ids))
 
 
+class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
+    """
+    - Perform all steps in the PurgeDeletedUsers/purge_deleted_users action
+    - Obfuscate User.email and User.username for all users purged in this step.
+
+    NOTE: Your database must have the pgcrypto extension installed e.g. with:
+      CREATE EXTENSION IF NOT EXISTS pgcrypto;
+    """
+    _action_sql = """
+        WITH purged_user_ids
+          AS (     UPDATE galaxy_user
+                      SET purged = true,
+                          disk_usage = 0,
+                          email = encode(digest(email || '{epoch_time}', 'sha1'), 'hex'),
+                          username = encode(digest(username || '{epoch_time}', 'sha1'), 'hex'){update_time_sql}
+                    WHERE deleted{force_retry_sql}
+                          AND update_time < (NOW() AT TIME ZONE 'utc' - interval '%(days)s days')
+                RETURNING id),
+             deleted_uga_ids
+          AS (DELETE FROM user_group_association
+                    USING purged_user_ids
+                    WHERE user_group_association.user_id = purged_user_ids.id
+                RETURNING user_group_association.user_id AS user_id,
+                          user_group_association.id AS id),
+             deleted_ura_ids
+          AS (DELETE FROM user_role_association
+                    USING role
+                    WHERE role.id = user_role_association.role_id
+                          AND role.type != 'private'
+                          AND user_role_association.user_id IN
+                            (SELECT id
+                               FROM purged_user_ids)
+                RETURNING user_role_association.user_id AS user_id,
+                          user_role_association.id AS id),
+             deleted_ua_ids
+          AS (DELETE FROM user_address
+                    USING purged_user_ids
+                    WHERE user_address.user_id = purged_user_ids.id
+                RETURNING user_address.user_id AS user_id,
+                          user_address.id AS id),
+             user_events
+          AS (INSERT INTO cleanup_event_user_association
+                          (create_time, cleanup_event_id, user_id)
+                   SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
+                     FROM purged_user_ids),
+             purged_history_ids
+          AS (     UPDATE history
+                      SET purged = true{update_time_sql}
+                     FROM purged_user_ids
+                    WHERE purged_user_ids.id = history.user_id
+                          AND NOT history.purged
+                RETURNING history.user_id AS user_id,
+                          history.id AS id),
+             history_events
+          AS (INSERT INTO cleanup_event_history_association
+                          (create_time, cleanup_event_id, history_id)
+                   SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
+                     FROM purged_history_ids),
+             purged_hda_ids
+          AS (     UPDATE history_dataset_association
+                      SET purged = true, deleted = true{update_time_sql}
+                     FROM purged_history_ids
+                    WHERE purged_history_ids.id = history_dataset_association.history_id
+                          AND NOT history_dataset_association.purged
+                RETURNING history_dataset_association.history_id AS history_id,
+                          history_dataset_association.id AS id),
+             hda_events
+          AS (INSERT INTO cleanup_event_hda_association
+                          (create_time, cleanup_event_id, hda_id)
+                   SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
+                     FROM purged_hda_ids),
+             {purge_hda_dependencies_sql}
+      SELECT purged_user_ids.id AS purged_user_id,
+             purged_user_ids.id AS zero_disk_usage_user_id,
+             purged_history_ids.id AS purged_history_id,
+             purged_hda_ids.id AS purged_hda_id,
+             deleted_metadata_file_ids.id AS deleted_metadata_file_id,
+             deleted_metadata_file_ids.object_store_id AS object_store_id,
+             deleted_icda_ids.id AS deleted_icda_id,
+             deleted_icda_ids.hda_id AS deleted_icda_hda_id,
+             deleted_uga_ids.id AS deleted_uga_id,
+             deleted_ura_ids.id AS deleted_ura_id,
+             deleted_ua_ids.id AS deleted_ua_id
+        FROM purged_user_ids
+             LEFT OUTER JOIN purged_history_ids
+                             ON purged_user_ids.id = purged_history_ids.user_id
+             LEFT OUTER JOIN purged_hda_ids
+                             ON purged_history_ids.id = purged_hda_ids.history_id
+             LEFT OUTER JOIN deleted_metadata_file_ids
+                             ON deleted_metadata_file_ids.hda_id = purged_hda_ids.id
+             LEFT OUTER JOIN deleted_icda_ids
+                             ON deleted_icda_ids.hda_parent_id = purged_hda_ids.id
+             LEFT OUTER JOIN deleted_uga_ids
+                             ON purged_user_ids.id = deleted_uga_ids.user_id
+             LEFT OUTER JOIN deleted_ura_ids
+                             ON purged_user_ids.id = deleted_ura_ids.user_id
+             LEFT OUTER JOIN deleted_ua_ids
+                             ON purged_user_ids.id = deleted_ua_ids.user_id
+    ORDER BY purged_user_ids.id
+    """
+    causals = (
+        ('purged_user_id', 'purged_history_id'),
+        ('purged_history_id', 'purged_hda_id'),
+        ('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
+        ('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
+        ('purged_user_id', 'deleted_uga_id'),
+        ('purged_user_id', 'deleted_ura_id'),
+        ('purged_user_id', 'deleted_ua_id'),
+    )
+
+    @classmethod
+    def name_c(cls):
+        return 'purge_deleted_users_gdpr'
+
+
 class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalculation, Action):
     """
     - Mark purged all HistoryDatasetAssociations currently marked deleted that are older than the
@@ -1103,7 +1225,8 @@ class Cleanup:
 
     def _execute(self, sql, args):
         cur = self.conn.cursor()
-        log.debug("SQL is: %s", cur.mogrify(sql, args))
+        sql_str = cur.mogrify(sql, args).decode('utf-8')
+        log.debug(f"SQL is: {sql_str}")
         log.info("Executing SQL")
         cur.execute(sql, args)
         log.info('Database status: %s', cur.statusmessage)

From 356ecd4281410bbbe267bdf456317ca711806b11 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 23 Jul 2021 18:21:03 -0400
Subject: [PATCH 219/452] Move options from workflow model to Vue props

---
 client/src/components/Form/FormDisplay.vue         | 14 ++++++++++++++
 .../Workflow/Run/WorkflowRunToolStep.vue           |  2 ++
 client/src/components/Workflow/Run/model.js        |  9 ---------
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 68f4a5cafdf..23f95003bef 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -17,6 +17,18 @@ export default {
             type: Array,
             required: true,
         },
+        sustainRepeats: {
+            type: Boolean,
+            default: false,
+        },
+        sustainConditionals: {
+            type: Boolean,
+            default: false,
+        },
+        hideOperations: {
+            type: Boolean,
+            default: false,
+        },
         textEnable: {
             type: String,
             default: null,
@@ -164,6 +176,8 @@ export default {
                     initial_errors: this.initialErrors,
                     text_enable: this.textEnable,
                     text_disable: this.textDisable,
+                    sustain_repeats: this.sustainRepeats,
+                    sustain_conditionals: this.sustainConditionals,
                     onchange: () => {
                         this.onChange();
                     },
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 41b440dc6d8..76328b9cf93 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -6,6 +6,8 @@
                 <FormDisplay
                     :inputs="model.inputs"
                     :form-config="formConfig"
+                    :sustainRepeats="true"
+                    :sustainConditionals="true"
                     :replace-params="replaceParams"
                     :validation-scroll-to="validationScrollTo"
                     @onChange="onChange"
diff --git a/client/src/components/Workflow/Run/model.js b/client/src/components/Workflow/Run/model.js
index c425c3f1cdc..5219c4db584 100644
--- a/client/src/components/Workflow/Run/model.js
+++ b/client/src/components/Workflow/Run/model.js
@@ -44,20 +44,11 @@ export class WorkflowRunModel {
                     citations: null,
                     collapsible: true,
                     collapsed: i > 0 && !isDataStep(step),
-                    sustain_version: true,
-                    sustain_repeats: true,
-                    sustain_conditionals: true,
-                    narrow: true,
                     text_enable: "Edit",
                     text_disable: "Undo",
                     cls_enable: "fa fa-edit",
                     cls_disable: "fa fa-undo",
                     errors: step.messages,
-                    initial_errors: true,
-                    cls: "ui-portlet-section",
-                    hide_operations: true,
-                    needs_refresh: false,
-                    always_refresh: step.step_type != "tool",
                 },
                 step
             );

From 8f384a8197f27e02cf69840cf755c0d97a957148 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 23 Jul 2021 18:37:07 -0400
Subject: [PATCH 220/452] Hyphenate form props

---
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 76328b9cf93..5dc2fc97d56 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -6,8 +6,8 @@
                 <FormDisplay
                     :inputs="model.inputs"
                     :form-config="formConfig"
-                    :sustainRepeats="true"
-                    :sustainConditionals="true"
+                    :sustain-repeats="true"
+                    :sustain-conditionals="true"
                     :replace-params="replaceParams"
                     :validation-scroll-to="validationScrollTo"
                     @onChange="onChange"

From 2a23d8ba1df9a15fb6b34e2c32b4e7497bae29ac Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 23 Jul 2021 21:46:22 -0400
Subject: [PATCH 221/452] Add additional wait for transiently failing history
 selenium test

---
 lib/galaxy_test/selenium/test_history_panel.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy_test/selenium/test_history_panel.py b/lib/galaxy_test/selenium/test_history_panel.py
index d6008255a24..01206979646 100644
--- a/lib/galaxy_test/selenium/test_history_panel.py
+++ b/lib/galaxy_test/selenium/test_history_panel.py
@@ -187,6 +187,7 @@ class HistoryPanelTestCase(SeleniumTestCase):
         self.history_panel_ensure_showing_item_details(hid=1)
         self.history_panel_item_body_component(1, wait=True)
         self.history_panel_refresh_click()
+        self.wait_for_history()
 
         # After the refresh, verify the details are still open.
         self.sleep_for(self.wait_types.UX_TRANSITION)

From 55ba08e9e55ffd483c0c33a8ac3f8b99c9f332c3 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Sat, 24 Jul 2021 18:50:38 +0530
Subject: [PATCH 222/452] Refactor k8s conditional

Co-authored-by: Marius van den Beek <m.vandenbeek@gmail.com>
---
 lib/galaxy/jobs/runners/util/pykube_util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py
index 066cd8f9bca..2a2b069110c 100644
--- a/lib/galaxy/jobs/runners/util/pykube_util.py
+++ b/lib/galaxy/jobs/runners/util/pykube_util.py
@@ -89,7 +89,7 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None):
 
 
 def is_pod_unschedulable(pykube_api, pod, namespace=None):
-    is_unschedulable = bool(len([c for c in pod.obj['status'].get('conditions', []) if c.get("reason") == "Unschedulable"]))
+    is_unschedulable = any(c.get("reason") == "Unschedulable" for c in pod.obj['status'].get('conditions', []))
     if pod.obj['status'].get('phase') == "Pending" and is_unschedulable:
         return True
 

From 47dc4e6d282b943b3a3e931a8544ff87bb75ca7c Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Sun, 25 Jul 2021 15:00:44 -0400
Subject: [PATCH 223/452] Correctly pass along user param from admin user
 management

---
 client/src/components/User/UserPreferencesModel.js | 13 +++++++------
 client/src/entry/analysis/AnalysisRouter.js        |  3 +--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/client/src/components/User/UserPreferencesModel.js b/client/src/components/User/UserPreferencesModel.js
index 16e69b87847..110bfc4cbd4 100644
--- a/client/src/components/User/UserPreferencesModel.js
+++ b/client/src/components/User/UserPreferencesModel.js
@@ -1,15 +1,16 @@
 import { getGalaxyInstance } from "app";
 import _l from "utils/localization";
 
-export const getUserPreferencesModel = () => {
+export const getUserPreferencesModel = (user_id) => {
     const Galaxy = getGalaxyInstance();
     const config = Galaxy.config;
+    user_id = user_id || Galaxy.user.id;
     return {
         information: {
             title: _l("Manage Information"),
             id: "edit-preferences-information",
             description: "Edit your email, addresses and custom parameters or change your public name.",
-            url: `api/users/${Galaxy.user.id}/information/inputs`,
+            url: `api/users/${user_id}/information/inputs`,
             icon: "fa-user",
             redirect: "user",
             shouldRender: !config.use_remote_user,
@@ -19,7 +20,7 @@ export const getUserPreferencesModel = () => {
             id: "edit-preferences-password",
             description: _l("Allows you to change your login credentials."),
             icon: "fa-unlock-alt",
-            url: `api/users/${Galaxy.user.id}/password/inputs`,
+            url: `api/users/${user_id}/password/inputs`,
             submit_title: "Save Password",
             redirect: "user",
             shouldRender: !config.use_remote_user,
@@ -38,7 +39,7 @@ export const getUserPreferencesModel = () => {
             id: "edit-preferences-permissions",
             description:
                 "Grant others default access to newly created histories. Changes made here will only affect histories created after these settings have been stored.",
-            url: `api/users/${Galaxy.user.id}/permissions/inputs`,
+            url: `api/users/${user_id}/permissions/inputs`,
             icon: "fa-users",
             submit_title: "Save Permissions",
             redirect: "user",
@@ -55,7 +56,7 @@ export const getUserPreferencesModel = () => {
             title: _l("Manage API Key"),
             id: "edit-preferences-api-key",
             description: _l("Access your current API key or create a new one."),
-            url: `api/users/${Galaxy.user.id}/api_key/inputs`,
+            url: `api/users/${user_id}/api_key/inputs`,
             icon: "fa-key",
             submit_title: "Create a new Key",
             submit_icon: "fa-check",
@@ -72,7 +73,7 @@ export const getUserPreferencesModel = () => {
             title: _l("Manage Toolbox Filters"),
             id: "edit-preferences-toolbox-filters",
             description: _l("Customize your Toolbox by displaying or omitting sets of Tools."),
-            url: `api/users/${Galaxy.user.id}/toolbox_filters/inputs`,
+            url: `api/users/${user_id}/toolbox_filters/inputs`,
             icon: "fa-filter",
             submit_title: "Save Filters",
             redirect: "user",
diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index b409c8374fc..e78327b9d62 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -152,8 +152,7 @@ export const getAnalysisRouter = (Galaxy) => {
 
         show_user_form: function (form_id) {
             const Galaxy = getGalaxyInstance();
-            const model = getUserPreferencesModel();
-            model.user_id = Galaxy.params.id;
+            const model = getUserPreferencesModel(Galaxy.params.id);
             this.page.display(new FormWrapper.View(_.extend(model[form_id], { active_tab: "user" })));
         },
 

From 99ede0da11978c9b1cdbbdb086e2762654429514 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Sun, 25 Jul 2021 15:01:38 -0400
Subject: [PATCH 224/452] Drop Galaxy object dependency here; use parsed params
 from router

---
 client/src/entry/analysis/AnalysisRouter.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index e78327b9d62..3ce25839b94 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -150,9 +150,8 @@ export const getAnalysisRouter = (Galaxy) => {
             });
         },
 
-        show_user_form: function (form_id) {
-            const Galaxy = getGalaxyInstance();
-            const model = getUserPreferencesModel(Galaxy.params.id);
+        show_user_form: function (form_id, params) {
+            const model = getUserPreferencesModel(params.id);
             this.page.display(new FormWrapper.View(_.extend(model[form_id], { active_tab: "user" })));
         },
 

From fbd2fb10ff115399c31e46e59da133f7bfe9de88 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 23 Jul 2021 10:19:56 -0400
Subject: [PATCH 225/452] Refactor base toolbox code de-coupled from app &
 config into tool_util.

This was originally decoupled by me for this purpose and this code was in galaxy-lib years ago, but it just never made it into galaxy-tool-util. This commit clears up that oversight I think.
---
 lib/galaxy/config_watchers.py                                 | 2 +-
 lib/galaxy/datatypes/sniff.py                                 | 3 +--
 lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py | 4 ++--
 lib/galaxy/{tools => tool_util}/toolbox/__init__.py           | 0
 lib/galaxy/{tools => tool_util}/toolbox/base.py               | 2 +-
 lib/galaxy/{tools => tool_util}/toolbox/filters/__init__.py   | 2 +-
 .../{tools => tool_util}/toolbox/filters/examples.py.sample   | 0
 lib/galaxy/{tools => tool_util}/toolbox/integrated_panel.py   | 0
 lib/galaxy/{tools => tool_util}/toolbox/lineages/__init__.py  | 0
 lib/galaxy/{tools => tool_util}/toolbox/lineages/factory.py   | 0
 lib/galaxy/{tools => tool_util}/toolbox/lineages/interface.py | 0
 lib/galaxy/{tools => tool_util}/toolbox/panel.py              | 0
 lib/galaxy/{tools => tool_util}/toolbox/parser.py             | 0
 lib/galaxy/{tools => tool_util}/toolbox/tags.py               | 0
 lib/galaxy/{tools => tool_util}/toolbox/watcher.py            | 0
 lib/galaxy/tools/__init__.py                                  | 2 +-
 lib/galaxy/tools/cache.py                                     | 2 +-
 lib/galaxy/tools/filters/__init__.py                          | 2 +-
 lib/galaxy/util/__init__.py                                   | 3 ++-
 lib/galaxy/util/odict.py                                      | 2 +-
 lib/galaxy/util/watcher.py                                    | 2 +-
 lib/galaxy/webapps/galaxy/api/users.py                        | 2 +-
 lib/galaxy/webapps/galaxy/config_schema.yml                   | 2 +-
 packages/app/requirements.txt                                 | 1 -
 packages/app/setup.py                                         | 3 ---
 packages/tool_util/requirements.txt                           | 1 +
 packages/tool_util/setup.py                                   | 3 +++
 test/unit/tool_util/toolbox/__init__.py                       | 0
 test/unit/{tools => tool_util/toolbox}/test_tool_panel.py     | 2 +-
 .../unit/{tools => tool_util/toolbox}/test_toolbox_filters.py | 2 +-
 test/unit/{tools => tool_util/toolbox}/test_watcher.py        | 2 +-
 test/unit/tools/conftest.py                                   | 2 +-
 32 files changed, 23 insertions(+), 23 deletions(-)
 rename lib/galaxy/{tools => tool_util}/toolbox/__init__.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/base.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/filters/__init__.py (98%)
 rename lib/galaxy/{tools => tool_util}/toolbox/filters/examples.py.sample (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/integrated_panel.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/lineages/__init__.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/lineages/factory.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/lineages/interface.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/panel.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/parser.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/tags.py (100%)
 rename lib/galaxy/{tools => tool_util}/toolbox/watcher.py (100%)
 create mode 100644 test/unit/tool_util/toolbox/__init__.py
 rename test/unit/{tools => tool_util/toolbox}/test_tool_panel.py (92%)
 rename test/unit/{tools => tool_util/toolbox}/test_toolbox_filters.py (98%)
 rename test/unit/{tools => tool_util/toolbox}/test_watcher.py (98%)

diff --git a/lib/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py
index c20ed8651e5..4601319deba 100644
--- a/lib/galaxy/config_watchers.py
+++ b/lib/galaxy/config_watchers.py
@@ -3,7 +3,7 @@ from os.path import dirname
 
 from galaxy.queue_worker import job_rule_modules
 from galaxy.structured_app import StructuredApp
-from galaxy.tools.toolbox.watcher import (
+from galaxy.tool_util.toolbox.watcher import (
     get_tool_conf_watcher,
     get_tool_watcher,
 )
diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py
index a899be82b5c..e67725ce893 100644
--- a/lib/galaxy/datatypes/sniff.py
+++ b/lib/galaxy/datatypes/sniff.py
@@ -3,7 +3,6 @@ File format detector
 """
 
 import bz2
-import codecs
 import gzip
 import io
 import logging
@@ -17,7 +16,7 @@ import urllib.request
 import zipfile
 
 from galaxy import util
-from galaxy.util import compression_utils
+from galaxy.util import compression_utils, stream_to_open_named_file
 from galaxy.util.checkers import (
     check_binary,
     check_bz2,
diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py b/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
index 8e39c3fe002..f7e49548d50 100644
--- a/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
+++ b/lib/galaxy/tool_shed/galaxy_install/tool_migration_manager.py
@@ -23,8 +23,8 @@ from galaxy.tool_shed.util.basic_util import remove_dir, strip_path
 from galaxy.tool_shed.util.hg_util import clone_repository, get_config_from_disk
 from galaxy.tool_shed.util.repository_util import create_or_update_tool_shed_repository
 from galaxy.tool_shed.util.tool_dependency_util import create_tool_dependency_objects
-from galaxy.tools.toolbox import ToolSection
-from galaxy.tools.toolbox.parser import ensure_tool_conf_item
+from galaxy.tool_util.toolbox import ToolSection
+from galaxy.tool_util.toolbox.parser import ensure_tool_conf_item
 from galaxy.util import RW_R__R__
 from galaxy.util.tool_shed import common_util
 from galaxy.util.tool_shed.xml_util import parse_xml
diff --git a/lib/galaxy/tools/toolbox/__init__.py b/lib/galaxy/tool_util/toolbox/__init__.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/__init__.py
rename to lib/galaxy/tool_util/toolbox/__init__.py
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/base.py
rename to lib/galaxy/tool_util/toolbox/base.py
index ce32b202369..4728139c2ff 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -26,8 +26,8 @@ from galaxy.util import (
     get_charset_from_http_headers,
     listify,
     parse_xml,
-    string_as_bool,
     stream_to_open_named_file,
+    string_as_bool,
     unicodify,
 )
 from galaxy.util.bunch import Bunch
diff --git a/lib/galaxy/tools/toolbox/filters/__init__.py b/lib/galaxy/tool_util/toolbox/filters/__init__.py
similarity index 98%
rename from lib/galaxy/tools/toolbox/filters/__init__.py
rename to lib/galaxy/tool_util/toolbox/filters/__init__.py
index 87bb8d696a2..3bdefbc98d6 100644
--- a/lib/galaxy/tools/toolbox/filters/__init__.py
+++ b/lib/galaxy/tool_util/toolbox/filters/__init__.py
@@ -22,7 +22,7 @@ class FilterFactory:
         self.default_filters = dict(tool=[_not_hidden, _handle_authorization], section=[], label=[])
         # Add dynamic filters to these default filters.
         config = toolbox.app.config
-        self.__base_modules = listify(getattr(config, "toolbox_filter_base_modules", "galaxy.tools.filters"))
+        self.__base_modules = listify(getattr(config, "toolbox_filter_base_modules", "galaxy.tool_util.filters"))
         self.__init_filters("tool", getattr(config, "tool_filters", ""), self.default_filters)
         self.__init_filters("section", getattr(config, "tool_section_filters", ""), self.default_filters)
         self.__init_filters("label", getattr(config, "tool_label_filters", ""), self.default_filters)
diff --git a/lib/galaxy/tools/toolbox/filters/examples.py.sample b/lib/galaxy/tool_util/toolbox/filters/examples.py.sample
similarity index 100%
rename from lib/galaxy/tools/toolbox/filters/examples.py.sample
rename to lib/galaxy/tool_util/toolbox/filters/examples.py.sample
diff --git a/lib/galaxy/tools/toolbox/integrated_panel.py b/lib/galaxy/tool_util/toolbox/integrated_panel.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/integrated_panel.py
rename to lib/galaxy/tool_util/toolbox/integrated_panel.py
diff --git a/lib/galaxy/tools/toolbox/lineages/__init__.py b/lib/galaxy/tool_util/toolbox/lineages/__init__.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/lineages/__init__.py
rename to lib/galaxy/tool_util/toolbox/lineages/__init__.py
diff --git a/lib/galaxy/tools/toolbox/lineages/factory.py b/lib/galaxy/tool_util/toolbox/lineages/factory.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/lineages/factory.py
rename to lib/galaxy/tool_util/toolbox/lineages/factory.py
diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tool_util/toolbox/lineages/interface.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/lineages/interface.py
rename to lib/galaxy/tool_util/toolbox/lineages/interface.py
diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/panel.py
rename to lib/galaxy/tool_util/toolbox/panel.py
diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tool_util/toolbox/parser.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/parser.py
rename to lib/galaxy/tool_util/toolbox/parser.py
diff --git a/lib/galaxy/tools/toolbox/tags.py b/lib/galaxy/tool_util/toolbox/tags.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/tags.py
rename to lib/galaxy/tool_util/toolbox/tags.py
diff --git a/lib/galaxy/tools/toolbox/watcher.py b/lib/galaxy/tool_util/toolbox/watcher.py
similarity index 100%
rename from lib/galaxy/tools/toolbox/watcher.py
rename to lib/galaxy/tool_util/toolbox/watcher.py
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index c6e71906154..a62ddf0e7f0 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -46,6 +46,7 @@ from galaxy.tool_util.parser import (
 )
 from galaxy.tool_util.parser.xml import XmlPageSource
 from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
+from galaxy.tool_util.toolbox import BaseGalaxyToolBox
 from galaxy.tools import expressions
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
@@ -80,7 +81,6 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator
 from galaxy.tools.parameters.meta import expand_meta_parameters
 from galaxy.tools.parameters.wrapped_json import json_wrap
 from galaxy.tools.test import parse_tests
-from galaxy.tools.toolbox import BaseGalaxyToolBox
 from galaxy.util import (
     in_directory,
     listify,
diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py
index dfdde299f8c..69617e5eda5 100644
--- a/lib/galaxy/tools/cache.py
+++ b/lib/galaxy/tools/cache.py
@@ -17,7 +17,7 @@ from sqlitedict import SqliteDict
 
 from galaxy.model.tool_shed_install import ToolShedRepository
 from galaxy.structured_app import MinimalManagerApp
-from galaxy.tools.toolbox.base import ToolConfRepository
+from galaxy.tool_util.toolbox.base import ToolConfRepository
 from galaxy.util import unicodify
 from galaxy.util.hash_util import md5_hash_file
 
diff --git a/lib/galaxy/tools/filters/__init__.py b/lib/galaxy/tools/filters/__init__.py
index c984816ff30..32b2b02421b 100644
--- a/lib/galaxy/tools/filters/__init__.py
+++ b/lib/galaxy/tools/filters/__init__.py
@@ -1,5 +1,5 @@
 """Deprecated module for toolbox filters.
 
 Filters placed in this module will still work - but filters should be
-moved to lib/galaxy/tools/toolbox/filters.
+moved to lib/galaxy/tool_util/toolbox/filters.
 """
diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index 640ade67e7e..4c236fadfe7 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -4,6 +4,7 @@ Utility functions used systemwide.
 """
 
 import binascii
+import codecs
 import collections
 import errno
 import importlib
@@ -1681,7 +1682,7 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source
     try:
         codecs.lookup(target_encoding)
     except Exception:
-        target_encoding = util.DEFAULT_ENCODING  # utf-8
+        target_encoding = DEFAULT_ENCODING  # utf-8
     use_source_encoding = source_encoding is not None
     while True:
         chunk = stream.read(CHUNK_SIZE)
diff --git a/lib/galaxy/util/odict.py b/lib/galaxy/util/odict.py
index b28de5642b4..ca284dfb459 100644
--- a/lib/galaxy/util/odict.py
+++ b/lib/galaxy/util/odict.py
@@ -2,7 +2,7 @@
 Ordered dictionary implementation with `insert` functionality.
 
 This is only used in one specific place in the codebase:
-    galaxy.tools.toolbox.panel
+    galaxy.tool_util.toolbox.panel
 
 Whenever possible the stdlib `collections.OrderedDict` should be used instead of
 this custom implementation.
diff --git a/lib/galaxy/util/watcher.py b/lib/galaxy/util/watcher.py
index 4c3de5ef198..1ed77c21512 100644
--- a/lib/galaxy/util/watcher.py
+++ b/lib/galaxy/util/watcher.py
@@ -1,4 +1,4 @@
-# TODO: this is largely copied from galaxy.tools.toolbox.galaxy and generalized, the tool-oriented watchers in that
+# TODO: this is largely copied from galaxy.tool_util.toolbox.galaxy and generalized, the tool-oriented watchers in that
 # module should probably be updated to use this where possible
 
 
diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py
index 25233326d73..607e8236ca9 100644
--- a/lib/galaxy/webapps/galaxy/api/users.py
+++ b/lib/galaxy/webapps/galaxy/api/users.py
@@ -30,7 +30,7 @@ from galaxy.security.validate_user_input import (
     validate_password,
     validate_publicname
 )
-from galaxy.tools.toolbox.filters import FilterFactory
+from galaxy.tool_util.toolbox.filters import FilterFactory
 from galaxy.util import (
     docstring_trim,
     listify
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 26bb3c91639..be85ddc24ef 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -3231,7 +3231,7 @@ mapping:
 
       toolbox_filter_base_modules:
         type: str
-        default: galaxy.tools.filters,galaxy.tools.toolbox.filters
+        default: galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters
         required: false
         desc: |
           The base module(s) that are searched for modules for toolbox filtering
diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt
index c7ff5fbe1b6..4faa725a7df 100644
--- a/packages/app/requirements.txt
+++ b/packages/app/requirements.txt
@@ -14,7 +14,6 @@ gxformat2
 Mako
 sqlitedict
 svgwrite
-sortedcontainers
 Markdown
 refgenconf>=0.7.0,<0.10.0 # https://github.com/galaxyproject/galaxy/issues/11601
 Fabric3
diff --git a/packages/app/setup.py b/packages/app/setup.py
index a69668442c4..f0b961fea2a 100644
--- a/packages/app/setup.py
+++ b/packages/app/setup.py
@@ -61,9 +61,6 @@ PACKAGES = [
     'galaxy.tools.imp_exp',
     'galaxy.tools.parameters',
     'galaxy.tools.search',
-    'galaxy.tools.toolbox',
-    'galaxy.tools.toolbox.filters',
-    'galaxy.tools.toolbox.lineages',
     'galaxy.tools.util',
     'galaxy.tools.util.galaxyops',
     'galaxy.tool_shed',
diff --git a/packages/tool_util/requirements.txt b/packages/tool_util/requirements.txt
index 19aaeae66b0..f0ac0b2b776 100644
--- a/packages/tool_util/requirements.txt
+++ b/packages/tool_util/requirements.txt
@@ -1,3 +1,4 @@
 galaxy-util>=20.1.0.dev0
 galaxy-containers
 lxml
+sortedcontainers
diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py
index d4c98ed57eb..2d3a6a349ad 100644
--- a/packages/tool_util/setup.py
+++ b/packages/tool_util/setup.py
@@ -40,6 +40,9 @@ PACKAGES = [
     'galaxy.tool_util.linters',
     'galaxy.tool_util.locations',
     'galaxy.tool_util.parser',
+    'galaxy.tool_util.toolbox',
+    'galaxy.tool_util.toolbox.filters',
+    'galaxy.tool_util.toolbox.lineages',
     'galaxy.tool_util.verify',
     'galaxy.tool_util.verify.asserts',
 ]
diff --git a/test/unit/tool_util/toolbox/__init__.py b/test/unit/tool_util/toolbox/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/test/unit/tools/test_tool_panel.py b/test/unit/tool_util/toolbox/test_tool_panel.py
similarity index 92%
rename from test/unit/tools/test_tool_panel.py
rename to test/unit/tool_util/toolbox/test_tool_panel.py
index c58eb8046ca..3beca3b57be 100644
--- a/test/unit/tools/test_tool_panel.py
+++ b/test/unit/tool_util/toolbox/test_tool_panel.py
@@ -1,4 +1,4 @@
-from galaxy.tools.toolbox import ToolSection
+from galaxy.tool_util.toolbox import ToolSection
 from galaxy.util import etree
 
 
diff --git a/test/unit/tools/test_toolbox_filters.py b/test/unit/tool_util/toolbox/test_toolbox_filters.py
similarity index 98%
rename from test/unit/tools/test_toolbox_filters.py
rename to test/unit/tool_util/toolbox/test_toolbox_filters.py
index f6212bf4337..55219d4f4fd 100644
--- a/test/unit/tools/test_toolbox_filters.py
+++ b/test/unit/tool_util/toolbox/test_toolbox_filters.py
@@ -1,4 +1,4 @@
-from galaxy.tools.toolbox.filters import FilterFactory
+from galaxy.tool_util.toolbox.filters import FilterFactory
 from galaxy.util.bunch import Bunch
 
 
diff --git a/test/unit/tools/test_watcher.py b/test/unit/tool_util/toolbox/test_watcher.py
similarity index 98%
rename from test/unit/tools/test_watcher.py
rename to test/unit/tool_util/toolbox/test_watcher.py
index be74e071bdf..827113c2217 100644
--- a/test/unit/tools/test_watcher.py
+++ b/test/unit/tool_util/toolbox/test_watcher.py
@@ -6,7 +6,7 @@ from shutil import rmtree
 
 import pytest
 
-from galaxy.tools.toolbox import watcher
+from galaxy.tool_util.toolbox import watcher
 from galaxy.util import bunch
 
 
diff --git a/test/unit/tools/conftest.py b/test/unit/tools/conftest.py
index ab8d702131e..4e6ff369dfb 100644
--- a/test/unit/tools/conftest.py
+++ b/test/unit/tools/conftest.py
@@ -4,8 +4,8 @@ import pytest
 
 from galaxy.model import tool_shed_install
 from galaxy.model.tool_shed_install import mapping
+from galaxy.tool_util.toolbox.base import ToolConfRepository
 from galaxy.tools.cache import ToolShedRepositoryCache
-from galaxy.tools.toolbox.base import ToolConfRepository
 from galaxy.util import bunch
 
 

From e6de9b74965edc1dace58becbe67b0980c47a2fe Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 22 Jul 2021 16:47:27 -0400
Subject: [PATCH 226/452] Allow bio.tools mappings for legacy tools.

Assigning these in the tools for new tool installs is great (https://github.com/galaxyproject/tools-iuc/pull/3411/files) but there will be thousands of old tools that are missing these and we should have a way to fill in this information as "missing info" bug fixes.
---
 lib/galaxy/tools/__init__.py           | 33 ++++++++++++++++++++++++--
 lib/galaxy/tools/biotools_mappings.tsv |  5 ++++
 lib/galaxy_test/api/test_tools.py      | 12 ++++++++++
 packages/app/MANIFEST.in               |  1 +
 4 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 lib/galaxy/tools/biotools_mappings.tsv

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index c6e71906154..08fd3949540 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -11,13 +11,14 @@ import tempfile
 import threading
 from datetime import datetime
 from pathlib import Path
-from typing import List, NamedTuple, Type, Union
+from typing import cast, Dict, List, NamedTuple, Optional, Tuple, Type, Union
 from urllib.parse import unquote_plus
 
 import packaging.version
 import webob.exc
 from lxml import etree
 from mako.template import Template
+from pkg_resources import resource_string
 from webob.compat import cgi_FieldStorage
 
 from galaxy import (
@@ -191,6 +192,9 @@ GALAXY_LIB_TOOLS_VERSIONED = {
     "winSplitter": packaging.version.parse("1.0.1"),
 }
 
+BIOTOOLS_MAPPING_CONTENT = resource_string(__name__, 'biotools_mappings.tsv').decode("UTF-8")
+BIOTOOLS_MAPPING: Dict[str, str] = dict([cast(Tuple[str, str], tuple(x.split("\t"))) for x in BIOTOOLS_MAPPING_CONTENT.splitlines() if not x.startswith("#")])
+
 
 class safe_update(NamedTuple):
     min_version: Union[packaging.version.LegacyVersion, packaging.version.Version]
@@ -960,7 +964,13 @@ class Tool(Dictifiable):
         self.containers = containers
 
         self.citations = self._parse_citations(tool_source)
-        self.xrefs = tool_source.parse_xrefs()
+        xrefs = tool_source.parse_xrefs()
+        has_biotools_reference = any(x["reftype"] == "bio.tools" for x in xrefs)
+        if not has_biotools_reference:
+            legacy_biotools_ref = self.legacy_biotools_external_reference
+            if legacy_biotools_ref is not None:
+                xrefs.append({"value": legacy_biotools_ref, "reftype": "bio.tools"})
+        self.xrefs = xrefs
 
         self.__parse_trackster_conf(tool_source)
         # Record macro paths so we can reload a tool if any of its macro has changes
@@ -1398,6 +1408,25 @@ class Tool(Dictifiable):
                 self.app, self.tool_shed, self.repository_owner, self.repository_name
             )
 
+    @property
+    def legacy_biotools_external_reference(self) -> Optional[str]:
+        """Return a bio.tools ID if any of tool's IDs are BIOTOOLS_MAPPING."""
+        for tool_id in self.all_ids:
+            if tool_id in BIOTOOLS_MAPPING:
+                return BIOTOOLS_MAPPING[tool_id]
+        return None
+
+    @property
+    def biotools_reference(self) -> Optional[str]:
+        """Return a bio.tools ID if external reference to it is found.
+
+        If multiple bio.tools references are found, return just the first one.
+        """
+        for xref in self.xrefs:
+            if xref["reftype"] == "bio.tools":
+                return xref["value"]
+        return None
+
     @property
     def help(self):
         if self.__help is HELP_UNINITIALIZED:
diff --git a/lib/galaxy/tools/biotools_mappings.tsv b/lib/galaxy/tools/biotools_mappings.tsv
new file mode 100644
index 00000000000..9a427fdd0c1
--- /dev/null
+++ b/lib/galaxy/tools/biotools_mappings.tsv
@@ -0,0 +1,5 @@
+# tool-id <tab> bio.tools-id
+abyss-pe	abyss
+allegro	allegro
+# testing tool mapping for test case
+cheetah_problem_unbound_var_input	bwa
\ No newline at end of file
diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index d5b316db16f..4d93a920943 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -216,6 +216,18 @@ class ToolsTestCase(ApiTestCase, TestsTools):
             assert galaxy_url.startswith("http")
             assert galaxy_url.endswith("tool_runner?tool_id=ratmine")
 
+    @skip_without_tool("cheetah_problem_unbound_var_input")
+    def test_legacy_biotools_xref_injection(self):
+        url = self._api_url("tools/cheetah_problem_unbound_var_input")
+        get_response = get(url)
+        get_response.raise_for_status()
+        get_json = get_response.json()
+        assert "xrefs" in get_json
+        assert len(get_json["xrefs"]) == 1
+        xref = get_json["xrefs"][0]
+        assert xref["reftype"] == "bio.tools"
+        assert xref["value"] == "bwa"
+
     @skip_without_tool("test_data_source")
     def test_data_source_ok_request(self):
         with self.dataset_populator.test_history() as history_id:
diff --git a/packages/app/MANIFEST.in b/packages/app/MANIFEST.in
index cd53553319a..35fc6a3a551 100644
--- a/packages/app/MANIFEST.in
+++ b/packages/app/MANIFEST.in
@@ -2,3 +2,4 @@ include *.rst *.txt LICENSE
 include galaxy/*.yml
 include galaxy/config/sample/*.sample
 include galaxy/jobs/runners/util/job_script/*.sh
+include galaxy/tools/*tsv

From 7469d674f98e2d9b8f8a4d656399f799f1a87352 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 27 Jul 2021 00:15:37 +0100
Subject: [PATCH 227/452] Fix unit tests after toolbox code move

---
 lib/galaxy/tool_util/toolbox/filters/__init__.py             | 2 +-
 .../{tools => tool_util/toolbox}/filter_modules/__init__.py  | 0
 .../{tools => tool_util/toolbox}/filter_modules/filtermod.py | 0
 test/unit/tool_util/toolbox/test_toolbox_filters.py          | 5 +++--
 test/unit/tools/test_toolbox.py                              | 2 +-
 5 files changed, 5 insertions(+), 4 deletions(-)
 rename test/unit/{tools => tool_util/toolbox}/filter_modules/__init__.py (100%)
 rename test/unit/{tools => tool_util/toolbox}/filter_modules/filtermod.py (100%)

diff --git a/lib/galaxy/tool_util/toolbox/filters/__init__.py b/lib/galaxy/tool_util/toolbox/filters/__init__.py
index 3bdefbc98d6..922323b5733 100644
--- a/lib/galaxy/tool_util/toolbox/filters/__init__.py
+++ b/lib/galaxy/tool_util/toolbox/filters/__init__.py
@@ -22,7 +22,7 @@ class FilterFactory:
         self.default_filters = dict(tool=[_not_hidden, _handle_authorization], section=[], label=[])
         # Add dynamic filters to these default filters.
         config = toolbox.app.config
-        self.__base_modules = listify(getattr(config, "toolbox_filter_base_modules", "galaxy.tool_util.filters"))
+        self.__base_modules = listify(getattr(config, "toolbox_filter_base_modules", "galaxy.tool_util.toolbox.filters"))
         self.__init_filters("tool", getattr(config, "tool_filters", ""), self.default_filters)
         self.__init_filters("section", getattr(config, "tool_section_filters", ""), self.default_filters)
         self.__init_filters("label", getattr(config, "tool_label_filters", ""), self.default_filters)
diff --git a/test/unit/tools/filter_modules/__init__.py b/test/unit/tool_util/toolbox/filter_modules/__init__.py
similarity index 100%
rename from test/unit/tools/filter_modules/__init__.py
rename to test/unit/tool_util/toolbox/filter_modules/__init__.py
diff --git a/test/unit/tools/filter_modules/filtermod.py b/test/unit/tool_util/toolbox/filter_modules/filtermod.py
similarity index 100%
rename from test/unit/tools/filter_modules/filtermod.py
rename to test/unit/tool_util/toolbox/filter_modules/filtermod.py
diff --git a/test/unit/tool_util/toolbox/test_toolbox_filters.py b/test/unit/tool_util/toolbox/test_toolbox_filters.py
index 55219d4f4fd..a803cd2fe68 100644
--- a/test/unit/tool_util/toolbox/test_toolbox_filters.py
+++ b/test/unit/tool_util/toolbox/test_toolbox_filters.py
@@ -32,7 +32,7 @@ def test_custom_filters():
     tool_filters = filters["tool"]
     # TODO: the fact that -1 is the custom filter is an implementation
     # detail that should not be tested here.
-    assert tool_filters[-1].__doc__ == "Test Filter Tool"
+    assert tool_filters[-1].__doc__ == "Test Filter Tool", tool_filters
 
     section_filters = filters["section"]
     assert section_filters[0].__doc__ == "Test Filter Section"
@@ -50,7 +50,8 @@ def filter_factory(config_dict=None):
             tool_label_filters=["filtermod:filter_label_1", "filtermod:filter_label_2"],
         )
     config = Bunch(**config_dict)
-    config.toolbox_filter_base_modules = "galaxy.tools.filters,unit.tools.filter_modules"
+    parent_module_name = '.'.join(__name__.split('.')[:-1])
+    config.toolbox_filter_base_modules = f"galaxy.tool_util.toolbox.filters,{parent_module_name}.filter_modules"
     app = Bunch(config=config)
     toolbox = Bunch(app=app)
     return FilterFactory(toolbox)
diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py
index 9004cfde49a..13ef949bd0f 100644
--- a/test/unit/tools/test_toolbox.py
+++ b/test/unit/tools/test_toolbox.py
@@ -15,7 +15,7 @@ from galaxy.model import tool_shed_install
 from galaxy.model.tool_shed_install import mapping
 from galaxy.tools import ToolBox
 from galaxy.tools.cache import ToolCache
-from .test_toolbox_filters import mock_trans
+from ..tool_util.toolbox.test_toolbox_filters import mock_trans
 from ..tools_support import UsesApp, UsesTools
 from ..unittest_utils.sample_data import SIMPLE_MACRO, SIMPLE_TOOL_WITH_MACRO
 

From 544903e7dde86e81ec4c3bc46d7902c2583c9ea7 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 28 Jul 2021 13:55:58 +0200
Subject: [PATCH 228/452] Allow lists and comma separated strings when
 filtering history item types

---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index eb6ded8120d..0f6dfdeff2c 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -141,12 +141,12 @@ class HistoryContentsIndexLegacyParams(HistoryContentsIndexLegacyParamsBase):
             "specific datasets will be provided. Also, setting this value will set `dataset_details` to `all`."
         ),
     )
-    types: Optional[str] = Field(
+    types: Optional[Union[List[HistoryContentType], str]] = Field(
         default=None,
         alias="type",  # Legacy alias
         title="Types",
         description=(
-            "A comma separated list of kinds of contents to return "
+            "A list or comma separated list of kinds of contents to return "
             "(currently just `dataset` and `dataset_collection` are available)."
         ),
     )

From 9ea3b87569b92d4b4bff52cbc7ab45a386fca928 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 28 Jul 2021 13:57:22 +0200
Subject: [PATCH 229/452] Fix model population only working on field alias

---
 lib/galaxy/schema/schema.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index b0c6f4ae6ea..8c31eb11ada 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -166,6 +166,7 @@ class Model(BaseModel):
     """Base model definition with common configuration used by all derived models."""
     class Config:
         use_enum_values = True  # when using .dict()
+        allow_population_by_field_name = True
 
 
 class UserModel(Model):
@@ -2089,11 +2090,7 @@ class DatasetPermissionAction(str, Enum):
     remove_restrictions = "remove_restrictions"
 
 
-class LibraryPermissionsPayloadBase(BaseModel):
-    class Config:
-        use_enum_values = True  # When using .dict()
-        allow_population_by_alias = True
-
+class LibraryPermissionsPayloadBase(Model):
     add_ids: Optional[RoleIdList] = Field(
         [],
         alias="add_ids[]",
@@ -2281,11 +2278,7 @@ class DatasetAssociationRoles(Model):
     )
 
 
-class UpdateDatasetPermissionsPayload(BaseModel):
-    class Config:
-        use_enum_values = True  # When using .dict()
-        allow_population_by_alias = True
-
+class UpdateDatasetPermissionsPayload(Model):
     action: Optional[DatasetPermissionAction] = Field(
         ...,
         title="Action",

From a73cf9e0d9bd42e9e28282fb355cff1344be878e Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 28 Jul 2021 14:24:23 +0200
Subject: [PATCH 230/452] Add API test for filtering history content by type

---
 lib/galaxy_test/api/test_history_contents.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 7f349dd7f74..831c2122585 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -621,3 +621,18 @@ class HistoryContentsApiTestCase(ApiTestCase):
                 if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['populated_state'] == 'ok'):
                     return
             raise Exception(f"History content update time query did not include populated_state update for dynamic nested collection {collection_id}")
+
+    def test_index_filter_by_type(self):
+        history_id = self.dataset_populator.new_history()
+        self.dataset_populator.new_dataset(history_id)
+        self.dataset_collection_populator.create_list_in_history(history_id=history_id)
+
+        contents_response = self._get(f"histories/{history_id}/contents").json()
+        num_items = len(contents_response)
+        expected_num_collections = 1
+        expected_num_datasets = num_items - expected_num_collections
+
+        contents_response = self._get(f"histories/{history_id}/contents?types=dataset").json()
+        assert len(contents_response) == expected_num_datasets
+        contents_response = self._get(f"histories/{history_id}/contents?types=dataset_collection").json()
+        assert len(contents_response) == expected_num_collections

From e5964432e840e97924380e77313340a4c6a9323d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 10:40:05 -0400
Subject: [PATCH 231/452] Add history provider to run workflow form, parse
 history id to tool step build request

---
 client/src/components/Tool/ToolForm.vue       |   2 +-
 .../Workflow/Run/WorkflowRunForm.vue          | 115 +++++++++---------
 .../Workflow/Run/WorkflowRunToolStep.vue      |   6 +-
 .../src/components/Workflow/Run/services.js   |   7 +-
 4 files changed, 71 insertions(+), 59 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index b6b36b3cacb..7990e82fadb 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -76,7 +76,7 @@
                             <ButtonSpinner
                                 id="execute"
                                 title="Execute"
-                                @onClick="onExecute(config, currentHistory.id)"
+                                @onClick="onExecute(config, currentHistory && currentHistory.id)"
                                 :wait="showExecuting"
                                 :tooltip="tooltip"
                             />
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 8d7f72de26c..644b4ae2417 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,69 +1,73 @@
 <template>
     <CurrentUser v-slot="{ user }">
-        <div>
-            <div class="h4 clearfix mb-3">
-                <b>Workflow: {{ model.name }}</b>
-                <ButtonSpinner
-                    id="run-workflow"
-                    class="float-right"
-                    title="Run Workflow"
-                    :wait="showExecuting"
-                    @onClick="onExecute"
-                />
-            </div>
-            <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
-                <template v-slot:body>
-                    <FormDisplay :inputs="wpInputs" @onChange="onWpInputs" />
-                </template>
-            </FormCard>
-            <FormCard title="History Options">
-                <template v-slot:body>
-                    <FormDisplay :inputs="historyInputs" @onChange="onHistoryInputs" />
-                </template>
-            </FormCard>
-            <FormCard v-if="reuseAllowed(user)" title="Job re-use Options">
-                <template v-slot:body>
-                    <FormElement
-                        title="Attempt to re-use jobs with identical parameters?"
-                        help="This may skip executing jobs that you have already run."
-                        type="boolean"
-                        v-model="useCachedJobs"
+        <UserHistories v-if="user" :user="user" v-slot="{ currentHistory }">
+            <div>
+                <div class="h4 clearfix mb-3">
+                    <b>Workflow: {{ model.name }}</b>
+                    <ButtonSpinner
+                        id="run-workflow"
+                        class="float-right"
+                        title="Run Workflow"
+                        :wait="showExecuting"
+                        @onClick="onExecute"
                     />
-                </template>
-            </FormCard>
-            <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
-                <template v-slot:body>
-                    <FormDisplay :inputs="resourceInputs" @onChange="onResourceInputs" />
-                </template>
-            </FormCard>
-            <div v-for="step in model.steps" :key="step.index">
-                <WorkflowRunToolStep
-                    v-if="step.step_type == 'tool'"
-                    :model="step"
-                    :step-data="stepData"
-                    :validation-scroll-to="getValidationScrollTo(step.index)"
-                    :wp-data="wpData"
-                    @onChange="onToolStepInputs"
-                    @onValidation="onValidation"
-                />
-                <WorkflowRunDefaultStep
-                    v-else
-                    :model="step"
-                    :validation-scroll-to="getValidationScrollTo(step.index)"
-                    @onChange="onDefaultStepInputs"
-                    @onValidation="onValidation"
-                />
+                </div>
+                <FormCard v-if="wpInputsAvailable" title="Workflow Parameters">
+                    <template v-slot:body>
+                        <FormDisplay :inputs="wpInputs" @onChange="onWpInputs" />
+                    </template>
+                </FormCard>
+                <FormCard title="History Options">
+                    <template v-slot:body>
+                        <FormDisplay :inputs="historyInputs" @onChange="onHistoryInputs" />
+                    </template>
+                </FormCard>
+                <FormCard v-if="reuseAllowed(user)" title="Job re-use Options">
+                    <template v-slot:body>
+                        <FormElement
+                            title="Attempt to re-use jobs with identical parameters?"
+                            help="This may skip executing jobs that you have already run."
+                            type="boolean"
+                            v-model="useCachedJobs"
+                        />
+                    </template>
+                </FormCard>
+                <FormCard v-if="resourceInputsAvailable" title="Workflow Resource Options">
+                    <template v-slot:body>
+                        <FormDisplay :inputs="resourceInputs" @onChange="onResourceInputs" />
+                    </template>
+                </FormCard>
+                <div v-for="step in model.steps" :key="step.index">
+                    <WorkflowRunToolStep
+                        v-if="step.step_type == 'tool'"
+                        :model="step"
+                        :step-data="stepData"
+                        :validation-scroll-to="getValidationScrollTo(step.index)"
+                        :wp-data="wpData"
+                        :history-id="currentHistory && currentHistory.id"
+                        @onChange="onToolStepInputs"
+                        @onValidation="onValidation"
+                    />
+                    <WorkflowRunDefaultStep
+                        v-else
+                        :model="step"
+                        :validation-scroll-to="getValidationScrollTo(step.index)"
+                        @onChange="onDefaultStepInputs"
+                        @onValidation="onValidation"
+                    />
+                </div>
             </div>
-        </div>
+        </UserHistories>
     </CurrentUser>
 </template>
 
 <script>
+import ButtonSpinner from "components/Common/ButtonSpinner";
 import CurrentUser from "components/providers/CurrentUser";
 import FormDisplay from "components/Form/FormDisplay";
 import FormCard from "components/Form/FormCard";
 import FormElement from "components/Form/FormElement";
-import ButtonSpinner from "components/Common/ButtonSpinner";
+import UserHistories from "components/History/providers/UserHistories";
 import WorkflowRunDefaultStep from "./WorkflowRunDefaultStep";
 import WorkflowRunToolStep from "./WorkflowRunToolStep";
 import { invokeWorkflow } from "./services";
@@ -71,11 +75,12 @@ import { allowCachedJobs } from "components/Tool/utilities";
 
 export default {
     components: {
+        ButtonSpinner,
         CurrentUser,
         FormDisplay,
         FormCard,
         FormElement,
-        ButtonSpinner,
+        UserHistories,
         WorkflowRunDefaultStep,
         WorkflowRunToolStep,
     },
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 5dc2fc97d56..c1f616f070d 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -50,6 +50,10 @@ export default {
             type: Array,
             required: true,
         },
+        historyId: {
+            type: String,
+            default: null,
+        },
     },
     data() {
         return {
@@ -125,7 +129,7 @@ export default {
             });
         },
         onChange(data) {
-            getTool(this.model.id, this.model.version, data).then(
+            getTool(this.model.id, this.model.version, data, this.historyId).then(
                 (formConfig) => {
                     this.formConfig = formConfig;
                 },
diff --git a/client/src/components/Workflow/Run/services.js b/client/src/components/Workflow/Run/services.js
index 5ae80968a0a..862269ce058 100644
--- a/client/src/components/Workflow/Run/services.js
+++ b/client/src/components/Workflow/Run/services.js
@@ -35,13 +35,16 @@ export async function invokeWorkflow(workflowId, invocationData) {
 /**
  * Request tool step data.
  *
- * @param {String} workflowId - (Stored?) Workflow ID to fetch data for.
+ * @param {String} toolId - Tool ID to fetch data for.
+ * @param {String} toolVersion - Corresponding tool version.
+ * @param {Object} toolInputs - Current tool state.
  */
-export async function getTool(toolId, toolVersion, toolInputs) {
+export async function getTool(toolId, toolVersion, toolInputs, historyId) {
     const requestData = {
         tool_id: toolId,
         tool_version: toolVersion,
         inputs: JSON.parse(JSON.stringify(toolInputs)),
+        history_id: historyId,
     };
     try {
         const { data } = await axios.post(`${getAppRoot()}api/tools/${toolId}/build`, requestData);

From 1be22d6bbcea469fc855857951b56530374d37a4 Mon Sep 17 00:00:00 2001
From: Aysam Guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 10:42:36 -0400
Subject: [PATCH 232/452] Update client/src/components/Tool/ToolForm.vue

Co-authored-by: Marius van den Beek <m.vandenbeek@gmail.com>
---
 client/src/components/Tool/ToolForm.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 7990e82fadb..675b93a156f 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -76,7 +76,7 @@
                             <ButtonSpinner
                                 id="execute"
                                 title="Execute"
-                                @onClick="onExecute(config, currentHistory && currentHistory.id)"
+                                @onClick="onExecute(config, currentHistory?.id)"
                                 :wait="showExecuting"
                                 :tooltip="tooltip"
                             />

From 8fb22e23937c3b44dd485c34207ac83dd38577b4 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 10:44:08 -0400
Subject: [PATCH 233/452] Add doc line to get tool helper

---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 2 +-
 client/src/components/Workflow/Run/services.js         | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 644b4ae2417..59c494bb0bc 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -44,7 +44,7 @@
                         :step-data="stepData"
                         :validation-scroll-to="getValidationScrollTo(step.index)"
                         :wp-data="wpData"
-                        :history-id="currentHistory && currentHistory.id"
+                        :history-id="currentHistory?.id"
                         @onChange="onToolStepInputs"
                         @onValidation="onValidation"
                     />
diff --git a/client/src/components/Workflow/Run/services.js b/client/src/components/Workflow/Run/services.js
index 862269ce058..b090b20a3fb 100644
--- a/client/src/components/Workflow/Run/services.js
+++ b/client/src/components/Workflow/Run/services.js
@@ -38,6 +38,7 @@ export async function invokeWorkflow(workflowId, invocationData) {
  * @param {String} toolId - Tool ID to fetch data for.
  * @param {String} toolVersion - Corresponding tool version.
  * @param {Object} toolInputs - Current tool state.
+ * @param {Object} historyId - History ID to populate data selection fields.
  */
 export async function getTool(toolId, toolVersion, toolInputs, historyId) {
     const requestData = {

From f6d8e4b9ebf92e960fd31f1c8633e88e8294b47b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 10:49:51 -0400
Subject: [PATCH 234/452] Wait until history provider resolves current history
 in tool and workflow run forms

---
 client/src/components/Tool/ToolForm.vue                | 4 ++--
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 675b93a156f..04bd7fe7209 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -2,7 +2,7 @@
     <ConfigProvider v-slot="{ config }">
         <CurrentUser v-slot="{ user }">
             <UserHistories v-if="user" :user="user" v-slot="{ currentHistory }">
-                <div>
+                <div v-if="currentHistory">
                     <LoadingSpan v-if="showLoading" message="Loading Tool" />
                     <div v-if="showEntryPoints">
                         <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
@@ -76,7 +76,7 @@
                             <ButtonSpinner
                                 id="execute"
                                 title="Execute"
-                                @onClick="onExecute(config, currentHistory?.id)"
+                                @onClick="onExecute(config, currentHistory.id)"
                                 :wait="showExecuting"
                                 :tooltip="tooltip"
                             />
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 59c494bb0bc..5225b4d88cf 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,7 +1,7 @@
 <template>
     <CurrentUser v-slot="{ user }">
         <UserHistories v-if="user" :user="user" v-slot="{ currentHistory }">
-            <div>
+            <div v-if="currentHistory">
                 <div class="h4 clearfix mb-3">
                     <b>Workflow: {{ model.name }}</b>
                     <ButtonSpinner
@@ -44,7 +44,7 @@
                         :step-data="stepData"
                         :validation-scroll-to="getValidationScrollTo(step.index)"
                         :wp-data="wpData"
-                        :history-id="currentHistory?.id"
+                        :history-id="currentHistory.id"
                         @onChange="onToolStepInputs"
                         @onValidation="onValidation"
                     />

From 81b0e5371cf0b0bfab4a570b10bd7b389682fbf8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 11:04:50 -0400
Subject: [PATCH 235/452] Use currentHistoryId slot in tool and run workflow
 forms

---
 client/src/components/Tool/ToolForm.vue                | 6 +++---
 client/src/components/Workflow/Run/WorkflowRunForm.vue | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 04bd7fe7209..33aadafb650 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -1,8 +1,8 @@
 <template>
     <ConfigProvider v-slot="{ config }">
         <CurrentUser v-slot="{ user }">
-            <UserHistories v-if="user" :user="user" v-slot="{ currentHistory }">
-                <div v-if="currentHistory">
+            <UserHistories v-if="user" :user="user" v-slot="{ currentHistoryId }">
+                <div v-if="currentHistoryId">
                     <LoadingSpan v-if="showLoading" message="Loading Tool" />
                     <div v-if="showEntryPoints">
                         <ToolEntryPoints v-for="job in entryPoints" :job-id="job.id" :key="job.id" />
@@ -76,7 +76,7 @@
                             <ButtonSpinner
                                 id="execute"
                                 title="Execute"
-                                @onClick="onExecute(config, currentHistory.id)"
+                                @onClick="onExecute(config, currentHistoryId)"
                                 :wait="showExecuting"
                                 :tooltip="tooltip"
                             />
diff --git a/client/src/components/Workflow/Run/WorkflowRunForm.vue b/client/src/components/Workflow/Run/WorkflowRunForm.vue
index 5225b4d88cf..212f9ec54f1 100644
--- a/client/src/components/Workflow/Run/WorkflowRunForm.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunForm.vue
@@ -1,7 +1,7 @@
 <template>
     <CurrentUser v-slot="{ user }">
-        <UserHistories v-if="user" :user="user" v-slot="{ currentHistory }">
-            <div v-if="currentHistory">
+        <UserHistories v-if="user" :user="user" v-slot="{ currentHistoryId }">
+            <div v-if="currentHistoryId">
                 <div class="h4 clearfix mb-3">
                     <b>Workflow: {{ model.name }}</b>
                     <ButtonSpinner
@@ -44,7 +44,7 @@
                         :step-data="stepData"
                         :validation-scroll-to="getValidationScrollTo(step.index)"
                         :wp-data="wpData"
-                        :history-id="currentHistory.id"
+                        :history-id="currentHistoryId"
                         @onChange="onToolStepInputs"
                         @onValidation="onValidation"
                     />

From a9ced37c3dc82794811beff18418cfca8653c090 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 11:10:49 -0400
Subject: [PATCH 236/452] Modify history provider to parse history identifier
 without accessing the history model

---
 .../components/History/providers/UserHistories/UserHistories.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/History/providers/UserHistories/UserHistories.js b/client/src/components/History/providers/UserHistories/UserHistories.js
index 56b3f761ebe..e9547e7db02 100644
--- a/client/src/components/History/providers/UserHistories/UserHistories.js
+++ b/client/src/components/History/providers/UserHistories/UserHistories.js
@@ -65,7 +65,7 @@ export default {
 
             // currently selected history object, should be a full object not just a summary
             currentHistory: this.currentHistoryModel,
-            currentHistoryId: this.currentHistoryModel?.id || null,
+            currentHistoryId: this.currentHistoryId,
 
             handlers: {
                 // Updates the history in the store without a trip to the server, in the event that a

From 1f7604c84bc77b67d48cf0143883e274dec7e0a1 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Wed, 28 Jul 2021 18:51:11 +0200
Subject: [PATCH 237/452] reword history import button text

Import from file seems outdated.
---
 lib/galaxy/selenium/navigation.yml               | 2 +-
 lib/galaxy/webapps/galaxy/controllers/history.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 8c7b6f29617..86b5dcffe29 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -290,7 +290,7 @@ collection_builders:
 
 histories:
   labels:
-    import_button: 'Import from file'
+    import_button: 'Import history'
 
   sharing:
     selectors:
diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py
index b613d8cfdf2..610786a63c8 100644
--- a/lib/galaxy/webapps/galaxy/controllers/history.py
+++ b/lib/galaxy/webapps/galaxy/controllers/history.py
@@ -118,7 +118,7 @@ class HistoryListGrid(grids.Grid):
             key="free-text-search", visible=False, filterable="standard")
     )
     global_actions = [
-        grids.GridAction("Import from file", dict(controller="", action="histories/import"))
+        grids.GridAction("Import history", dict(controller="", action="histories/import"))
     ]
     operations = [
         grids.GridOperation("Switch", allow_multiple=False, condition=(lambda item: not item.deleted), async_compatible=True),

From 02cb253f7a9bb5faf85702a754ca01034d93ef3e Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 29 Jul 2021 12:59:28 +0200
Subject: [PATCH 238/452] Fix order of dataset instances

---
 lib/galaxy/model/__init__.py          |  4 ++-
 test/unit/data/test_galaxy_mapping.py | 37 +++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 7a818541c37..14e50ddfe63 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -4302,17 +4302,19 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
 
             depth_collection_type = self.collection_type
             select_from = dc.outerjoin(de, de.c.dataset_collection_id == dc.c.id)
+            order_by_columns = [de.c.element_index]
 
             while ":" in depth_collection_type:
                 child_collection = alias(DatasetCollection.table)
                 child_collection_element = alias(DatasetCollectionElement.table)
                 select_from = select_from.outerjoin(child_collection, child_collection.c.id == de.c.child_collection_id)
                 select_from = select_from.outerjoin(child_collection_element, child_collection_element.c.dataset_collection_id == child_collection.c.id)
+                order_by_columns.append(child_collection_element.c.element_index)
 
                 de = child_collection_element
                 depth_collection_type = depth_collection_type.split(":", 1)[1]
             select_from = select_from.outerjoin(hda, hda.c.id == de.c.hda_id)
-            select_stmt = select([hda]).select_from(select_from).where(dc.c.id == self.id).distinct()
+            select_stmt = select([hda]).select_from(select_from).where(dc.c.id == self.id).distinct(*order_by_columns).order_by(*order_by_columns)
             return db_session.query(HistoryDatasetAssociation).select_entity_from(select_stmt).filter(HistoryDatasetAssociation.id.isnot(None)).all()
         else:
             # Sessionless context
diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py
index 12788b63ebf..ca7514de342 100644
--- a/test/unit/data/test_galaxy_mapping.py
+++ b/test/unit/data/test_galaxy_mapping.py
@@ -1,5 +1,6 @@
 import collections
 import os
+import random
 import unittest
 import uuid
 
@@ -288,6 +289,42 @@ class MappingTests(BaseModelTestCase):
         for i in range(elements):
             assert c1[i] == dces[i]
 
+    def test_dataset_instance_order(self):
+        model = self.model
+        u = model.User(email="mary@example.com", password="password")
+        h1 = model.History(name="History 1", user=u)
+        elements = []
+        list_pair = model.DatasetCollection(collection_type="list:paired")
+        for i in range(20):
+            pair = model.DatasetCollection(collection_type="pair")
+            forward = model.HistoryDatasetAssociation(extension="txt", history=h1, name=f"forward_{i}", create_dataset=True, sa_session=model.session)
+            reverse = model.HistoryDatasetAssociation(extension="bam", history=h1, name=f"reverse_{i}", create_dataset=True, sa_session=model.session)
+            dce1 = model.DatasetCollectionElement(collection=pair, element=forward, element_identifier=f"forward_{i}", element_index=1)
+            dce2 = model.DatasetCollectionElement(collection=pair, element=reverse, element_identifier=f"reverse_{i}", element_index=2)
+            to_persist = [(forward, reverse), (dce1, dce2)]
+            self.persist(pair)
+            for item in to_persist:
+                if i % 2:
+                    self.persist(item[0])
+                    self.persist(item[1])
+                else:
+                    self.persist(item[1])
+                    self.persist(item[0])
+            elements.append(model.DatasetCollectionElement(collection=list_pair, element=pair, element_index=i, element_identifier=str(i)))
+        self.persist(list_pair)
+        random.shuffle(elements)
+        for item in elements:
+            self.persist(item)
+        forward = []
+        reverse = []
+        for i, dataset_instance in enumerate(list_pair.dataset_instances):
+            if i % 2:
+                reverse.append(dataset_instance)
+            else:
+                forward.append(dataset_instance)
+        assert all(d.name == f"forward_{i}" for i, d in enumerate(forward))
+        assert all(d.name == f"reverse_{i}" for i, d in enumerate(reverse))
+
     def test_collections_in_histories(self):
         model = self.model
 

From fee4c39371ee8ae3fc0ddafc6f2876484fb5556b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 29 Jul 2021 20:03:38 +0200
Subject: [PATCH 239/452] Fix repository_update test

---
 test/integration/test_repository_operations.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/integration/test_repository_operations.py b/test/integration/test_repository_operations.py
index de17c8a9b56..d3a5e45bad4 100644
--- a/test/integration/test_repository_operations.py
+++ b/test/integration/test_repository_operations.py
@@ -59,7 +59,7 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
 
     def test_repository_update(self):
         response = self._install_repository(revision=REVISION_4, version="0.0.3")[0]
-        assert response['ctx_rev'] == '4'
+        assert int(response['ctx_rev']) >= 4
         repo_response = self._get("/api/tool_shed_repositories/%s" % response['id']).json()
         assert repo_response['tool_shed_status']['revision_update'] == 'False'  # that should really be a boolean
         # now checkout revision 3 and attempt an update
@@ -108,7 +108,8 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
         response = self._install_repository(revision=REVISION_4, version="0.0.3", verify_tool_absent=False)[0]
         assert response['changeset_revision'] == REVISION_4
         assert response['installed_changeset_revision'] == REVISION_3
-        assert response['ctx_rev'] == '4'
+        # should be 4 or newer
+        assert int(response['ctx_rev']) >= 4
 
     def _uninstall_repository(self):
         tool = self._get('/api/tools/collection_column_join').json()

From 8aea13ce2e61fc990149190a2b97c9a9e9385209 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 30 Jul 2021 11:18:59 +0200
Subject: [PATCH 240/452] Fix test_repository_update

for real this time.
---
 test/integration/test_repository_operations.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/test/integration/test_repository_operations.py b/test/integration/test_repository_operations.py
index d3a5e45bad4..5ba8c2ad58d 100644
--- a/test/integration/test_repository_operations.py
+++ b/test/integration/test_repository_operations.py
@@ -60,11 +60,12 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
     def test_repository_update(self):
         response = self._install_repository(revision=REVISION_4, version="0.0.3")[0]
         assert int(response['ctx_rev']) >= 4
+        latest_revision = response['changeset_revision']
         repo_response = self._get("/api/tool_shed_repositories/%s" % response['id']).json()
         assert repo_response['tool_shed_status']['revision_update'] == 'False'  # that should really be a boolean
         # now checkout revision 3 and attempt an update
         path_components = [
-            self._app.config.shed_tools_dir,
+            os.path.dirname(self._app.config.shed_tool_config_file),
             'toolshed.g2.bx.psu.edu',
             'repos',
             REPO.owner,
@@ -73,25 +74,25 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
             REPO.name
         ]
         revision_3_path = os.path.join(*path_components[:6])
-        revision_4_path = os.path.join(*path_components[:5] + [REVISION_4])
+        latest_revision_path = os.path.join(*path_components[:5] + [latest_revision])
         repository_path = os.path.join(*path_components)
         # Move repo to location expected before minor update
-        os.rename(revision_4_path, revision_3_path)
+        os.rename(latest_revision_path, revision_3_path)
         # Checkout revision 3
         hg_util.update_repository(repository_path, ctx_rev='3')
         # change repo to revision 3 in database
         model = self._app.install_model
         tsr = model.context.query(model.ToolShedRepository).first()
         assert tsr.name == REPO.name
-        assert tsr.changeset_revision == REVISION_4
-        assert tsr.ctx_rev == '4'
+        assert tsr.changeset_revision == latest_revision
+        assert int(tsr.ctx_rev) >= 4
         tsr.ctx_rev = '3'
         tsr.installed_changeset_revision = REVISION_3
         tsr.changeset_revision = REVISION_3
         model.context.flush()
         # update shed_tool_conf.xml to look like revision 3 was the installed_changeset_revision
         with open(self._app.config.shed_tool_config_file) as shed_config:
-            shed_text = shed_config.read().replace(REVISION_4, REVISION_3)
+            shed_text = shed_config.read().replace(latest_revision, REVISION_3)
         with open(self._app.config.shed_tool_config_file, 'w') as shed_config:
             shed_config.write(shed_text)
         self._get(
@@ -106,7 +107,7 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
         assert repo_response['ctx_rev'] == '3'
         # now install revision 4 (a.k.a a minor update)
         response = self._install_repository(revision=REVISION_4, version="0.0.3", verify_tool_absent=False)[0]
-        assert response['changeset_revision'] == REVISION_4
+        assert response['changeset_revision'] == latest_revision
         assert response['installed_changeset_revision'] == REVISION_3
         # should be 4 or newer
         assert int(response['ctx_rev']) >= 4

From b8cd3381f8e69ccce4ac88f76e03eea8e9c300af Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 30 Jul 2021 10:46:59 +0200
Subject: [PATCH 241/452] Add webdriver setup

How did this work before ? Preloaded webdriver ?
---
 .github/workflows/integration_selenium.yaml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/integration_selenium.yaml b/.github/workflows/integration_selenium.yaml
index 4d0e3b1d706..2bc331c14f6 100644
--- a/.github/workflows/integration_selenium.yaml
+++ b/.github/workflows/integration_selenium.yaml
@@ -2,8 +2,6 @@ name: Integration Selenium
 on: [push, pull_request]
 env:
   GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
-  GALAXY_TEST_SELENIUM_REMOTE: '1'
-  GALAXY_TEST_SELENIUM_REMOTE_PORT: "4444"
   GALAXY_SKIP_CLIENT_BUILD: '0'
   GALAXY_TEST_SELENIUM_RETRIES: 1
 jobs:
@@ -22,10 +20,6 @@ jobs:
           POSTGRES_DB: postgres
         ports:
           - 5432:5432
-      selenium:
-        image: selenium/standalone-chrome:3.141.59
-        ports:
-          - 4444:4444
     steps:
       - name: Prune unused docker image, volumes and containers
         run: docker system prune -a -f
@@ -41,6 +35,7 @@ jobs:
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - uses: nanasess/setup-chromedriver@master
       - name: Run tests
         run: './run_tests.sh -integration test/integration_selenium'
         working-directory: 'galaxy root'

From cb8d157f24380ba59652b6321d157b9d5f71b8b2 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 30 Jul 2021 15:50:13 +0200
Subject: [PATCH 242/452] Fix repo test setUp / tearDown

---
 test/integration/test_repository_operations.py | 4 +++-
 test/integration/uses_shed.py                  | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/integration/test_repository_operations.py b/test/integration/test_repository_operations.py
index 5ba8c2ad58d..e27193b4395 100644
--- a/test/integration/test_repository_operations.py
+++ b/test/integration/test_repository_operations.py
@@ -26,10 +26,12 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
 
     def setUp(self):
         super().setUp()
+        self.setup_shed_config()
         self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
 
     def tearDown(self):
         self.reset_shed_tools()
+        return super().tearDown()
 
     def test_repository_installation(self):
         """
@@ -105,7 +107,7 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
         assert repo_response['tool_shed_status']['revision_update'] == 'True'
         assert repo_response['changeset_revision'] == REVISION_3
         assert repo_response['ctx_rev'] == '3'
-        # now install revision 4 (a.k.a a minor update)
+        # now install revision 4 (or newer) (a.k.a a minor update)
         response = self._install_repository(revision=REVISION_4, version="0.0.3", verify_tool_absent=False)[0]
         assert response['changeset_revision'] == latest_revision
         assert response['installed_changeset_revision'] == REVISION_3
diff --git a/test/integration/uses_shed.py b/test/integration/uses_shed.py
index f43550c3e71..8125a0fb59f 100644
--- a/test/integration/uses_shed.py
+++ b/test/integration/uses_shed.py
@@ -59,13 +59,17 @@ class UsesShed:
         config["conda_auto_install"] = True
         config["conda_prefix"] = os.environ.get('GALAXY_TEST_CONDA_PREFIX') or os.path.join(cls.conda_tmp_prefix, 'conda')
 
-    def reset_shed_tools(self):
+    def setup_shed_config(self):
         shutil.rmtree(self._app.config.shed_tools_dir, ignore_errors=True)
         os.makedirs(self._app.config.shed_tools_dir)
+        self._app.config.shed_tools_dir = self.shed_tools_dir
         with open(self._app.config.shed_tool_config_file, "w") as tool_conf_file:
             tool_conf_file.write(SHED_TOOL_CONF.substitute(shed_tools_path=self._app.config.shed_tools_dir))
         # deleting the containing folder doesn't trigger a toolbox reload, so signal it now and wait until it's done
         self._app.queue_worker.send_control_task('reload_toolbox', get_response=True)
+
+    def reset_shed_tools(self):
+        self.setup_shed_config()
         model = self._app.install_model
         models_to_delete = [
             model.RepositoryRepositoryDependencyAssociation,

From fb6522121cbb7d1dcf956e77f59e95d47c9d61b3 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Sat, 31 Jul 2021 18:50:29 +0530
Subject: [PATCH 243/452] [21.05] Make authnz end session endpoint optional to
 support various providers

---
 lib/galaxy/authnz/custos_authnz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py
index 73c5aa1d2c9..b9ed90c4aa8 100644
--- a/lib/galaxy/authnz/custos_authnz.py
+++ b/lib/galaxy/authnz/custos_authnz.py
@@ -310,7 +310,7 @@ class CustosAuthnz(IdentityProvider):
         self.config['authorization_endpoint'] = well_known_oidc_config['authorization_endpoint']
         self.config['token_endpoint'] = well_known_oidc_config['token_endpoint']
         self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint']
-        self.config['end_session_endpoint'] = well_known_oidc_config['end_session_endpoint']
+        self.config['end_session_endpoint'] = well_known_oidc_config.get('end_session_endpoint')
 
     def _get_verify_param(self):
         """Return 'ca_bundle' if 'verify_ssl' is true and 'ca_bundle' is configured."""

From cfce1aa2cb8d8d5266e7695a6ad3ce52de75cb11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= <bjoern@gruenings.eu>
Date: Sun, 1 Aug 2021 12:02:59 +0200
Subject: [PATCH 244/452] allow HTML in DatasetStorage view

---
 client/src/components/Dataset/DatasetStorage/DatasetStorage.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
index 7232d97e508..83589c5ce76 100644
--- a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
+++ b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
@@ -66,7 +66,7 @@ export default {
             const description = storageInfo.description;
             this.storageInfo = storageInfo;
             if (description) {
-                this.descriptionRendered = MarkdownIt().render(storageInfo.description);
+                this.descriptionRendered = MarkdownIt({html: true}).render(storageInfo.description);
             } else {
                 this.descriptionRendered = null;
             }

From 1fee6893248169c17dc69ad3c4961a4d0313b338 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Mon, 26 Jul 2021 16:52:29 +0530
Subject: [PATCH 245/452] Allow absolute or relative url in assertion

---
 test/unit/authnz/test_custos_authnz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/authnz/test_custos_authnz.py b/test/unit/authnz/test_custos_authnz.py
index 2b71f40d16b..4cf04f52215 100644
--- a/test/unit/authnz/test_custos_authnz.py
+++ b/test/unit/authnz/test_custos_authnz.py
@@ -313,7 +313,7 @@ class CustosAuthnzTestCase(unittest.TestCase):
         self.assertTrue(self._create_oauth2_session_called)
         self.assertTrue(self._fetch_token_called)
         self.assertTrue(self._get_userinfo_called)
-        self.assertEqual(login_redirect_url, "http://localhost:8000/")
+        self.assertEqual(login_redirect_url, "/")
         self.assertIsNotNone(user)
 
     def test_callback_nonce_validation_with_bad_nonce(self):

From f2413b1c0781e6a850251ff860eb2676923940e0 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Fri, 30 Jul 2021 11:20:14 +0200
Subject: [PATCH 246/452] add output linter test

making sure that an output collection with statically defined
elements that use `format_source` are accepted (which is not the case
yet)
---
 test/unit/tool_util/test_tool_linters.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index 5323ea820a8..fa65480507c 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -1,7 +1,7 @@
 import pytest
 
 from galaxy.tool_util.lint import LintContext
-from galaxy.tool_util.linters import general, inputs
+from galaxy.tool_util.linters import general, inputs, outputs
 from galaxy.tool_util.parser.xml import XmlToolSource
 from galaxy.util import etree
 
@@ -134,6 +134,19 @@ VALIDATOR_INCOMPATIBILITIES = """
 </tool>
 """
 
+OUTPUTS_COLLECTION_FORMAT_SOURCE = """
+<tool name="BWA Mapper" id="bwa" version="1.0.1" is_multi_byte="true" display_interface="true" require_login="true" hidden="true">
+    <description>The BWA Mapper</description>
+    <version_command interpreter="python">bwa.py --version</version_command>
+    <outputs>
+        <collection name="output_collection" type="paired">
+            <data name="forward" format_source="input_readpair" />
+            <data name="reverse" format_source="input_readpair" />
+        </collection>
+    </outputs>
+</tool>
+"""
+
 TESTS = [
     (
         NO_SECTIONS_XML, inputs.lint_inputs,
@@ -202,6 +215,11 @@ TESTS = [
             and "Parameter [param_name]: 'regex' validators need to define an 'expression' attribute" in x.error_messages
             and len(x.warn_messages) == 1 and len(x.error_messages) == 4
     ),
+    (
+        OUTPUTS_COLLECTION_FORMAT_SOURCE, outputs.lint_output,
+        lambda x:
+            len(x.warn_messages) == 0 and len(x.error_messages) == 0
+    ),
 ]
 
 TEST_IDS = [
@@ -212,7 +230,8 @@ TEST_IDS = [
     'select deprecations',
     'select option definitions',
     'hazardous whitespace',
-    'validator imcompatibilities'
+    'validator imcompatibilities',
+    'outputs collection static elements with format_source'
 ]
 
 

From d945f57efa7e8492597d177bddcc86687a5d73f4 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Fri, 30 Jul 2021 13:16:26 +0200
Subject: [PATCH 247/452] linter support for format_source in static collection
 elements

add support for linting `format_source` in static collection elements.
also warn for using `format_source` and `ext`/`format`.
---
 lib/galaxy/tool_util/linters/outputs.py  | 10 ++++++----
 test/unit/tool_util/test_tool_linters.py |  5 +++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/tool_util/linters/outputs.py b/lib/galaxy/tool_util/linters/outputs.py
index 42e5ca68ea8..6b5dfff886b 100644
--- a/lib/galaxy/tool_util/linters/outputs.py
+++ b/lib/galaxy/tool_util/linters/outputs.py
@@ -38,8 +38,6 @@ def lint_output(tool_xml, lint_ctx):
                 lint_ctx.warn("Collection output with undefined 'type' found.")
             if "structured_like" in output.attrib and "inherit_format" in output.attrib:
                 format_set = True
-        if "format_source" in output.attrib:
-            format_set = True
         for sub in output:
             if __check_pattern(sub):
                 format_set = True
@@ -54,12 +52,16 @@ def lint_output(tool_xml, lint_ctx):
 
 def __check_format(node, lint_ctx, allow_ext=False):
     """
-    check if format/ext attribute is set in a given node
+    check if format/ext/format_source attribute is set in a given node
     issue a warning if the value is input
     return true (node defines format/ext) / false (else)
     """
-    fmt = None
+    if "format_source" in node.attrib and ("ext" in node.attrib or "format" in node.attrib):
+        lint_ctx.warn(f"Tool {node.tag} output {node.attrib.get('name', 'with missing name')} should use either format_source or format/ext")
+    if "format_source" in node.attrib:
+        return True
     # if allowed (e.g. for discover_datasets), ext takes precedence over format
+    fmt = None
     if allow_ext:
         fmt = node.attrib.get("ext")
     if fmt is None:
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index fa65480507c..2a15b99023c 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -141,7 +141,7 @@ OUTPUTS_COLLECTION_FORMAT_SOURCE = """
     <outputs>
         <collection name="output_collection" type="paired">
             <data name="forward" format_source="input_readpair" />
-            <data name="reverse" format_source="input_readpair" />
+            <data name="reverse" format_source="input_readpair" format="fastq"/>
         </collection>
     </outputs>
 </tool>
@@ -218,7 +218,8 @@ TESTS = [
     (
         OUTPUTS_COLLECTION_FORMAT_SOURCE, outputs.lint_output,
         lambda x:
-            len(x.warn_messages) == 0 and len(x.error_messages) == 0
+            "Tool data output reverse should use either format_source or format/ext" in x.warn_messages
+            and len(x.warn_messages) == 1 and len(x.error_messages) == 0
     ),
 ]
 

From a00c50f49e65e820c24f79978aa6a4efeb93d740 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Mon, 2 Aug 2021 15:39:08 +0200
Subject: [PATCH 248/452] xsd: document the delta attribute of has_size assert

---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 578e6067ef1..719962630b6 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -2027,14 +2027,25 @@ module.
       </xs:element>
       <xs:element name="has_archive_member" type="AssertHasArchiveMember">
       </xs:element>
-      <xs:element name="has_size" type="xs:anyType">
-        <xs:annotation>
-          <xs:documentation xml:lang="en"><![CDATA[Asserts the output has a size (in bytes) of the specified ``value``; if the size is allowed to deviate from this value, the maximum difference can be optionally specified with ``delta`` (e.g. ``<has_size value="10000" delta="100" />``).]]>
-          </xs:documentation>
-        </xs:annotation>
+      <xs:element name="has_size" type="AssertHasSize">
       </xs:element>
     </xs:choice>
   </xs:group>
+  <xs:complexType name="AssertHasSize">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[Asserts the output has specific size (in bytes), e.g. ``<has_size value="10000" delta="100" />``.]]></xs:documentation>
+    </xs:annotation>
+    <xs:attribute name="value" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Desired size of the outpyt (in bytes)</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="delta" type="xs:integer" use="optional">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Maximum allowed size difference (default is 0).</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:complexType>
   <xs:complexType name="AssertHasH5Keys">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[Asserts HDF5 output has a set of attributes (``keys``), specified as a comma-separated list (e.g. ``<has_h5_keys keys="bins,chroms,indexes,pixels,chroms/lengths" />``).]]></xs:documentation>

From 6c477eb7c3a420381e2211c9a73849bc6245c11d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 25 Jul 2021 14:47:22 -0400
Subject: [PATCH 249/452] Replace form views in charts with mounted FormDisplay
 component

---
 .../mvc/visualization/chart/views/groups.js    | 18 ++++++++----------
 .../mvc/visualization/chart/views/settings.js  | 17 ++++++++---------
 client/src/utils/mountVueComponent.js          |  8 ++++++++
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/client/src/mvc/visualization/chart/views/groups.js b/client/src/mvc/visualization/chart/views/groups.js
index 18651df6096..d26259fcba8 100644
--- a/client/src/mvc/visualization/chart/views/groups.js
+++ b/client/src/mvc/visualization/chart/views/groups.js
@@ -5,9 +5,10 @@ import $ from "jquery";
 import Backbone from "backbone";
 import { getAppRoot } from "onload/loadConfig";
 import Utils from "utils/utils";
-import Form from "mvc/form/form-view";
 import Repeat from "mvc/form/form-repeat";
 import FormData from "mvc/form/form-data";
+import FormDisplay from "components/Form/FormDisplay";
+import { appendVueComponent } from "utils/mountVueComponent";
 
 var GroupView = Backbone.View.extend({
     initialize: function (app, options) {
@@ -66,18 +67,15 @@ var GroupView = Backbone.View.extend({
                             value: data_columns,
                         });
                         self.chart.state("ok", "Metadata initialized...");
-                        self.form = new Form({
+                        const instance = appendVueComponent(self.$el, FormDisplay, {
                             inputs: inputs,
-                            onchange: function () {
-                                self.group.set(self.form.data.create());
-                                self.chart.set("modified", true);
-                                self.chart.trigger("redraw");
-                            },
                         });
-                        self.group.set(self.form.data.create());
-                        self.$el.empty().append(self.form.$el);
+                        instance.$on("onChange", (data) => {
+                            self.group.set(data);
+                            self.chart.set("modified", true);
+                            self.chart.trigger("redraw");
+                        });
                         process.resolve();
-                        self.chart.trigger("redraw");
                     },
                 });
             });
diff --git a/client/src/mvc/visualization/chart/views/settings.js b/client/src/mvc/visualization/chart/views/settings.js
index 1cebb8b7f51..dcd4c7f49f0 100644
--- a/client/src/mvc/visualization/chart/views/settings.js
+++ b/client/src/mvc/visualization/chart/views/settings.js
@@ -2,11 +2,12 @@
 import _ from "underscore";
 import Backbone from "backbone";
 import Utils from "utils/utils";
-import Form from "mvc/form/form-view";
+import FormDisplay from "components/Form/FormDisplay";
 import FormData from "mvc/form/form-data";
+import { appendVueComponent } from "utils/mountVueComponent";
 
 export default Backbone.View.extend({
-    initialize: function (app, options) {
+    initialize: function (app) {
         var self = this;
         this.chart = app.chart;
         this.setElement("<div/>");
@@ -37,15 +38,13 @@ export default Backbone.View.extend({
                     input.value = model_value;
                 }
             });
-            this.form = new Form({
+            const instance = appendVueComponent(this.$el, FormDisplay, {
                 inputs: inputs,
-                onchange: function () {
-                    self.chart.settings.set(self.form.data.create());
-                    self.chart.trigger("redraw");
-                },
             });
-            this.chart.settings.set(this.form.data.create());
-            this.$el.append(this.form.$el);
+            instance.$on("onChange", (data) => {
+                this.chart.settings.set(data);
+                this.chart.trigger("redraw");
+            });
         }
     },
 });
diff --git a/client/src/utils/mountVueComponent.js b/client/src/utils/mountVueComponent.js
index 3c8a0df173f..8f685614822 100644
--- a/client/src/utils/mountVueComponent.js
+++ b/client/src/utils/mountVueComponent.js
@@ -28,3 +28,11 @@ export const mountVueComponent = (ComponentDefinition) => {
     const component = Vue.extend(ComponentDefinition);
     return (propsData, el) => new component({ store, propsData, el });
 };
+
+export const appendVueComponent = ($el, ComponentDefinition, propsData = {}) => {
+    const container = document.createElement("div");
+    $el.empty().append(container);
+    const component = Vue.extend(ComponentDefinition);
+    const mountFn = (propsData, el) => new component({ propsData, el });
+    return mountFn(propsData, container);
+};

From ea10fbe89bc7f67cfcd8eb81042126772029b88e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 25 Jul 2021 21:37:15 -0400
Subject: [PATCH 250/452] Make colors optional in nvd3 visualizations

---
 client/src/mvc/dataset/dataset-edit-attributes.js               | 1 -
 client/src/mvc/visualization/chart/views/settings.js            | 2 +-
 .../plugins/visualizations/nvd3/nvd3_bar/config/nvd3_shared.xml | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/mvc/dataset/dataset-edit-attributes.js b/client/src/mvc/dataset/dataset-edit-attributes.js
index 9d32f43a5d3..f14bfaad0ee 100644
--- a/client/src/mvc/dataset/dataset-edit-attributes.js
+++ b/client/src/mvc/dataset/dataset-edit-attributes.js
@@ -5,7 +5,6 @@ import Backbone from "backbone";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
 import _l from "utils/localization";
-// import Utils from "utils/utils";
 import Tabs from "mvc/ui/ui-tabs";
 import Ui from "mvc/ui/ui-misc";
 import Form from "mvc/form/form-view";
diff --git a/client/src/mvc/visualization/chart/views/settings.js b/client/src/mvc/visualization/chart/views/settings.js
index dcd4c7f49f0..e78404b223d 100644
--- a/client/src/mvc/visualization/chart/views/settings.js
+++ b/client/src/mvc/visualization/chart/views/settings.js
@@ -2,8 +2,8 @@
 import _ from "underscore";
 import Backbone from "backbone";
 import Utils from "utils/utils";
-import FormDisplay from "components/Form/FormDisplay";
 import FormData from "mvc/form/form-data";
+import FormDisplay from "components/Form/FormDisplay";
 import { appendVueComponent } from "utils/mountVueComponent";
 
 export default Backbone.View.extend({
diff --git a/config/plugins/visualizations/nvd3/nvd3_bar/config/nvd3_shared.xml b/config/plugins/visualizations/nvd3/nvd3_bar/config/nvd3_shared.xml
index c96f4104468..c218842603b 100644
--- a/config/plugins/visualizations/nvd3/nvd3_bar/config/nvd3_shared.xml
+++ b/config/plugins/visualizations/nvd3/nvd3_bar/config/nvd3_shared.xml
@@ -26,6 +26,7 @@
             <label>Pick a series color</label>
             <name>color</name>
             <type>color</type>
+            <optional>true</optional>
         </input>
         <input>
             <label>Column of data point labels</label>

From eb2e064fc7c6039027e9ff90adb31fa03a5e4e32 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 2 Aug 2021 11:20:10 -0400
Subject: [PATCH 251/452] prettier.

---
 client/src/components/Dataset/DatasetStorage/DatasetStorage.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
index 83589c5ce76..37f661fe0a1 100644
--- a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
+++ b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
@@ -66,7 +66,7 @@ export default {
             const description = storageInfo.description;
             this.storageInfo = storageInfo;
             if (description) {
-                this.descriptionRendered = MarkdownIt({html: true}).render(storageInfo.description);
+                this.descriptionRendered = MarkdownIt({ html: true }).render(storageInfo.description);
             } else {
                 this.descriptionRendered = null;
             }

From e540ef5775f9f8257aa4620d798bf710d416c6a4 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 2 Aug 2021 17:27:50 +0200
Subject: [PATCH 252/452] Continue handler startup even if job working
 directory can't be recovered

---
 lib/galaxy/jobs/handler.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py
index 4c5fa582025..7326a3dc02c 100644
--- a/lib/galaxy/jobs/handler.py
+++ b/lib/galaxy/jobs/handler.py
@@ -21,6 +21,7 @@ from sqlalchemy.sql.expression import (
 )
 
 from galaxy import model
+from galaxy.exceptions import ObjectNotFound
 from galaxy.jobs import (
     JobDestination,
     JobWrapper,
@@ -1087,6 +1088,10 @@ class DefaultJobDispatcher:
         except KeyError:
             log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
             job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
+        except ObjectNotFound:
+            msg = "Could not recover job working directory after Galaxy restart"
+            log.exception(f"recover(): ({job_wrapper.job_id}) {msg}")
+            job_wrapper.fail(msg)
 
     def shutdown(self):
         failures = []

From 2667d79f4f3bf1329e55074cb4bafbf90f660c8d Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Tue, 3 Aug 2021 17:54:26 +0530
Subject: [PATCH 253/452] Default k8s_unschedulable_walltime_limit to None and
 allow for unlimited queue wait time

---
 lib/galaxy/jobs/runners/kubernetes.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py
index 57c6a950fdb..7cb7923f6e0 100644
--- a/lib/galaxy/jobs/runners/kubernetes.py
+++ b/lib/galaxy/jobs/runners/kubernetes.py
@@ -83,7 +83,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"),
             k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=3),
             k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800),
-            k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=1800),
+            k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None),
             k8s_interactivetools_use_ssl=dict(map=bool, default=False),
             k8s_interactivetools_ingress_annotations=dict(map=str),)
 
@@ -676,11 +676,16 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             elif active > 0 and failed <= max_pod_retries:
                 if not job_state.running:
                     if self.__job_pending_due_to_unschedulable_pod(job_state):
-                        creation_time_str = job.obj['metadata'].get('creationTimestamp')
-                        creation_time = datetime.strptime(creation_time_str, '%Y-%m-%dT%H:%M:%SZ')
-                        elapsed_seconds = (datetime.utcnow() - creation_time).total_seconds()
-                        if elapsed_seconds > self.runner_params['k8s_unschedulable_walltime_limit']:
-                            return self._handle_unschedulable_job(job, job_state)
+                        if self.runner_params.get('k8s_unschedulable_walltime_limit'):
+                            creation_time_str = job.obj['metadata'].get('creationTimestamp')
+                            creation_time = datetime.strptime(creation_time_str, '%Y-%m-%dT%H:%M:%SZ')
+                            elapsed_seconds = (datetime.utcnow() - creation_time).total_seconds()
+                            if elapsed_seconds > self.runner_params['k8s_unschedulable_walltime_limit']:
+                                return self._handle_unschedulable_job(job, job_state)
+                            else:
+                                pass
+                        else:
+                            pass
                     else:
                         job_state.running = True
                         job_state.job_wrapper.change_state(model.Job.states.RUNNING)

From 36ef4ab1d960c57d894130049e00cd064d5a5f85 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Tue, 3 Aug 2021 19:18:48 +0530
Subject: [PATCH 254/452] Added documentation on
 k8s_unschedulable_walltime_limit parameter

---
 lib/galaxy/config/sample/job_conf.xml.sample_advanced | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced
index 795998ed9be..fa78f51f8f7 100644
--- a/lib/galaxy/config/sample/job_conf.xml.sample_advanced
+++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced
@@ -322,6 +322,9 @@
             <!-- Whether to enabled HTTPS access in the k8s ingress spec generated for each interactive tools -->
             <!-- <param id="k8s_interactivetools_ingress_annotations">{'cert-manager.io/cluster-issuer': 'letsencrypt-prod'}</param> -->
             <!-- Annotations to add to the metadata section in the k8s ingress generated for each interactive tools -->
+            <!-- <param id="k8s_unschedulable_walltime_limit">172800</param> -->
+            <!-- The amount of time (in seconds) to let a job remain in an unschedulable state before being flagged
+                 as having failed. The default is None (unlimited time). -->
 
         </plugin>
         <plugin id="godocker" type="runner" load="galaxy.jobs.runners.godocker:GodockerJobRunner">

From fbb7c0dad70b880b5cbdb111954b3411d11ad2a8 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Thu, 29 Jul 2021 21:55:27 +0200
Subject: [PATCH 255/452] fix recursive

---
 lib/galaxy/files/sources/posix.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/files/sources/posix.py b/lib/galaxy/files/sources/posix.py
index dcdd53fbe82..3dcbf08ea77 100644
--- a/lib/galaxy/files/sources/posix.py
+++ b/lib/galaxy/files/sources/posix.py
@@ -39,7 +39,8 @@ class PosixFilesSource(BaseFilesSource):
         if recursive:
             res = []
             for (p, dirs, files) in safe_walk(dir_path, allowlist=self._allowlist):
-                rel_dir = os.path.relpath(p, dir_path)
+                dir_path_root = os.path.split(dir_path)[0]
+                rel_dir = os.path.relpath(p, dir_path_root)
                 to_dict = functools.partial(self._resource_info_to_dict, rel_dir, user_context=user_context)
                 res.extend(map(to_dict, dirs))
                 res.extend(map(to_dict, files))

From 93218821048f547fe5fc99b7332c0e57706fdb36 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Fri, 30 Jul 2021 16:57:49 +0200
Subject: [PATCH 256/452] use root path

---
 lib/galaxy/files/sources/posix.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/files/sources/posix.py b/lib/galaxy/files/sources/posix.py
index 3dcbf08ea77..aab43fce6f1 100644
--- a/lib/galaxy/files/sources/posix.py
+++ b/lib/galaxy/files/sources/posix.py
@@ -39,8 +39,7 @@ class PosixFilesSource(BaseFilesSource):
         if recursive:
             res = []
             for (p, dirs, files) in safe_walk(dir_path, allowlist=self._allowlist):
-                dir_path_root = os.path.split(dir_path)[0]
-                rel_dir = os.path.relpath(p, dir_path_root)
+                rel_dir = os.path.relpath(p, self.root)
                 to_dict = functools.partial(self._resource_info_to_dict, rel_dir, user_context=user_context)
                 res.extend(map(to_dict, dirs))
                 res.extend(map(to_dict, files))

From 1e04bf91ecaba0031010c10c3e4a6e84597bfa32 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Tue, 3 Aug 2021 18:47:17 +0200
Subject: [PATCH 257/452] fix integration test

---
 lib/galaxy/files/sources/posix.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/files/sources/posix.py b/lib/galaxy/files/sources/posix.py
index aab43fce6f1..43bc1d95d9f 100644
--- a/lib/galaxy/files/sources/posix.py
+++ b/lib/galaxy/files/sources/posix.py
@@ -38,8 +38,9 @@ class PosixFilesSource(BaseFilesSource):
             raise exceptions.ObjectNotFound(f'The specified directory does not exist [{dir_path}].')
         if recursive:
             res = []
+            effective_root = self._effective_root(user_context)
             for (p, dirs, files) in safe_walk(dir_path, allowlist=self._allowlist):
-                rel_dir = os.path.relpath(p, self.root)
+                rel_dir = os.path.relpath(p, effective_root)
                 to_dict = functools.partial(self._resource_info_to_dict, rel_dir, user_context=user_context)
                 res.extend(map(to_dict, dirs))
                 res.extend(map(to_dict, files))

From 28394f006827fc848f710a87c551a0da7496d473 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Wed, 4 Aug 2021 12:48:25 +0200
Subject: [PATCH 258/452] make linter accept from_tool_provided_metadata

as means to specify the format for discovered datasets
---
 lib/galaxy/tool_util/linters/outputs.py  |  7 ++++++-
 test/unit/tool_util/test_tool_linters.py | 22 +++++++++++++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tool_util/linters/outputs.py b/lib/galaxy/tool_util/linters/outputs.py
index 6b5dfff886b..8d0b7fe858f 100644
--- a/lib/galaxy/tool_util/linters/outputs.py
+++ b/lib/galaxy/tool_util/linters/outputs.py
@@ -1,4 +1,5 @@
 """This module contains a linting functions for tool outputs."""
+from galaxy.util import string_as_bool
 from ._util import is_valid_cheetah_placeholder
 
 
@@ -73,10 +74,14 @@ def __check_format(node, lint_ctx, allow_ext=False):
 
 def __check_pattern(node):
     """
-    check if pattern attribute is set and defines the extension
+    check if
+    - pattern attribute is set and defines the extension or
+    - from_tool_provided_metadata is true
     """
     if node.tag != "discover_datasets":
         return False
+    if "from_tool_provided_metadata" in node.attrib and string_as_bool(node.attrib.get("from_tool_provided_metadata", "false")):
+        return True
     if "pattern" not in node.attrib:
         return False
     if node.attrib["pattern"] == "__default__":
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index 2a15b99023c..0dab32dba2e 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -134,6 +134,8 @@ VALIDATOR_INCOMPATIBILITIES = """
 </tool>
 """
 
+# check that linter accepts format source for collection elements as means to specify format
+# and that the linter warns if format and format_source are used
 OUTPUTS_COLLECTION_FORMAT_SOURCE = """
 <tool name="BWA Mapper" id="bwa" version="1.0.1" is_multi_byte="true" display_interface="true" require_login="true" hidden="true">
     <description>The BWA Mapper</description>
@@ -147,6 +149,18 @@ OUTPUTS_COLLECTION_FORMAT_SOURCE = """
 </tool>
 """
 
+# check that linter does not complain about missing format if from_tool_provided_metadata is used
+OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA = """
+<tool name="BWA Mapper" id="bwa" version="1.0.1" is_multi_byte="true" display_interface="true" require_login="true" hidden="true">
+    <description>The BWA Mapper</description>
+    <version_command interpreter="python">bwa.py --version</version_command>
+    <outputs>
+        <data name="output">
+            <discover_datasets from_tool_provided_metadata="true"/>
+        </data>
+    </outputs>
+</tool>
+"""
 TESTS = [
     (
         NO_SECTIONS_XML, inputs.lint_inputs,
@@ -221,6 +235,11 @@ TESTS = [
             "Tool data output reverse should use either format_source or format/ext" in x.warn_messages
             and len(x.warn_messages) == 1 and len(x.error_messages) == 0
     ),
+    (
+        OUTPUTS_DISCOVER_TOOL_PROVIDED_METADATA, outputs.lint_output,
+        lambda x:
+            len(x.warn_messages) == 0 and len(x.error_messages) == 0
+    ),
 ]
 
 TEST_IDS = [
@@ -232,7 +251,8 @@ TEST_IDS = [
     'select option definitions',
     'hazardous whitespace',
     'validator imcompatibilities',
-    'outputs collection static elements with format_source'
+    'outputs collection static elements with format_source',
+    'outputs discover datatsets with tool provided metadata'
 ]
 
 

From a7bba11208924186245ca31f2cf166f191cab992 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 26 Jul 2021 15:28:27 -0400
Subject: [PATCH 259/452] Use modified SingleQueryProvider (thx @nerdinacan) to
 Vueify Dataset Error display

---
 client/package.json                           |  1 +
 .../DatasetInformation/DatasetError.vue       | 84 +++++++++++++++++++
 .../src/components/providers/JobProvider.js   | 18 ++++
 .../providers/SingleQueryProvider.js          | 40 +++++++--
 client/src/entry/analysis/AnalysisRouter.js   |  7 +-
 client/yarn.lock                              |  5 ++
 6 files changed, 144 insertions(+), 11 deletions(-)
 create mode 100644 client/src/components/DatasetInformation/DatasetError.vue
 create mode 100644 client/src/components/providers/JobProvider.js

diff --git a/client/package.json b/client/package.json
index bf6b816241c..2c70f9711bf 100644
--- a/client/package.json
+++ b/client/package.json
@@ -55,6 +55,7 @@
     "markdown-it": "^11.0.0",
     "markdown-it-regexp": "^0.4.0",
     "moment": "2.26.0",
+    "object-hash": "^2.2.0",
     "popper.js": "^1.16.1",
     "pouchdb": "^7.2.2",
     "pouchdb-adapter-memory": "^7.2.2",
diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
new file mode 100644
index 00000000000..255fc27a7b0
--- /dev/null
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -0,0 +1,84 @@
+<template>
+    <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading }">
+        <div v-if="!loading">
+            <JobDetailsProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
+                <div v-if="!loading">
+                    <div class="page-container edit-attr">
+                        <div class="response-message"></div>
+                    </div>
+                    <h2>Dataset Error Report</h2>
+                    <p>
+                        An error occurred while running the tool <b>{{ result.tool_id }}</b
+                        >.
+                    </p>
+                    <div v-if="result.tool_stderr || result.job_stderr || result.job_messages">
+                        <h3>Details</h3>
+                    </div>
+                    <div v-if="result.job_messages">
+                        <p>Execution resulted in the following messages:</p>
+                        <div v-for="job_message in result.job_messages">
+                            <pre class="rounded code">{{ job_message["desc"] }}</pre>
+                        </div>
+                    </div>
+                    <div v-if="result.tool_stderr">
+                        <p>Tool generated the following standard error:</p>
+                        <pre class="rounded code">{{ result.tool_stderr }}</pre>
+                    </div>
+                    <div v-if="result.job_stderr">
+                        <p>Galaxy job runner generated the following standard error:</p>
+                        <pre class="rounded code">{{ result.job_stderr }}</pre>
+                    </div>
+                </div>
+            </JobDetailsProvider>
+            <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
+                <h3 class="common_problems"></h3>
+            </JobProblemProvider>
+            <h3>Troubleshooting</h3>
+            <p>
+                There are a number of helpful resources to self diagnose and correct problems.
+                <br />
+                Start here:
+                <b>
+                    <a href="https://galaxyproject.org/support/tool-error/" target="_blank">
+                        My job ended with an error. What can I do?
+                    </a>
+                </b>
+            </p>
+            <h3>Issue Report</h3>
+        </div>
+    </DatasetProvider>
+</template>
+
+<script>
+import { DatasetProvider } from "components/WorkflowInvocationState/providers";
+import { JobDetailsProvider, JobProblemProvider } from "components/providers/JobProvider";
+import FormDisplay from "components/Form/FormDisplay";
+import FormCard from "components/Form/FormCard";
+
+export default {
+    components: {
+        DatasetProvider,
+        FormDisplay,
+        FormCard,
+        JobDetailsProvider,
+        JobProblemProvider,
+    },
+    props: {
+        datasetId: {
+            type: String,
+            required: true,
+        },
+    },
+    data() {
+        return {};
+    },
+    computed: {
+        inputs() {
+            return [];
+        },
+    },
+    methods: {
+        onChange(data) {},
+    },
+};
+</script>
diff --git a/client/src/components/providers/JobProvider.js b/client/src/components/providers/JobProvider.js
new file mode 100644
index 00000000000..913af215cc4
--- /dev/null
+++ b/client/src/components/providers/JobProvider.js
@@ -0,0 +1,18 @@
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
+
+async function jobDetails({ jobid }) {
+    const url = `${getAppRoot()}api/jobs/${jobid}?full=True`;
+    const { data } = await axios.get(url);
+    return data;
+}
+
+async function jobProblems({ jobid }) {
+    const url = `${getAppRoot()}api/jobs/${jobid}/common_problems`;
+    const { data } = await axios.get(url);
+    return data;
+}
+
+export const JobDetailsProvider = SingleQueryProvider(jobDetails);
+export const JobProblemProvider = SingleQueryProvider(jobProblems);
diff --git a/client/src/components/providers/SingleQueryProvider.js b/client/src/components/providers/SingleQueryProvider.js
index 2bea088d34e..5327ac4e506 100644
--- a/client/src/components/providers/SingleQueryProvider.js
+++ b/client/src/components/providers/SingleQueryProvider.js
@@ -1,7 +1,16 @@
-export const SingleQueryProvider = (cfg = {}) => {
-    const { resultName = "result", lookup } = cfg;
+import hash from "object-hash";
 
-    let cached_result;
+/**
+ * Builds a provider that gets its result from a single promise-based query function and
+ * caches the result of lookup for subsequent instantitations.
+ *
+ * @param   {Function}  lookup  async function that loads the result, paramters will be an object
+ *                              whose properties are the attributes assigned to the provider component
+ * @return  {VueComponentOptions} Vue component options definition
+ */
+export const SingleQueryProvider = (lookup) => {
+    const promiseCache = new Map();
+    const resultCache = new Map();
 
     return {
         data() {
@@ -13,17 +22,32 @@ export const SingleQueryProvider = (cfg = {}) => {
             loading() {
                 return this.result === undefined;
             },
+            cacheKey() {
+                hash(this.$attrs || {});
+            },
         },
-        async created() {
-            if (undefined === cached_result) {
-                cached_result = await lookup();
+        mounted() {
+            const cachedResult = resultCache.get(this.cacheKey);
+            if (cachedResult) {
+                this.result = cachedResult;
+                return;
             }
-            this.result = cached_result;
+
+            let lookupPromise = promiseCache.get(this.cacheKey);
+            if (!lookupPromise) {
+                lookupPromise = lookup(this.$attrs);
+                promiseCache.set(this.cacheKey, lookupPromise);
+            }
+
+            lookupPromise.then((result) => {
+                resultCache.set(this.cacheKey, result);
+                this.result = result;
+            });
         },
         render() {
             return this.$scopedSlots.default({
                 loading: this.loading,
-                [resultName]: this.result,
+                result: this.result,
             });
         },
     };
diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index d0e31fa7618..1e34faef65a 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -41,7 +41,7 @@ import ToolsJson from "components/ToolsView/ToolsSchemaJson/ToolsJson.vue";
 import HistoryList from "mvc/history/history-list";
 import PluginList from "components/PluginList.vue";
 import QueryStringParsing from "utils/query-string-parsing";
-import DatasetError from "mvc/dataset/dataset-error";
+import DatasetError from "components/DatasetInformation/DatasetError";
 import DatasetEditAttributes from "mvc/dataset/dataset-edit-attributes";
 import Citations from "components/Citation/Citations.vue";
 import DisplayStructure from "components/DisplayStructured.vue";
@@ -389,8 +389,9 @@ export const getAnalysisRouter = (Galaxy) => {
             this.page.display(new DatasetEditAttributes.View());
         },
 
-        show_dataset_error: function () {
-            this.page.display(new DatasetError.View());
+        show_dataset_error: function (params) {
+            const datasetId = params.dataset_id;
+            this._display_vue_helper(DatasetError, { datasetId: datasetId });
         },
 
         /**  */
diff --git a/client/yarn.lock b/client/yarn.lock
index cdc734cab1e..829749ec72d 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -10333,6 +10333,11 @@ object-copy@^0.1.0:
     define-property "^0.2.5"
     kind-of "^3.0.3"
 
+object-hash@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5"
+  integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==
+
 object-inspect@^1.8.0:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.9.0.tgz#c90521d74e1127b67266ded3394ad6116986533a"

From 00418f138ae7057e5a01808a81a53b0061199dbd Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 26 Jul 2021 15:37:49 -0400
Subject: [PATCH 260/452] Add common job problem section to dataset error
 component

---
 .../components/DatasetInformation/DatasetError.vue   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 255fc27a7b0..12134ca3585 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -31,7 +31,17 @@
                 </div>
             </JobDetailsProvider>
             <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
-                <h3 class="common_problems"></h3>
+                <div v-if="result.has_duplicate_inputs || result.has_empty_inputs">
+                    <h3 class="common_problems">Detected Common Potential Problems</h3>
+                    <p v-if="result.has_empty_inputs">
+                        The tool was executed with one or more empty input datasets. This frequently
+                        results in tool errors due to problematic input choices.
+                    <p>
+                    <p v-if="result.has_duplicate_inputs">
+                        The tool was executed with one or more duplicate input datasets. This frequently
+                        results in tool errors due to problematic input choices.
+                    <p>
+                </div>
             </JobProblemProvider>
             <h3>Troubleshooting</h3>
             <p>

From 7dd797d8ce86b8fd3d2ee2787d235146a6ff40cf Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 27 Jul 2021 18:02:05 -0400
Subject: [PATCH 261/452] Remove legacy dataset error backbone module, replace
 with Vue component

---
 .../DatasetInformation/DatasetError.vue       | 166 +++++++-----
 client/src/mvc/dataset/dataset-error.js       | 244 ------------------
 2 files changed, 100 insertions(+), 310 deletions(-)
 delete mode 100644 client/src/mvc/dataset/dataset-error.js

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 12134ca3585..56109288092 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -1,75 +1,98 @@
 <template>
-    <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading }">
-        <div v-if="!loading">
-            <JobDetailsProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
-                <div v-if="!loading">
-                    <div class="page-container edit-attr">
-                        <div class="response-message"></div>
-                    </div>
-                    <h2>Dataset Error Report</h2>
-                    <p>
-                        An error occurred while running the tool <b>{{ result.tool_id }}</b
-                        >.
-                    </p>
-                    <div v-if="result.tool_stderr || result.job_stderr || result.job_messages">
-                        <h3>Details</h3>
-                    </div>
-                    <div v-if="result.job_messages">
-                        <p>Execution resulted in the following messages:</p>
-                        <div v-for="job_message in result.job_messages">
-                            <pre class="rounded code">{{ job_message["desc"] }}</pre>
+    <CurrentUser v-slot="{ user }">
+        <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading }">
+            <div v-if="!loading">
+                <JobDetailsProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
+                    <div v-if="!loading">
+                        <div class="page-container edit-attr">
+                            <div class="response-message"></div>
+                        </div>
+                        <h2>Dataset Error Report</h2>
+                        <p>
+                            An error occurred while running the tool <b>{{ result.tool_id }}</b
+                            >.
+                        </p>
+                        <div v-if="result.tool_stderr || result.job_stderr || result.job_messages">
+                            <h3>Details</h3>
+                        </div>
+                        <div v-if="result.job_messages">
+                            <p>Execution resulted in the following messages:</p>
+                            <div v-for="job_message in result.job_messages">
+                                <pre class="rounded code">{{ job_message["desc"] }}</pre>
+                            </div>
+                        </div>
+                        <div v-if="result.tool_stderr">
+                            <p>Tool generated the following standard error:</p>
+                            <pre class="rounded code">{{ result.tool_stderr }}</pre>
+                        </div>
+                        <div v-if="result.job_stderr">
+                            <p>Galaxy job runner generated the following standard error:</p>
+                            <pre class="rounded code">{{ result.job_stderr }}</pre>
                         </div>
                     </div>
-                    <div v-if="result.tool_stderr">
-                        <p>Tool generated the following standard error:</p>
-                        <pre class="rounded code">{{ result.tool_stderr }}</pre>
+                </JobDetailsProvider>
+                <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
+                    <div v-if="result && (result.has_duplicate_inputs || result.has_empty_inputs)">
+                        <h3 class="common_problems">Detected Common Potential Problems</h3>
+                        <p v-if="result.has_empty_inputs">
+                            The tool was executed with one or more empty input datasets. This frequently results in tool
+                            errors due to problematic input choices.
+                        </p>
+                        <p v-if="result && result.has_duplicate_inputs">
+                            The tool was executed with one or more duplicate input datasets. This frequently results in
+                            tool errors due to problematic input choices.
+                        </p>
                     </div>
-                    <div v-if="result.job_stderr">
-                        <p>Galaxy job runner generated the following standard error:</p>
-                        <pre class="rounded code">{{ result.job_stderr }}</pre>
-                    </div>
-                </div>
-            </JobDetailsProvider>
-            <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
-                <div v-if="result.has_duplicate_inputs || result.has_empty_inputs">
-                    <h3 class="common_problems">Detected Common Potential Problems</h3>
-                    <p v-if="result.has_empty_inputs">
-                        The tool was executed with one or more empty input datasets. This frequently
-                        results in tool errors due to problematic input choices.
-                    <p>
-                    <p v-if="result.has_duplicate_inputs">
-                        The tool was executed with one or more duplicate input datasets. This frequently
-                        results in tool errors due to problematic input choices.
-                    <p>
-                </div>
-            </JobProblemProvider>
-            <h3>Troubleshooting</h3>
-            <p>
-                There are a number of helpful resources to self diagnose and correct problems.
-                <br />
-                Start here:
-                <b>
-                    <a href="https://galaxyproject.org/support/tool-error/" target="_blank">
-                        My job ended with an error. What can I do?
-                    </a>
-                </b>
-            </p>
-            <h3>Issue Report</h3>
-        </div>
-    </DatasetProvider>
+                </JobProblemProvider>
+                <h3>Troubleshooting</h3>
+                <p>
+                    There are a number of helpful resources to self diagnose and correct problems.
+                    <br />
+                    Start here:
+                    <b>
+                        <a href="https://galaxyproject.org/support/tool-error/" target="_blank">
+                            My job ended with an error. What can I do?
+                        </a>
+                    </b>
+                </p>
+                <h3 class="mb-2">Issue Report</h3>
+                <b-alert v-if="errorMessage" variant="danger" show>{{ errorMessage }}</b-alert>
+                <b-alert v-for="resultMessage in resultMessages" :variant="resultMessage[1]" show>{{
+                    resultMessage[0]
+                }}</b-alert>
+                <FormElement v-if="!user.email" id="email" v-model="email" title="Please provide your email:" />
+                <FormElement
+                    id="message"
+                    v-model="message"
+                    :area="true"
+                    title="Please provide detailed information on the activities leading to this issue:"
+                />
+                <b-button variant="primary" @click="submit(dataset, user.email)" class="mt-2">
+                    <font-awesome-icon icon="bug" class="mr-1" />Report
+                </b-button>
+            </div>
+        </DatasetProvider>
+    </CurrentUser>
 </template>
 
 <script>
+import CurrentUser from "components/providers/CurrentUser";
+import FormElement from "components/Form/FormElement";
 import { DatasetProvider } from "components/WorkflowInvocationState/providers";
 import { JobDetailsProvider, JobProblemProvider } from "components/providers/JobProvider";
-import FormDisplay from "components/Form/FormDisplay";
-import FormCard from "components/Form/FormCard";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faBug } from "@fortawesome/free-solid-svg-icons";
+import { sendErrorReport } from "./services";
+
+library.add(faBug);
 
 export default {
     components: {
+        CurrentUser,
         DatasetProvider,
-        FormDisplay,
-        FormCard,
+        FontAwesomeIcon,
+        FormElement,
         JobDetailsProvider,
         JobProblemProvider,
     },
@@ -80,15 +103,26 @@ export default {
         },
     },
     data() {
-        return {};
-    },
-    computed: {
-        inputs() {
-            return [];
-        },
+        return {
+            message: null,
+            email: null,
+            errorMessage: null,
+            resultMessages: [],
+        };
     },
     methods: {
-        onChange(data) {},
+        submit(dataset, userEmail) {
+            const email = this.email || userEmail;
+            const message = this.message;
+            sendErrorReport(dataset, message, email).then(
+                (resultMessages) => {
+                    this.resultMessages = resultMessages;
+                },
+                (errorMessage) => {
+                    this.errorMessage = errorMessage;
+                }
+            );
+        },
     },
 };
 </script>
diff --git a/client/src/mvc/dataset/dataset-error.js b/client/src/mvc/dataset/dataset-error.js
deleted file mode 100644
index 2312a55e1b9..00000000000
--- a/client/src/mvc/dataset/dataset-error.js
+++ /dev/null
@@ -1,244 +0,0 @@
-import _ from "underscore";
-import $ from "jquery";
-import Backbone from "backbone";
-import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
-import _l from "utils/localization";
-import Utils from "utils/utils";
-import Ui from "mvc/ui/ui-misc";
-import Form from "mvc/form/form-view";
-
-/** Dataset edit attributes view */
-var View = Backbone.View.extend({
-    initialize: function () {
-        const Galaxy = getGalaxyInstance();
-        this.setElement("<div/>");
-        this.model = new Backbone.Model({
-            dataset_id: Galaxy.params.dataset_id,
-        });
-        this.active_tab = "user";
-        this.render();
-    },
-
-    // Fetch data for the selected dataset and
-    render: function () {
-        var data_url = `${getAppRoot()}api/datasets/${this.model.get("dataset_id")}`;
-        Utils.get({
-            url: data_url,
-            success: (dataset) => {
-                var job_url = `${getAppRoot()}api/jobs/${dataset.creating_job}?full=True`;
-                Utils.get({
-                    url: job_url,
-                    success: (job) => {
-                        this.render_error_page(dataset, job);
-                        this.find_common_problems(job);
-                    },
-                    error: () => {
-                        var error_response = {
-                            status: "error",
-                            message: "Error occurred while loading the job.",
-                            persistent: true,
-                            cls: "errormessage",
-                        };
-                        this.display_message(error_response, this.$(".response-message"));
-                    },
-                });
-            },
-            error: () => {
-                var error_response = {
-                    status: "error",
-                    message: "Error occurred while loading the dataset.",
-                    persistent: true,
-                    cls: "errormessage",
-                };
-                this.display_message(error_response, this.$(".response-message"));
-            },
-        });
-    },
-
-    find_common_problems: function (job) {
-        var job_url = `${getAppRoot()}api/jobs/${job.id}/common_problems`;
-        Utils.get({
-            url: job_url,
-            success: (common_problems) => {
-                this.render_common_problems(common_problems);
-            },
-            error: (response) => {
-                console.log("error");
-                console.log(response);
-            },
-        });
-        return;
-    },
-
-    render_common_problems: function (common_problems) {
-        const has_duplicate_inputs = common_problems.has_duplicate_inputs;
-        const has_empty_inputs = common_problems.has_empty_inputs;
-        if (has_duplicate_inputs || has_empty_inputs) {
-            const reportEl = this.$el.find(".common_problems");
-            reportEl.text("Detected Common Potential Problems");
-            if (has_empty_inputs) {
-                reportEl.after(`
-                    <p>
-                        The tool was executed with one or more empty input datasets. This frequently
-                        results in tool errors due to problematic input choices.
-                    <p>`);
-            }
-            if (has_duplicate_inputs) {
-                reportEl.after(`
-                    <p>
-                        The tool was executed with one or more duplicate input datasets. This frequently
-                        results in tool errors due to problematic input choices.
-                    <p>`);
-            }
-        }
-    },
-
-    /** Render the view */
-    render_error_page: function (dataset, job) {
-        this.$el.empty().append(`
-            ${this._templateHeader()}
-            <h2>Dataset Error Report</h2>
-            <p>An error occurred while running the tool <b>${job.tool_id}</b>.</p>
-            ${this.job_summary(job)}
-            <h3>Troubleshooting</h3>
-            <p>
-                There are a number of helpful resources to self diagnose and
-                correct problems.
-                <br>
-                Start here:
-                <b>
-                    <a href="https://galaxyproject.org/support/tool-error/" target="_blank">
-                        My job ended with an error. What can I do?
-                    </a>
-                </b>
-            </p>`);
-        this.$el.append("<h3>Issue Report</h3>").append(this._getBugFormTemplate(dataset, job));
-    },
-
-    job_summary: function (job) {
-        const tool_stderr = job.tool_stderr;
-        const job_stderr = job.job_stderr;
-        const job_messages = job.job_messages;
-        if (!tool_stderr && !job_stderr && !job_messages) {
-            return '<h3 class="common_problems"></h3>';
-        }
-        var message = "<h3>Details</h3>";
-        if (job_messages) {
-            message += "<p>Execution resulted in the following messages:</p>";
-            for (const job_message of job_messages) {
-                message += `<p><pre class="rounded code">${_.escape(job_message["desc"])}</pre></p>`;
-            }
-        }
-        if (tool_stderr) {
-            message += "<p>Tool generated the following standard error:</p>";
-            message += `<pre class="rounded code">${_.escape(tool_stderr)}</pre>`;
-        }
-        if (job_stderr) {
-            message += "<p>Galaxy job runner generated the following standard error:</p>";
-            message += `<pre class="code">${_.escape(job_stderr)}</pre>`;
-        }
-        message += `<h3 class="common_problems"></h3>`;
-        return message;
-    },
-
-    /** Display actions messages */
-    display_message: function (response, $el, doNotClear, safe) {
-        if (!safe) {
-            if (doNotClear) {
-                $el.append(new Ui.Message(response).$el);
-            } else {
-                $el.empty().html(new Ui.Message(response).$el);
-            }
-        } else {
-            if (doNotClear) {
-                $el.append(new Ui.UnescapedMessage(response).$el);
-            } else {
-                $el.empty().html(new Ui.UnescapedMessage(response).$el);
-            }
-        }
-    },
-
-    /** Main template */
-    _templateHeader: function () {
-        return `<div class="page-container edit-attr"><div class="response-message"></div></div>`;
-    },
-
-    /** Convert tab template */
-    _getBugFormTemplate: function (dataset, job) {
-        const Galaxy = getGalaxyInstance();
-        const userEmail = Galaxy.user.get("email");
-        const form = new Form({
-            inputs: [
-                {
-                    type: "text",
-                    hidden: !!userEmail,
-                    name: "email",
-                    value: userEmail,
-                    label: "Please provide your email:",
-                },
-                {
-                    type: "text",
-                    area: true,
-                    name: "message",
-                    label: "Please provide detailed information on the activities leading to this issue:",
-                },
-            ],
-            buttons: {
-                save: new Ui.Button({
-                    icon: "fa-bug",
-                    title: _l("Report"),
-                    cls: "btn btn-primary",
-                    floating: "clear",
-                    onclick: () => {
-                        var form_data = form.data.create();
-                        var url = `${getAppRoot()}api/jobs/${job.id}/error`;
-                        form_data.dataset_id = dataset.id;
-                        this.submit(form_data, url);
-                    },
-                }),
-            },
-        });
-        return form.$el;
-    },
-
-    /** Make ajax request */
-    submit: function (form_data, url) {
-        // Some required metadata
-        $.ajax({
-            type: "POST",
-            url: url,
-            data: form_data,
-            success: (response) => {
-                // Clear out the div
-                this.$el.empty().append(this._templateHeader());
-                // And display the messages.
-                response.messages.forEach((message) => {
-                    this.display_message(
-                        {
-                            status: message[1],
-                            message: message[0],
-                            persistent: true,
-                        },
-                        this.$(".response-message"),
-                        true,
-                        true
-                    );
-                });
-            },
-            error: () => {
-                var error_response = {
-                    status: "error",
-                    message: "Error occurred while saving. Please fill all the required fields and try again.",
-                    persistent: true,
-                    cls: "errormessage",
-                };
-                this.display_message(error_response, this.$(".response-message"));
-            },
-        });
-    },
-});
-
-export default {
-    View: View,
-};

From 982464d8efee908f8b7b492cbcf597362379f442 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 27 Jul 2021 18:06:09 -0400
Subject: [PATCH 262/452] Align styling to previous dataset error reporting
 page

---
 client/src/components/DatasetInformation/DatasetError.vue | 4 ++--
 client/src/components/Form/Elements/FormInput.vue         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 56109288092..e46e5910cc0 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -55,7 +55,7 @@
                         </a>
                     </b>
                 </p>
-                <h3 class="mb-2">Issue Report</h3>
+                <h3 class="mb-3">Issue Report</h3>
                 <b-alert v-if="errorMessage" variant="danger" show>{{ errorMessage }}</b-alert>
                 <b-alert v-for="resultMessage in resultMessages" :variant="resultMessage[1]" show>{{
                     resultMessage[0]
@@ -67,7 +67,7 @@
                     :area="true"
                     title="Please provide detailed information on the activities leading to this issue:"
                 />
-                <b-button variant="primary" @click="submit(dataset, user.email)" class="mt-2">
+                <b-button variant="primary" @click="submit(dataset, user.email)" class="mt-3">
                     <font-awesome-icon icon="bug" class="mr-1" />Report
                 </b-button>
             </div>
diff --git a/client/src/components/Form/Elements/FormInput.vue b/client/src/components/Form/Elements/FormInput.vue
index 05758a19b28..4bf6c1d70e9 100644
--- a/client/src/components/Form/Elements/FormInput.vue
+++ b/client/src/components/Form/Elements/FormInput.vue
@@ -1,5 +1,5 @@
 <template>
-    <textarea v-if="area" class="ui-input" v-model="currentValue" />
+    <textarea v-if="area" class="ui-textarea" v-model="currentValue"/>
     <input v-else class="ui-input" v-model="currentValue" />
 </template>
 <script>

From f6cf42d58cc632957a3cc8f6fcc2aad2b36a1773 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 27 Jul 2021 23:52:03 -0400
Subject: [PATCH 263/452] Use email returned from job details instead using the
 user provider, fix linting issues

---
 .../DatasetInformation/DatasetError.vue       | 76 +++++++++----------
 .../components/Form/Elements/FormInput.vue    |  2 +-
 2 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index e46e5910cc0..e4196333f3f 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -1,37 +1,33 @@
 <template>
-    <CurrentUser v-slot="{ user }">
-        <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading }">
+    <DatasetProvider :id="datasetId" v-slot="{ item: dataset }">
+        <JobDetailsProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
             <div v-if="!loading">
-                <JobDetailsProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
-                    <div v-if="!loading">
-                        <div class="page-container edit-attr">
-                            <div class="response-message"></div>
-                        </div>
-                        <h2>Dataset Error Report</h2>
-                        <p>
-                            An error occurred while running the tool <b>{{ result.tool_id }}</b
-                            >.
-                        </p>
-                        <div v-if="result.tool_stderr || result.job_stderr || result.job_messages">
-                            <h3>Details</h3>
-                        </div>
-                        <div v-if="result.job_messages">
-                            <p>Execution resulted in the following messages:</p>
-                            <div v-for="job_message in result.job_messages">
-                                <pre class="rounded code">{{ job_message["desc"] }}</pre>
-                            </div>
-                        </div>
-                        <div v-if="result.tool_stderr">
-                            <p>Tool generated the following standard error:</p>
-                            <pre class="rounded code">{{ result.tool_stderr }}</pre>
-                        </div>
-                        <div v-if="result.job_stderr">
-                            <p>Galaxy job runner generated the following standard error:</p>
-                            <pre class="rounded code">{{ result.job_stderr }}</pre>
-                        </div>
+                <div class="page-container edit-attr">
+                    <div class="response-message"></div>
+                </div>
+                <h2>Dataset Error Report</h2>
+                <p>
+                    An error occurred while running the tool <b>{{ result.tool_id }}</b
+                    >.
+                </p>
+                <div v-if="result.tool_stderr || result.job_stderr || result.job_messages">
+                    <h3>Details</h3>
+                </div>
+                <div v-if="result.job_messages">
+                    <p>Execution resulted in the following messages:</p>
+                    <div v-for="(job_message, index) in result.job_messages" :key="index">
+                        <pre class="rounded code">{{ job_message["desc"] }}</pre>
                     </div>
-                </JobDetailsProvider>
-                <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
+                </div>
+                <div v-if="result.tool_stderr">
+                    <p>Tool generated the following standard error:</p>
+                    <pre class="rounded code">{{ result.tool_stderr }}</pre>
+                </div>
+                <div v-if="result.job_stderr">
+                    <p>Galaxy job runner generated the following standard error:</p>
+                    <pre class="rounded code">{{ result.job_stderr }}</pre>
+                </div>
+                <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result }">
                     <div v-if="result && (result.has_duplicate_inputs || result.has_empty_inputs)">
                         <h3 class="common_problems">Detected Common Potential Problems</h3>
                         <p v-if="result.has_empty_inputs">
@@ -57,22 +53,26 @@
                 </p>
                 <h3 class="mb-3">Issue Report</h3>
                 <b-alert v-if="errorMessage" variant="danger" show>{{ errorMessage }}</b-alert>
-                <b-alert v-for="resultMessage in resultMessages" :variant="resultMessage[1]" show>{{
-                    resultMessage[0]
-                }}</b-alert>
-                <FormElement v-if="!user.email" id="email" v-model="email" title="Please provide your email:" />
+                <b-alert
+                    v-for="(resultMessage, index) in resultMessages"
+                    :key="index"
+                    :variant="resultMessage[1]"
+                    show
+                    >{{ resultMessage[0] }}</b-alert
+                >
+                <FormElement v-if="!result.user_email" id="email" v-model="email" title="Please provide your email:" />
                 <FormElement
                     id="message"
                     v-model="message"
                     :area="true"
                     title="Please provide detailed information on the activities leading to this issue:"
                 />
-                <b-button variant="primary" @click="submit(dataset, user.email)" class="mt-3">
+                <b-button variant="primary" @click="submit(dataset, result.user_email)" class="mt-3">
                     <font-awesome-icon icon="bug" class="mr-1" />Report
                 </b-button>
             </div>
-        </DatasetProvider>
-    </CurrentUser>
+        </JobDetailsProvider>
+    </DatasetProvider>
 </template>
 
 <script>
diff --git a/client/src/components/Form/Elements/FormInput.vue b/client/src/components/Form/Elements/FormInput.vue
index 4bf6c1d70e9..c667572e13c 100644
--- a/client/src/components/Form/Elements/FormInput.vue
+++ b/client/src/components/Form/Elements/FormInput.vue
@@ -1,5 +1,5 @@
 <template>
-    <textarea v-if="area" class="ui-textarea" v-model="currentValue"/>
+    <textarea v-if="area" class="ui-textarea" v-model="currentValue" />
     <input v-else class="ui-input" v-model="currentValue" />
 </template>
 <script>

From 629aae1008717ad1dc4eb82b29522d977b8e6c86 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 10:20:11 -0400
Subject: [PATCH 264/452] Use proper provider key names in error display
 component, add services to submit error report

---
 .../DatasetInformation/DatasetError.vue       | 41 +++++++++++--------
 .../components/DatasetInformation/services.js | 18 ++++++++
 2 files changed, 43 insertions(+), 16 deletions(-)
 create mode 100644 client/src/components/DatasetInformation/services.js

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index e4196333f3f..aa8b346ea36 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -1,40 +1,44 @@
 <template>
-    <DatasetProvider :id="datasetId" v-slot="{ item: dataset }">
-        <JobDetailsProvider :jobid="dataset.creating_job" v-slot="{ result, loading }">
+    <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading: datasetLoading }">
+        <JobDetailsProvider
+            v-if="!datasetLoading"
+            :jobid="dataset.creating_job"
+            v-slot="{ result: jobDetails, loading }"
+        >
             <div v-if="!loading">
                 <div class="page-container edit-attr">
                     <div class="response-message"></div>
                 </div>
                 <h2>Dataset Error Report</h2>
                 <p>
-                    An error occurred while running the tool <b>{{ result.tool_id }}</b
+                    An error occurred while running the tool <b>{{ jobDetails.tool_id }}</b
                     >.
                 </p>
-                <div v-if="result.tool_stderr || result.job_stderr || result.job_messages">
+                <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">
                     <h3>Details</h3>
                 </div>
-                <div v-if="result.job_messages">
+                <div v-if="jobDetails.job_messages">
                     <p>Execution resulted in the following messages:</p>
-                    <div v-for="(job_message, index) in result.job_messages" :key="index">
+                    <div v-for="(job_message, index) in jobDetails.job_messages" :key="index">
                         <pre class="rounded code">{{ job_message["desc"] }}</pre>
                     </div>
                 </div>
-                <div v-if="result.tool_stderr">
+                <div v-if="jobDetails.tool_stderr">
                     <p>Tool generated the following standard error:</p>
-                    <pre class="rounded code">{{ result.tool_stderr }}</pre>
+                    <pre class="rounded code">{{ jobDetails.tool_stderr }}</pre>
                 </div>
-                <div v-if="result.job_stderr">
+                <div v-if="jobDetails.job_stderr">
                     <p>Galaxy job runner generated the following standard error:</p>
-                    <pre class="rounded code">{{ result.job_stderr }}</pre>
+                    <pre class="rounded code">{{ jobDetails.job_stderr }}</pre>
                 </div>
-                <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result }">
-                    <div v-if="result && (result.has_duplicate_inputs || result.has_empty_inputs)">
+                <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result: jobProblems }">
+                    <div v-if="jobProblems && (jobProblems.has_duplicate_inputs || jobProblems.has_empty_inputs)">
                         <h3 class="common_problems">Detected Common Potential Problems</h3>
-                        <p v-if="result.has_empty_inputs">
+                        <p v-if="jobProblems.has_empty_inputs">
                             The tool was executed with one or more empty input datasets. This frequently results in tool
                             errors due to problematic input choices.
                         </p>
-                        <p v-if="result && result.has_duplicate_inputs">
+                        <p v-if="jobProblems.has_duplicate_inputs">
                             The tool was executed with one or more duplicate input datasets. This frequently results in
                             tool errors due to problematic input choices.
                         </p>
@@ -60,14 +64,19 @@
                     show
                     >{{ resultMessage[0] }}</b-alert
                 >
-                <FormElement v-if="!result.user_email" id="email" v-model="email" title="Please provide your email:" />
+                <FormElement
+                    v-if="!jobDetails.user_email"
+                    id="email"
+                    v-model="email"
+                    title="Please provide your email:"
+                />
                 <FormElement
                     id="message"
                     v-model="message"
                     :area="true"
                     title="Please provide detailed information on the activities leading to this issue:"
                 />
-                <b-button variant="primary" @click="submit(dataset, result.user_email)" class="mt-3">
+                <b-button variant="primary" @click="submit(dataset, jobDetails.user_email)" class="mt-3">
                     <font-awesome-icon icon="bug" class="mr-1" />Report
                 </b-button>
             </div>
diff --git a/client/src/components/DatasetInformation/services.js b/client/src/components/DatasetInformation/services.js
new file mode 100644
index 00000000000..7739f7fed83
--- /dev/null
+++ b/client/src/components/DatasetInformation/services.js
@@ -0,0 +1,18 @@
+import axios from "axios";
+import { rethrowSimple } from "utils/simple-error";
+import { getAppRoot } from "onload/loadConfig";
+
+export async function sendErrorReport(dataset, message, email) {
+    const payload = {
+        dataset_id: dataset.id,
+        message,
+        email,
+    };
+    const url = `${getAppRoot()}api/jobs/${dataset.creating_job}/error`;
+    try {
+        const { data } = await axios.post(url, payload);
+        return data.messages;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}

From 7e09bf7d85810f1bdb81a8c95594a747768c5853 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 28 Jul 2021 11:01:06 -0400
Subject: [PATCH 265/452] Remove unused import from dataset error component

---
 client/src/components/DatasetInformation/DatasetError.vue | 2 --
 1 file changed, 2 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index aa8b346ea36..6c09f3856c9 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -85,7 +85,6 @@
 </template>
 
 <script>
-import CurrentUser from "components/providers/CurrentUser";
 import FormElement from "components/Form/FormElement";
 import { DatasetProvider } from "components/WorkflowInvocationState/providers";
 import { JobDetailsProvider, JobProblemProvider } from "components/providers/JobProvider";
@@ -98,7 +97,6 @@ library.add(faBug);
 
 export default {
     components: {
-        CurrentUser,
         DatasetProvider,
         FontAwesomeIcon,
         FormElement,

From 00644733374e71e496a7120d5e98c109c22a303d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 2 Aug 2021 11:19:57 -0400
Subject: [PATCH 266/452] Slightly increase margins in error page sections

---
 client/src/components/DatasetInformation/DatasetError.vue | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 6c09f3856c9..8802b74f70d 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -33,7 +33,7 @@
                 </div>
                 <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result: jobProblems }">
                     <div v-if="jobProblems && (jobProblems.has_duplicate_inputs || jobProblems.has_empty_inputs)">
-                        <h3 class="common_problems">Detected Common Potential Problems</h3>
+                        <h3 class="common_problems mt-3">Detected Common Potential Problems</h3>
                         <p v-if="jobProblems.has_empty_inputs">
                             The tool was executed with one or more empty input datasets. This frequently results in tool
                             errors due to problematic input choices.
@@ -44,7 +44,7 @@
                         </p>
                     </div>
                 </JobProblemProvider>
-                <h3>Troubleshooting</h3>
+                <h3 class="mt-3">Troubleshooting</h3>
                 <p>
                     There are a number of helpful resources to self diagnose and correct problems.
                     <br />

From 6451f7ff89817f82f8d6e6daf19a25d5f2fa6f67 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 2 Aug 2021 12:05:30 -0400
Subject: [PATCH 267/452] Add error handling to single query provider

---
 .../DatasetInformation/DatasetError.vue       | 171 +++++++++---------
 .../src/components/providers/JobProvider.js   |  17 +-
 .../providers/SingleQueryProvider.js          |   4 +-
 3 files changed, 106 insertions(+), 86 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 8802b74f70d..b3be71720b2 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -1,87 +1,93 @@
 <template>
-    <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading: datasetLoading }">
-        <JobDetailsProvider
-            v-if="!datasetLoading"
-            :jobid="dataset.creating_job"
-            v-slot="{ result: jobDetails, loading }"
-        >
-            <div v-if="!loading">
-                <div class="page-container edit-attr">
-                    <div class="response-message"></div>
-                </div>
-                <h2>Dataset Error Report</h2>
-                <p>
-                    An error occurred while running the tool <b>{{ jobDetails.tool_id }}</b
-                    >.
-                </p>
-                <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">
-                    <h3>Details</h3>
-                </div>
-                <div v-if="jobDetails.job_messages">
-                    <p>Execution resulted in the following messages:</p>
-                    <div v-for="(job_message, index) in jobDetails.job_messages" :key="index">
-                        <pre class="rounded code">{{ job_message["desc"] }}</pre>
+    <div>
+        <b-alert v-if="errorMessage" variant="danger" show>
+            <h4 class="alert-heading">Failed to access Dataset details.</h4>
+            {{ errorMessage }}
+        </b-alert>
+        <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading: datasetLoading }">
+            <JobDetailsProvider
+                v-if="!datasetLoading"
+                :jobid="dataset.creating_job"
+                v-slot="{ result: jobDetails, loading }"
+                @error="onError"
+            >
+                <div v-if="!loading">
+                    <div class="page-container edit-attr">
+                        <div class="response-message"></div>
                     </div>
-                </div>
-                <div v-if="jobDetails.tool_stderr">
-                    <p>Tool generated the following standard error:</p>
-                    <pre class="rounded code">{{ jobDetails.tool_stderr }}</pre>
-                </div>
-                <div v-if="jobDetails.job_stderr">
-                    <p>Galaxy job runner generated the following standard error:</p>
-                    <pre class="rounded code">{{ jobDetails.job_stderr }}</pre>
-                </div>
-                <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result: jobProblems }">
-                    <div v-if="jobProblems && (jobProblems.has_duplicate_inputs || jobProblems.has_empty_inputs)">
-                        <h3 class="common_problems mt-3">Detected Common Potential Problems</h3>
-                        <p v-if="jobProblems.has_empty_inputs">
-                            The tool was executed with one or more empty input datasets. This frequently results in tool
-                            errors due to problematic input choices.
-                        </p>
-                        <p v-if="jobProblems.has_duplicate_inputs">
-                            The tool was executed with one or more duplicate input datasets. This frequently results in
-                            tool errors due to problematic input choices.
-                        </p>
+                    <h2>Dataset Error Report</h2>
+                    <p>
+                        An error occurred while running the tool <b>{{ jobDetails.tool_id }}</b
+                        >.
+                    </p>
+                    <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">
+                        <h3>Details</h3>
                     </div>
-                </JobProblemProvider>
-                <h3 class="mt-3">Troubleshooting</h3>
-                <p>
-                    There are a number of helpful resources to self diagnose and correct problems.
-                    <br />
-                    Start here:
-                    <b>
-                        <a href="https://galaxyproject.org/support/tool-error/" target="_blank">
-                            My job ended with an error. What can I do?
-                        </a>
-                    </b>
-                </p>
-                <h3 class="mb-3">Issue Report</h3>
-                <b-alert v-if="errorMessage" variant="danger" show>{{ errorMessage }}</b-alert>
-                <b-alert
-                    v-for="(resultMessage, index) in resultMessages"
-                    :key="index"
-                    :variant="resultMessage[1]"
-                    show
-                    >{{ resultMessage[0] }}</b-alert
-                >
-                <FormElement
-                    v-if="!jobDetails.user_email"
-                    id="email"
-                    v-model="email"
-                    title="Please provide your email:"
-                />
-                <FormElement
-                    id="message"
-                    v-model="message"
-                    :area="true"
-                    title="Please provide detailed information on the activities leading to this issue:"
-                />
-                <b-button variant="primary" @click="submit(dataset, jobDetails.user_email)" class="mt-3">
-                    <font-awesome-icon icon="bug" class="mr-1" />Report
-                </b-button>
-            </div>
-        </JobDetailsProvider>
-    </DatasetProvider>
+                    <div v-if="jobDetails.job_messages">
+                        <p>Execution resulted in the following messages:</p>
+                        <div v-for="(job_message, index) in jobDetails.job_messages" :key="index">
+                            <pre class="rounded code">{{ job_message["desc"] }}</pre>
+                        </div>
+                    </div>
+                    <div v-if="jobDetails.tool_stderr">
+                        <p>Tool generated the following standard error:</p>
+                        <pre class="rounded code">{{ jobDetails.tool_stderr }}</pre>
+                    </div>
+                    <div v-if="jobDetails.job_stderr">
+                        <p>Galaxy job runner generated the following standard error:</p>
+                        <pre class="rounded code">{{ jobDetails.job_stderr }}</pre>
+                    </div>
+                    <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result: jobProblems }" @error="onError">
+                        <div v-if="jobProblems && (jobProblems.has_duplicate_inputs || jobProblems.has_empty_inputs)">
+                            <h3 class="common_problems mt-3">Detected Common Potential Problems</h3>
+                            <p v-if="jobProblems.has_empty_inputs">
+                                The tool was executed with one or more empty input datasets. This frequently results in
+                                tool errors due to problematic input choices.
+                            </p>
+                            <p v-if="jobProblems.has_duplicate_inputs">
+                                The tool was executed with one or more duplicate input datasets. This frequently results
+                                in tool errors due to problematic input choices.
+                            </p>
+                        </div>
+                    </JobProblemProvider>
+                    <h3 class="mt-3">Troubleshooting</h3>
+                    <p>
+                        There are a number of helpful resources to self diagnose and correct problems.
+                        <br />
+                        Start here:
+                        <b>
+                            <a href="https://galaxyproject.org/support/tool-error/" target="_blank">
+                                My job ended with an error. What can I do?
+                            </a>
+                        </b>
+                    </p>
+                    <h3 class="mb-3">Issue Report</h3>
+                    <b-alert
+                        v-for="(resultMessage, index) in resultMessages"
+                        :key="index"
+                        :variant="resultMessage[1]"
+                        show
+                        >{{ resultMessage[0] }}</b-alert
+                    >
+                    <FormElement
+                        v-if="!jobDetails.user_email"
+                        id="email"
+                        v-model="email"
+                        title="Please provide your email:"
+                    />
+                    <FormElement
+                        id="message"
+                        v-model="message"
+                        :area="true"
+                        title="Please provide detailed information on the activities leading to this issue:"
+                    />
+                    <b-button variant="primary" @click="submit(dataset, jobDetails.user_email)" class="mt-3">
+                        <font-awesome-icon icon="bug" class="mr-1" />Report
+                    </b-button>
+                </div>
+            </JobDetailsProvider>
+        </DatasetProvider>
+    </div>
 </template>
 
 <script>
@@ -118,6 +124,9 @@ export default {
         };
     },
     methods: {
+        onError(err) {
+            this.errorMessage = err;
+        },
         submit(dataset, userEmail) {
             const email = this.email || userEmail;
             const message = this.message;
diff --git a/client/src/components/providers/JobProvider.js b/client/src/components/providers/JobProvider.js
index 913af215cc4..484432e3415 100644
--- a/client/src/components/providers/JobProvider.js
+++ b/client/src/components/providers/JobProvider.js
@@ -1,17 +1,26 @@
 import axios from "axios";
 import { getAppRoot } from "onload/loadConfig";
 import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
+import { rethrowSimple } from "utils/simple-error";
 
 async function jobDetails({ jobid }) {
     const url = `${getAppRoot()}api/jobs/${jobid}?full=True`;
-    const { data } = await axios.get(url);
-    return data;
+    try {
+        const { data } = await axios.get(url);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
 }
 
 async function jobProblems({ jobid }) {
     const url = `${getAppRoot()}api/jobs/${jobid}/common_problems`;
-    const { data } = await axios.get(url);
-    return data;
+    try {
+        const { data } = await axios.get(url);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
 }
 
 export const JobDetailsProvider = SingleQueryProvider(jobDetails);
diff --git a/client/src/components/providers/SingleQueryProvider.js b/client/src/components/providers/SingleQueryProvider.js
index 5327ac4e506..dff317ba5d4 100644
--- a/client/src/components/providers/SingleQueryProvider.js
+++ b/client/src/components/providers/SingleQueryProvider.js
@@ -35,7 +35,9 @@ export const SingleQueryProvider = (lookup) => {
 
             let lookupPromise = promiseCache.get(this.cacheKey);
             if (!lookupPromise) {
-                lookupPromise = lookup(this.$attrs);
+                lookupPromise = lookup(this.$attrs).catch((err) => {
+                    this.$emit("error", err);
+                });
                 promiseCache.set(this.cacheKey, lookupPromise);
             }
 

From 421427af2650ebdca9c7b70312b05ddb260b4664 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 3 Aug 2021 16:48:32 -0400
Subject: [PATCH 268/452] Add DatasetError test case

---
 .../DatasetInformation/DatasetError.test.js   | 40 +++++++++++++++++++
 .../DatasetInformation/DatasetError.vue       |  5 ++-
 .../src/components/providers/MockProvider.js  | 11 +++++
 3 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 client/src/components/DatasetInformation/DatasetError.test.js
 create mode 100644 client/src/components/providers/MockProvider.js

diff --git a/client/src/components/DatasetInformation/DatasetError.test.js b/client/src/components/DatasetInformation/DatasetError.test.js
new file mode 100644
index 00000000000..b1af6fb2da9
--- /dev/null
+++ b/client/src/components/DatasetInformation/DatasetError.test.js
@@ -0,0 +1,40 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import DatasetError from "./DatasetError";
+import MockProvider from "../providers/MockProvider";
+
+jest.mock("components/WorkflowInvocationState/providers", () => {
+    return {}; // stubbed below
+});
+
+const localVue = getLocalVue();
+
+describe("DatasetError", () => {
+    let wrapper;
+    
+    beforeEach(() => {
+        wrapper = mount(DatasetError, {
+            propsData: {
+                datasetId: "dataset_id",
+            },
+            localVue,
+            stubs: {
+                JobDetailsProvider: MockProvider({
+                    result: { tool_id: "tool_id", tool_stderr: "tool_stderr", job_stderr: "job_stderr" },
+                }),
+                JobProblemProvider: MockProvider({ result: { has_duplicate_inputs: true, has_empty_inputs: true } }),
+                DatasetProvider: MockProvider({
+                    resultLabel: "item",
+                    result: { id: "dataset_id", creating_job: "creating_job" },
+                }),
+                FontAwesomeIcon: false,
+                FormElement: false,
+            },
+        });
+    });
+
+    it("check props", async () => {
+        expect(wrapper.find("#dataset-error-tool-id").text()).toBe("tool_id");
+        expect(wrapper.find("#dataset-error-tool-stderr").text()).toBe("tool_stderr");
+    });
+});
diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index b3be71720b2..616ede5624f 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -17,7 +17,8 @@
                     </div>
                     <h2>Dataset Error Report</h2>
                     <p>
-                        An error occurred while running the tool <b>{{ jobDetails.tool_id }}</b
+                        An error occurred while running the tool
+                        <b id="dataset-error-tool-id">{{ jobDetails.tool_id }}</b
                         >.
                     </p>
                     <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">
@@ -31,7 +32,7 @@
                     </div>
                     <div v-if="jobDetails.tool_stderr">
                         <p>Tool generated the following standard error:</p>
-                        <pre class="rounded code">{{ jobDetails.tool_stderr }}</pre>
+                        <pre id="dataset-error-tool-stderr" class="rounded code">{{ jobDetails.tool_stderr }}</pre>
                     </div>
                     <div v-if="jobDetails.job_stderr">
                         <p>Galaxy job runner generated the following standard error:</p>
diff --git a/client/src/components/providers/MockProvider.js b/client/src/components/providers/MockProvider.js
new file mode 100644
index 00000000000..c35e49d92e5
--- /dev/null
+++ b/client/src/components/providers/MockProvider.js
@@ -0,0 +1,11 @@
+const MockProvider = (fakeData) => ({
+    render() {
+        const resultLabel = fakeData.resultLabel || "result";
+        return this.$scopedSlots.default({
+            [resultLabel]: fakeData.result,
+            loaded: true,
+        });
+    },
+});
+
+export default MockProvider;

From eb1439cae3547833deda536f9a2bfc956830c9bf Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 3 Aug 2021 17:18:17 -0400
Subject: [PATCH 269/452] Expand dataset error rendering test case

---
 .../DatasetInformation/DatasetError.test.js   | 67 ++++++++++++-------
 .../DatasetInformation/DatasetError.vue       |  8 +--
 2 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.test.js b/client/src/components/DatasetInformation/DatasetError.test.js
index b1af6fb2da9..c50488aac85 100644
--- a/client/src/components/DatasetInformation/DatasetError.test.js
+++ b/client/src/components/DatasetInformation/DatasetError.test.js
@@ -9,32 +9,53 @@ jest.mock("components/WorkflowInvocationState/providers", () => {
 
 const localVue = getLocalVue();
 
-describe("DatasetError", () => {
-    let wrapper;
-    
-    beforeEach(() => {
-        wrapper = mount(DatasetError, {
-            propsData: {
-                datasetId: "dataset_id",
-            },
-            localVue,
-            stubs: {
-                JobDetailsProvider: MockProvider({
-                    result: { tool_id: "tool_id", tool_stderr: "tool_stderr", job_stderr: "job_stderr" },
-                }),
-                JobProblemProvider: MockProvider({ result: { has_duplicate_inputs: true, has_empty_inputs: true } }),
-                DatasetProvider: MockProvider({
-                    resultLabel: "item",
-                    result: { id: "dataset_id", creating_job: "creating_job" },
-                }),
-                FontAwesomeIcon: false,
-                FormElement: false,
-            },
-        });
+function buildWrapper(has_duplicate_inputs = true, has_empty_inputs = true) {
+    return mount(DatasetError, {
+        propsData: {
+            datasetId: "dataset_id",
+        },
+        localVue,
+        stubs: {
+            JobDetailsProvider: MockProvider({
+                result: {
+                    tool_id: "tool_id",
+                    tool_stderr: "tool_stderr",
+                    job_stderr: "job_stderr",
+                    job_messages: [{ desc: "message_1" }, { desc: "message_2" }],
+                },
+            }),
+            JobProblemProvider: MockProvider({
+                result: { has_duplicate_inputs: has_duplicate_inputs, has_empty_inputs: has_empty_inputs },
+            }),
+            DatasetProvider: MockProvider({
+                resultLabel: "item",
+                result: { id: "dataset_id", creating_job: "creating_job" },
+            }),
+            FontAwesomeIcon: false,
+            FormElement: false,
+        },
     });
+}
 
-    it("check props", async () => {
+describe("DatasetError", () => {
+    it("check props with common problems", async () => {
+        const wrapper = buildWrapper();
         expect(wrapper.find("#dataset-error-tool-id").text()).toBe("tool_id");
         expect(wrapper.find("#dataset-error-tool-stderr").text()).toBe("tool_stderr");
+        expect(wrapper.find("#dataset-error-job-stderr").text()).toBe("job_stderr");
+        const messages = wrapper.findAll("#dataset-error-job-messages .code");
+        expect(messages.at(0).text()).toBe("message_1");
+        expect(messages.at(1).text()).toBe("message_2");
+        expect(wrapper.find("#dataset-error-has-empty-inputs")).toBeDefined();
+        expect(wrapper.find("#dataset-error-has-duplicate-inputs")).toBeDefined();
+    });
+
+    it("check props without common problems", async () => {
+        const wrapper = buildWrapper(false, false);
+        expect(wrapper.find("#dataset-error-tool-id").text()).toBe("tool_id");
+        expect(wrapper.find("#dataset-error-tool-stderr").text()).toBe("tool_stderr");
+        expect(wrapper.find("#dataset-error-job-stderr").text()).toBe("job_stderr");
+        expect(wrapper.findAll("#dataset-error-has-empty-inputs").length).toBe(0);
+        expect(wrapper.findAll("#dataset-error-has-duplicate-inputs").length).toBe(0);
     });
 });
diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 616ede5624f..5f5d27957b0 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -24,7 +24,7 @@
                     <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">
                         <h3>Details</h3>
                     </div>
-                    <div v-if="jobDetails.job_messages">
+                    <div v-if="jobDetails.job_messages" id="dataset-error-job-messages">
                         <p>Execution resulted in the following messages:</p>
                         <div v-for="(job_message, index) in jobDetails.job_messages" :key="index">
                             <pre class="rounded code">{{ job_message["desc"] }}</pre>
@@ -36,16 +36,16 @@
                     </div>
                     <div v-if="jobDetails.job_stderr">
                         <p>Galaxy job runner generated the following standard error:</p>
-                        <pre class="rounded code">{{ jobDetails.job_stderr }}</pre>
+                        <pre id="dataset-error-job-stderr" class="rounded code">{{ jobDetails.job_stderr }}</pre>
                     </div>
                     <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result: jobProblems }" @error="onError">
                         <div v-if="jobProblems && (jobProblems.has_duplicate_inputs || jobProblems.has_empty_inputs)">
                             <h3 class="common_problems mt-3">Detected Common Potential Problems</h3>
-                            <p v-if="jobProblems.has_empty_inputs">
+                            <p v-if="jobProblems.has_empty_inputs" id="dataset-error-has-empty-inputs">
                                 The tool was executed with one or more empty input datasets. This frequently results in
                                 tool errors due to problematic input choices.
                             </p>
-                            <p v-if="jobProblems.has_duplicate_inputs">
+                            <p v-if="jobProblems.has_duplicate_inputs" id="dataset-error-has-duplicate-inputs">
                                 The tool was executed with one or more duplicate input datasets. This frequently results
                                 in tool errors due to problematic input choices.
                             </p>

From 3398d1e617ef1965a893a55655fef48cf9c38095 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 3 Aug 2021 17:28:04 -0400
Subject: [PATCH 270/452] Add email input element rendering test to dataset
 error

---
 .../DatasetInformation/DatasetError.test.js           |  7 +++++--
 .../components/DatasetInformation/DatasetError.vue    | 11 ++++++++---
 client/src/components/Form/Elements/FormInput.vue     |  4 ++--
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.test.js b/client/src/components/DatasetInformation/DatasetError.test.js
index c50488aac85..9fd95ad9310 100644
--- a/client/src/components/DatasetInformation/DatasetError.test.js
+++ b/client/src/components/DatasetInformation/DatasetError.test.js
@@ -9,7 +9,7 @@ jest.mock("components/WorkflowInvocationState/providers", () => {
 
 const localVue = getLocalVue();
 
-function buildWrapper(has_duplicate_inputs = true, has_empty_inputs = true) {
+function buildWrapper(has_duplicate_inputs = true, has_empty_inputs = true, user_email = "") {
     return mount(DatasetError, {
         propsData: {
             datasetId: "dataset_id",
@@ -22,6 +22,7 @@ function buildWrapper(has_duplicate_inputs = true, has_empty_inputs = true) {
                     tool_stderr: "tool_stderr",
                     job_stderr: "job_stderr",
                     job_messages: [{ desc: "message_1" }, { desc: "message_2" }],
+                    user_email: user_email,
                 },
             }),
             JobProblemProvider: MockProvider({
@@ -48,14 +49,16 @@ describe("DatasetError", () => {
         expect(messages.at(1).text()).toBe("message_2");
         expect(wrapper.find("#dataset-error-has-empty-inputs")).toBeDefined();
         expect(wrapper.find("#dataset-error-has-duplicate-inputs")).toBeDefined();
+        expect(wrapper.findAll("#dataset-error-email").length).toBe(1);
     });
 
     it("check props without common problems", async () => {
-        const wrapper = buildWrapper(false, false);
+        const wrapper = buildWrapper(false, false, "user_email");
         expect(wrapper.find("#dataset-error-tool-id").text()).toBe("tool_id");
         expect(wrapper.find("#dataset-error-tool-stderr").text()).toBe("tool_stderr");
         expect(wrapper.find("#dataset-error-job-stderr").text()).toBe("job_stderr");
         expect(wrapper.findAll("#dataset-error-has-empty-inputs").length).toBe(0);
         expect(wrapper.findAll("#dataset-error-has-duplicate-inputs").length).toBe(0);
+        expect(wrapper.findAll("#dataset-error-email").length).toBe(0);
     });
 });
diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 5f5d27957b0..2e3c2c14ec7 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -72,17 +72,22 @@
                     >
                     <FormElement
                         v-if="!jobDetails.user_email"
-                        id="email"
+                        id="dataset-error-email"
                         v-model="email"
                         title="Please provide your email:"
                     />
                     <FormElement
-                        id="message"
+                        id="dataset-error-message"
                         v-model="message"
                         :area="true"
                         title="Please provide detailed information on the activities leading to this issue:"
                     />
-                    <b-button variant="primary" @click="submit(dataset, jobDetails.user_email)" class="mt-3">
+                    <b-button
+                        id="dataset-error-submit"
+                        variant="primary"
+                        @click="submit(dataset, jobDetails.user_email)"
+                        class="mt-3"
+                    >
                         <font-awesome-icon icon="bug" class="mr-1" />Report
                     </b-button>
                 </div>
diff --git a/client/src/components/Form/Elements/FormInput.vue b/client/src/components/Form/Elements/FormInput.vue
index c667572e13c..d2cc0264769 100644
--- a/client/src/components/Form/Elements/FormInput.vue
+++ b/client/src/components/Form/Elements/FormInput.vue
@@ -1,6 +1,6 @@
 <template>
-    <textarea v-if="area" class="ui-textarea" v-model="currentValue" />
-    <input v-else class="ui-input" v-model="currentValue" />
+    <textarea v-if="area" :id="id" class="ui-textarea" v-model="currentValue" />
+    <input v-else :id="id" class="ui-input" v-model="currentValue" />
 </template>
 <script>
 export default {

From 20e7c8f534d717e5976ec9bdce2630de6b5bba49 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 3 Aug 2021 23:16:54 -0400
Subject: [PATCH 271/452] Fix tool id style in dataset error report to avoid
 overflow

---
 client/src/components/DatasetInformation/DatasetError.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 2e3c2c14ec7..5618117ab4d 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -18,7 +18,7 @@
                     <h2>Dataset Error Report</h2>
                     <p>
                         An error occurred while running the tool
-                        <b id="dataset-error-tool-id">{{ jobDetails.tool_id }}</b
+                        <b id="dataset-error-tool-id" class="text-break">{{ jobDetails.tool_id }}</b
                         >.
                     </p>
                     <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">

From 2907c284abcb323c4964801c46830f2df8c4bc00 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 07:54:41 -0400
Subject: [PATCH 272/452] Use workflow step title as defined in the workflow
 model

---
 client/src/components/Workflow/Run/WorkflowRunToolStep.vue | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index c1f616f070d..5d28ea26698 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -1,6 +1,6 @@
 <template>
     <div :step-label="model.step_label">
-        <FormCard :title="title" icon="fa-wrench" :collapsible="true" :collapsed.sync="collapsed">
+        <FormCard :title="model.fixed_title" icon="fa-wrench" :collapsible="true" :collapsed.sync="collapsed">
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <FormDisplay
@@ -77,11 +77,6 @@ export default {
             }
         },
     },
-    computed: {
-        title() {
-            return `${this.model.name} ${this.model.description} (Galaxy Version ${this.model.version})`;
-        },
-    },
     methods: {
         onReplaceParams() {
             const params = {};

From aebdfe9bba8fbcec5fdba36e3d61d55810f325ed Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 15:27:08 -0400
Subject: [PATCH 273/452] Redirect analysis route from dataset edit attribute
 module to vue component

---
 client/src/entry/analysis/AnalysisRouter.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index 1e34faef65a..09b69422919 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -42,7 +42,7 @@ import HistoryList from "mvc/history/history-list";
 import PluginList from "components/PluginList.vue";
 import QueryStringParsing from "utils/query-string-parsing";
 import DatasetError from "components/DatasetInformation/DatasetError";
-import DatasetEditAttributes from "mvc/dataset/dataset-edit-attributes";
+import DatasetAttributes from "components/DatasetInformation/DatasetAttributes";
 import Citations from "components/Citation/Citations.vue";
 import DisplayStructure from "components/DisplayStructured.vue";
 import { CloudAuth } from "components/User/CloudAuth";
@@ -385,8 +385,9 @@ export const getAnalysisRouter = (Galaxy) => {
             this._display_vue_helper(CustomBuilds);
         },
 
-        show_dataset_edit_attributes: function () {
-            this.page.display(new DatasetEditAttributes.View());
+        show_dataset_edit_attributes: function (params) {
+            const datasetId = params.dataset_id;
+            this._display_vue_helper(DatasetAttributes, { datasetId: datasetId });
         },
 
         show_dataset_error: function (params) {

From 39cdadd5d444d52efc28e82a77305c10fdead2b1 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 15:43:18 -0400
Subject: [PATCH 274/452] Add preliminary dataset attributes vue component
 shell

---
 .../DatasetInformation/DatasetAttributes.vue  | 92 +++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 client/src/components/DatasetInformation/DatasetAttributes.vue

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
new file mode 100644
index 00000000000..1a44606107d
--- /dev/null
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -0,0 +1,92 @@
+<template>
+    <div>
+        <h4>Edit dataset attributes</h4>
+        <b-alert v-if="messageText" :variant="messageVariant" show>
+            <h4 class="alert-heading">Failed to access Dataset details.</h4>
+            {{ messageText }}
+        </b-alert>
+        <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading: datasetLoading }">
+            <div class="mt-3">
+                <b-tabs>
+                    <b-tab>
+                        <template #title>
+                            <font-awesome-icon icon="bars" class="mr-1"/>Attributes
+                        </template>
+                        <p>I'm the first tab</p>
+                    </b-tab>
+                    <b-tab>
+                        <template #title>
+                            <font-awesome-icon icon="fas-gear" class="mr-1"/>Convert
+                        </template>
+                        <p>I'm the second tab</p>
+                    </b-tab>
+                    <b-tab>
+                        <template #title>
+                            <font-awesome-icon icon="database" class="mr-1"/>Datatypes
+                        </template>
+                        <p>I'm the second tab</p>
+                    </b-tab>
+                    <b-tab>
+                        <template #title>
+                            <font-awesome-icon icon="user" class="mr-1"/>Permissions
+                        </template>
+                        <p>I'm the second tab</p>
+                    </b-tab>
+                </b-tabs>
+            </div>
+        </DatasetProvider>
+    </div>
+</template>
+
+<script>
+import FormElement from "components/Form/FormElement";
+import { DatasetProvider } from "components/WorkflowInvocationState/providers";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faBars, faDatabase, faGear, faUser } from "@fortawesome/free-solid-svg-icons";
+import { sendErrorReport } from "./services";
+
+library.add(faBars);
+library.add(faDatabase);
+library.add(faGear);
+library.add(faUser);
+
+export default {
+    components: {
+        DatasetProvider,
+        FontAwesomeIcon,
+    },
+    props: {
+        datasetId: {
+            type: String,
+            required: true,
+        },
+    },
+    data() {
+        return {
+            message: null,
+            email: null,
+            messageText: null,
+            messageVariant: null,
+            resultMessages: [],
+        };
+    },
+    methods: {
+        onError(err) {
+            this.messageText = err;
+        },
+        submit(dataset, userEmail) {
+            const email = this.email || userEmail;
+            const message = this.message;
+            sendErrorReport(dataset, message, email).then(
+                (resultMessages) => {
+                    this.resultMessages = resultMessages;
+                },
+                (messageText) => {
+                    this.messageText = messageText;
+                }
+            );
+        },
+    },
+};
+</script>

From 1d0a7c6a1e672322458a26057892da4cb5fd7c94 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 15:51:59 -0400
Subject: [PATCH 275/452] Add tabs, icons and titles to dataset attributes
 component

---
 .../DatasetInformation/DatasetAttributes.vue  | 29 +++++++------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index 1a44606107d..b469840b7b7 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -1,36 +1,28 @@
 <template>
     <div>
-        <h4>Edit dataset attributes</h4>
+        <h4>Edit Dataset Attributes</h4>
         <b-alert v-if="messageText" :variant="messageVariant" show>
-            <h4 class="alert-heading">Failed to access Dataset details.</h4>
+            <h4 class="alert-heading">Failed to access dataset details.</h4>
             {{ messageText }}
         </b-alert>
         <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading: datasetLoading }">
             <div class="mt-3">
                 <b-tabs>
                     <b-tab>
-                        <template #title>
-                            <font-awesome-icon icon="bars" class="mr-1"/>Attributes
-                        </template>
+                        <template #title><font-awesome-icon icon="bars" class="mr-1" />Attributes</template>
                         <p>I'm the first tab</p>
                     </b-tab>
                     <b-tab>
-                        <template #title>
-                            <font-awesome-icon icon="fas-gear" class="mr-1"/>Convert
-                        </template>
+                        <template #title><font-awesome-icon icon="cog" class="mr-1" />Convert</template>
                         <p>I'm the second tab</p>
                     </b-tab>
                     <b-tab>
-                        <template #title>
-                            <font-awesome-icon icon="database" class="mr-1"/>Datatypes
-                        </template>
-                        <p>I'm the second tab</p>
+                        <template #title><font-awesome-icon icon="database" class="mr-1" />Datatypes</template>
+                        <p>I'm the third tab</p>
                     </b-tab>
                     <b-tab>
-                        <template #title>
-                            <font-awesome-icon icon="user" class="mr-1"/>Permissions
-                        </template>
-                        <p>I'm the second tab</p>
+                        <template #title><font-awesome-icon icon="user" class="mr-1" />Permissions</template>
+                        <p>I'm the fourth tab</p>
                     </b-tab>
                 </b-tabs>
             </div>
@@ -43,12 +35,12 @@ import FormElement from "components/Form/FormElement";
 import { DatasetProvider } from "components/WorkflowInvocationState/providers";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
-import { faBars, faDatabase, faGear, faUser } from "@fortawesome/free-solid-svg-icons";
+import { faBars, faDatabase, faCog, faUser } from "@fortawesome/free-solid-svg-icons";
 import { sendErrorReport } from "./services";
 
 library.add(faBars);
 library.add(faDatabase);
-library.add(faGear);
+library.add(faCog);
 library.add(faUser);
 
 export default {
@@ -68,7 +60,6 @@ export default {
             email: null,
             messageText: null,
             messageVariant: null,
-            resultMessages: [],
         };
     },
     methods: {

From fc62f82a2e534f20cb60a5bcb8f5eb66c70a6ed0 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 16:05:35 -0400
Subject: [PATCH 276/452] Add dataset attributes provider and embed in
 component

---
 .../DatasetInformation/DatasetAttributes.vue     | 10 +++++-----
 .../src/components/providers/DatasetProvider.js  | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 5 deletions(-)
 create mode 100644 client/src/components/providers/DatasetProvider.js

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index b469840b7b7..e9f33aa7e31 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -5,8 +5,8 @@
             <h4 class="alert-heading">Failed to access dataset details.</h4>
             {{ messageText }}
         </b-alert>
-        <DatasetProvider :id="datasetId" v-slot="{ item: dataset, loading: datasetLoading }">
-            <div class="mt-3">
+        <DatasetAttributesProvider :id="datasetId" v-slot="{ result: dataset, loading }">
+            <div v-if="!loading" class="mt-3">
                 <b-tabs>
                     <b-tab>
                         <template #title><font-awesome-icon icon="bars" class="mr-1" />Attributes</template>
@@ -26,13 +26,13 @@
                     </b-tab>
                 </b-tabs>
             </div>
-        </DatasetProvider>
+        </DatasetAttributesProvider>
     </div>
 </template>
 
 <script>
 import FormElement from "components/Form/FormElement";
-import { DatasetProvider } from "components/WorkflowInvocationState/providers";
+import { DatasetAttributesProvider } from "components/providers/DatasetProvider";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faBars, faDatabase, faCog, faUser } from "@fortawesome/free-solid-svg-icons";
@@ -45,7 +45,7 @@ library.add(faUser);
 
 export default {
     components: {
-        DatasetProvider,
+        DatasetAttributesProvider,
         FontAwesomeIcon,
     },
     props: {
diff --git a/client/src/components/providers/DatasetProvider.js b/client/src/components/providers/DatasetProvider.js
new file mode 100644
index 00000000000..0ab3e5e2c0a
--- /dev/null
+++ b/client/src/components/providers/DatasetProvider.js
@@ -0,0 +1,16 @@
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
+import { rethrowSimple } from "utils/simple-error";
+
+async function datasetAttributes({ id }) {
+    const url = `${getAppRoot()}dataset/get_edit?dataset_id=${id}`;
+    try {
+        const { data } = await axios.get(url);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}
+
+export const DatasetAttributesProvider = SingleQueryProvider(datasetAttributes);

From ea530cec514d3c9d48b409be8bbb77db13a4d359 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 16:17:38 -0400
Subject: [PATCH 277/452] Add edit forms to dataset attributes component

---
 .../DatasetInformation/DatasetAttributes.vue  | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index e9f33aa7e31..2f4dcebe89c 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -5,24 +5,24 @@
             <h4 class="alert-heading">Failed to access dataset details.</h4>
             {{ messageText }}
         </b-alert>
-        <DatasetAttributesProvider :id="datasetId" v-slot="{ result: dataset, loading }">
+        <DatasetAttributesProvider :id="datasetId" v-slot="{ result, loading }">
             <div v-if="!loading" class="mt-3">
                 <b-tabs>
-                    <b-tab>
+                    <b-tab v-if="!result['attribute_disable']">
                         <template #title><font-awesome-icon icon="bars" class="mr-1" />Attributes</template>
-                        <p>I'm the first tab</p>
+                        <FormDisplay :inputs="result['attribute_inputs']" />
                     </b-tab>
-                    <b-tab>
+                    <b-tab v-if="!result['conversion_disable']">
                         <template #title><font-awesome-icon icon="cog" class="mr-1" />Convert</template>
-                        <p>I'm the second tab</p>
+                        <FormDisplay :inputs="result['conversion_inputs']" />
                     </b-tab>
-                    <b-tab>
+                    <b-tab v-if="!result['datatype_disable']">
                         <template #title><font-awesome-icon icon="database" class="mr-1" />Datatypes</template>
-                        <p>I'm the third tab</p>
+                        <FormDisplay :inputs="result['datatype_inputs']" />
                     </b-tab>
-                    <b-tab>
+                    <b-tab v-if="!result['permission_disable']">
                         <template #title><font-awesome-icon icon="user" class="mr-1" />Permissions</template>
-                        <p>I'm the fourth tab</p>
+                        <FormDisplay :inputs="result['permission_inputs']" />
                     </b-tab>
                 </b-tabs>
             </div>
@@ -31,7 +31,7 @@
 </template>
 
 <script>
-import FormElement from "components/Form/FormElement";
+import FormDisplay from "components/Form/FormDisplay";
 import { DatasetAttributesProvider } from "components/providers/DatasetProvider";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
@@ -47,6 +47,7 @@ export default {
     components: {
         DatasetAttributesProvider,
         FontAwesomeIcon,
+        FormDisplay,
     },
     props: {
         datasetId: {

From 55e3812e97ad0047323aff33c926012c00162437 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 16:53:28 -0400
Subject: [PATCH 278/452] Add operation buttons to dataset attribute editing
 component

---
 .../DatasetInformation/DatasetAttributes.vue  | 27 +++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index 2f4dcebe89c..e5adf62c881 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -11,18 +11,38 @@
                     <b-tab v-if="!result['attribute_disable']">
                         <template #title><font-awesome-icon icon="bars" class="mr-1" />Attributes</template>
                         <FormDisplay :inputs="result['attribute_inputs']" />
+                        <div class="mt-2">
+                            <b-button variant="primary" class="mr-1">
+                                <font-awesome-icon icon="save" class="mr-1" />Save
+                            </b-button>
+                            <b-button> <font-awesome-icon icon="redo" class="mr-1" />Auto-detect </b-button>
+                        </div>
                     </b-tab>
                     <b-tab v-if="!result['conversion_disable']">
                         <template #title><font-awesome-icon icon="cog" class="mr-1" />Convert</template>
                         <FormDisplay :inputs="result['conversion_inputs']" />
+                        <div class="mt-2">
+                            <b-button variant="primary">
+                                <font-awesome-icon icon="exchange-alt" class="mr-1" />Create Dataset
+                            </b-button>
+                        </div>
                     </b-tab>
                     <b-tab v-if="!result['datatype_disable']">
                         <template #title><font-awesome-icon icon="database" class="mr-1" />Datatypes</template>
                         <FormDisplay :inputs="result['datatype_inputs']" />
+                        <div class="mt-2">
+                            <b-button variant="primary" class="mr-1">
+                                <font-awesome-icon icon="save" class="mr-1" />Save
+                            </b-button>
+                            <b-button> <font-awesome-icon icon="redo" class="mr-1" />Auto-detect </b-button>
+                        </div>
                     </b-tab>
                     <b-tab v-if="!result['permission_disable']">
                         <template #title><font-awesome-icon icon="user" class="mr-1" />Permissions</template>
                         <FormDisplay :inputs="result['permission_inputs']" />
+                        <div class="mt-2">
+                            <b-button variant="primary"> <font-awesome-icon icon="save" class="mr-1" />Save </b-button>
+                        </div>
                     </b-tab>
                 </b-tabs>
             </div>
@@ -35,12 +55,15 @@ import FormDisplay from "components/Form/FormDisplay";
 import { DatasetAttributesProvider } from "components/providers/DatasetProvider";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
-import { faBars, faDatabase, faCog, faUser } from "@fortawesome/free-solid-svg-icons";
+import { faBars, faCog, faDatabase, faExchangeAlt, faRedo, faSave, faUser } from "@fortawesome/free-solid-svg-icons";
 import { sendErrorReport } from "./services";
 
 library.add(faBars);
-library.add(faDatabase);
 library.add(faCog);
+library.add(faDatabase);
+library.add(faExchangeAlt);
+library.add(faRedo);
+library.add(faSave);
 library.add(faUser);
 
 export default {

From d3b3d6e54999ec37fd4ce92a7a08f4cc0bcfa323 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 17:33:57 -0400
Subject: [PATCH 279/452] Add operation handling to dataset attributes
 component

---
 .../DatasetInformation/DatasetAttributes.vue  | 87 ++++++++++++-------
 .../components/DatasetInformation/services.js | 15 ++++
 2 files changed, 72 insertions(+), 30 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index e5adf62c881..602ec36e225 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -2,46 +2,55 @@
     <div>
         <h4>Edit Dataset Attributes</h4>
         <b-alert v-if="messageText" :variant="messageVariant" show>
-            <h4 class="alert-heading">Failed to access dataset details.</h4>
-            {{ messageText }}
+            {{ messageText | l }}
         </b-alert>
         <DatasetAttributesProvider :id="datasetId" v-slot="{ result, loading }">
             <div v-if="!loading" class="mt-3">
                 <b-tabs>
                     <b-tab v-if="!result['attribute_disable']">
-                        <template #title><font-awesome-icon icon="bars" class="mr-1" />Attributes</template>
-                        <FormDisplay :inputs="result['attribute_inputs']" />
+                        <template #title><font-awesome-icon icon="bars" class="mr-1" />{{ "Attributes" | l }}</template>
+                        <FormDisplay :inputs="result['attribute_inputs']" @onChange="onAttribute" />
                         <div class="mt-2">
-                            <b-button variant="primary" class="mr-1">
-                                <font-awesome-icon icon="save" class="mr-1" />Save
+                            <b-button @click="submit('attribute', 'attributes')" variant="primary" class="mr-1">
+                                <font-awesome-icon icon="save" class="mr-1" />{{ "Save" | l }}
+                            </b-button>
+                            <b-button @click="submit('attribute', 'autodetect')">
+                                <font-awesome-icon icon="redo" class="mr-1" />{{ "Auto-detect" | l }}
                             </b-button>
-                            <b-button> <font-awesome-icon icon="redo" class="mr-1" />Auto-detect </b-button>
                         </div>
                     </b-tab>
                     <b-tab v-if="!result['conversion_disable']">
-                        <template #title><font-awesome-icon icon="cog" class="mr-1" />Convert</template>
-                        <FormDisplay :inputs="result['conversion_inputs']" />
+                        <template #title><font-awesome-icon icon="cog" class="mr-1" />{{ "Convert" | l }}</template>
+                        <FormDisplay :inputs="result['conversion_inputs']" @onChange="onConversion" />
                         <div class="mt-2">
-                            <b-button variant="primary">
-                                <font-awesome-icon icon="exchange-alt" class="mr-1" />Create Dataset
+                            <b-button @click="submit('conversion', 'conversion')" variant="primary">
+                                <font-awesome-icon icon="exchange-alt" class="mr-1" />{{ "Create Dataset" | l }}
                             </b-button>
                         </div>
                     </b-tab>
                     <b-tab v-if="!result['datatype_disable']">
-                        <template #title><font-awesome-icon icon="database" class="mr-1" />Datatypes</template>
-                        <FormDisplay :inputs="result['datatype_inputs']" />
+                        <template #title
+                            ><font-awesome-icon icon="database" class="mr-1" />{{ "Datatypes" | l }}</template
+                        >
+                        <FormDisplay :inputs="result['datatype_inputs']" @onChange="onDatatype" />
                         <div class="mt-2">
-                            <b-button variant="primary" class="mr-1">
-                                <font-awesome-icon icon="save" class="mr-1" />Save
+                            <b-button @click="submit('datatype', 'datatype')" variant="primary" class="mr-1">
+                                <font-awesome-icon icon="save" class="mr-1" />{{ "Save" | l }}
+                            </b-button>
+                            <b-button @click="submit('datatype', 'datatype_detect')">
+                                <font-awesome-icon icon="redo" class="mr-1" />{{ "Auto-detect" | l }}
                             </b-button>
-                            <b-button> <font-awesome-icon icon="redo" class="mr-1" />Auto-detect </b-button>
                         </div>
                     </b-tab>
                     <b-tab v-if="!result['permission_disable']">
-                        <template #title><font-awesome-icon icon="user" class="mr-1" />Permissions</template>
-                        <FormDisplay :inputs="result['permission_inputs']" />
+                        <template #title
+                            ><font-awesome-icon icon="user" class="mr-1" />{{ "Permissions" | l }}</template
+                        >
+                        <FormDisplay :inputs="result['permission_inputs']" @onChange="onPermission" />
                         <div class="mt-2">
-                            <b-button variant="primary"> <font-awesome-icon icon="save" class="mr-1" />Save </b-button>
+                            <b-button @click="submit('permission', 'permission')" variant="primary">
+                                <font-awesome-icon icon="save" class="mr-1" />{{ "Save" | l }}
+                            </b-button>
                         </div>
                     </b-tab>
                 </b-tabs>
@@ -51,12 +60,13 @@
 </template>
 
 <script>
+import { getGalaxyInstance } from "app";
 import FormDisplay from "components/Form/FormDisplay";
 import { DatasetAttributesProvider } from "components/providers/DatasetProvider";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faBars, faCog, faDatabase, faExchangeAlt, faRedo, faSave, faUser } from "@fortawesome/free-solid-svg-icons";
-import { sendErrorReport } from "./services";
+import { setAttributes } from "./services";
 
 library.add(faBars);
 library.add(faCog);
@@ -80,28 +90,45 @@ export default {
     },
     data() {
         return {
-            message: null,
-            email: null,
             messageText: null,
             messageVariant: null,
+            formData: {},
         };
     },
     methods: {
-        onError(err) {
-            this.messageText = err;
+        onAttribute(data) {
+            this.formData["attribute"] = data;
         },
-        submit(dataset, userEmail) {
-            const email = this.email || userEmail;
-            const message = this.message;
-            sendErrorReport(dataset, message, email).then(
-                (resultMessages) => {
-                    this.resultMessages = resultMessages;
+        onConversion(data) {
+            this.formData["conversion"] = data;
+        },
+        onDatatype(data) {
+            this.formData["datatype"] = data;
+        },
+        onPermission(data) {
+            this.formData["permission"] = data;
+        },
+        submit(key, operation) {
+            setAttributes(this.datasetId, this.formData[key], operation).then(
+                (response) => {
+                    this.messageText = response.message;
+                    this.messageVariant = response.status;
+                    this._reloadHistory();
                 },
                 (messageText) => {
                     this.messageText = messageText;
+                    this.messageVariant = "danger";
                 }
             );
         },
+
+        /** reload Galaxy's history after updating dataset's attributes */
+        _reloadHistory: function () {
+            const Galaxy = getGalaxyInstance();
+            if (Galaxy) {
+                Galaxy.currHistoryPanel.loadCurrentHistory();
+            }
+        },
     },
 };
 </script>
diff --git a/client/src/components/DatasetInformation/services.js b/client/src/components/DatasetInformation/services.js
index 7739f7fed83..310c5b51ca0 100644
--- a/client/src/components/DatasetInformation/services.js
+++ b/client/src/components/DatasetInformation/services.js
@@ -16,3 +16,18 @@ export async function sendErrorReport(dataset, message, email) {
         rethrowSimple(e);
     }
 }
+
+export async function setAttributes(datasetId, settings, operation) {
+    const payload = {
+        dataset_id: datasetId,
+        operation: operation,
+        ...settings,
+    };
+    const url = `${getAppRoot()}dataset/set_edit`;
+    try {
+        const { data } = await axios.put(url, payload);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}

From 9f3fe3ab7827882ed7ff647fcefdb0e4d5a88887 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 17:34:33 -0400
Subject: [PATCH 280/452] Remove legacy backbone based dataset attribute edit
 view

---
 .../mvc/dataset/dataset-edit-attributes.js    | 225 ------------------
 1 file changed, 225 deletions(-)
 delete mode 100644 client/src/mvc/dataset/dataset-edit-attributes.js

diff --git a/client/src/mvc/dataset/dataset-edit-attributes.js b/client/src/mvc/dataset/dataset-edit-attributes.js
deleted file mode 100644
index f14bfaad0ee..00000000000
--- a/client/src/mvc/dataset/dataset-edit-attributes.js
+++ /dev/null
@@ -1,225 +0,0 @@
-/** Dataset edit attributes view */
-import _ from "underscore";
-import $ from "jquery";
-import Backbone from "backbone";
-import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
-import _l from "utils/localization";
-import Tabs from "mvc/ui/ui-tabs";
-import Ui from "mvc/ui/ui-misc";
-import Form from "mvc/form/form-view";
-
-var View = Backbone.View.extend({
-    initialize: function () {
-        const Galaxy = getGalaxyInstance();
-        this.setElement("<div/>");
-        this.model = new Backbone.Model({
-            dataset_id: Galaxy.params.dataset_id,
-        });
-        this.message = new Ui.Message({ persistent: true });
-        this.tabs = this._createTabs();
-        this.active_tab = "user";
-        this.$el
-            .append($("<h4/>").append("Edit dataset attributes"))
-            .append(this.message.$el)
-            .append("<p/>")
-            .append(this.tabs.$el)
-            .hide();
-        this.render();
-    },
-
-    /** fetch data for the selected dataset and build forms */
-    render: function () {
-        var self = this;
-        $.ajax({
-            url: `${getAppRoot()}dataset/get_edit?dataset_id=${self.model.get("dataset_id")}`,
-            success: function (response) {
-                !self.initial_message && self.message.update(response);
-                self.initial_message = true;
-                _.each(self.forms, (form, key) => {
-                    form.model.set("inputs", response[`${key}_inputs`]);
-                    form.model.set("hide_operations", response[`${key}_disable`]);
-                    form.render();
-                });
-                self.$el.show();
-            },
-            error: function (response) {
-                var err_msg = response.responseJSON && response.responseJSON.err_msg;
-                self.message.update({
-                    status: "danger",
-                    message: err_msg || "Error occurred while loading the dataset.",
-                });
-            },
-        });
-    },
-
-    /** submit data to backend to update attributes */
-    _submit: function (operation, form) {
-        var self = this;
-        var data = form.data.create();
-        data.dataset_id = this.model.get("dataset_id");
-        data.operation = operation;
-        $.ajax({
-            type: "PUT",
-            url: `${getAppRoot()}dataset/set_edit`,
-            data: data,
-            success: function (response) {
-                self.message.update(response);
-                self.render();
-                self._reloadHistory();
-            },
-            error: function (response) {
-                var err_msg = response.responseJSON && response.responseJSON.err_msg;
-                self.message.update({
-                    status: "danger",
-                    message: err_msg || "Error occurred while editing the dataset attributes.",
-                });
-            },
-        });
-    },
-
-    /** create tabs for different dataset attribute categories*/
-    _createTabs: function () {
-        this.forms = {
-            attribute: this._getAttribute(),
-            conversion: this._getConversion(),
-            datatype: this._getDatatype(),
-            permission: this._getPermission(),
-        };
-        var tabs = new Tabs.View();
-        tabs.add({
-            id: "attribute",
-            title: _l("Attributes"),
-            icon: "fa fa-bars",
-            tooltip: _l("Edit dataset attributes"),
-            $el: this.forms.attribute.$el,
-        });
-        tabs.add({
-            id: "convert",
-            title: _l("Convert"),
-            icon: "fa-gear",
-            tooltip: _l("Convert to new format"),
-            $el: this.forms.conversion.$el,
-        });
-        tabs.add({
-            id: "datatype",
-            title: _l("Datatypes"),
-            icon: "fa-database",
-            tooltip: _l("Change data type"),
-            $el: this.forms.datatype.$el,
-        });
-        tabs.add({
-            id: "permissions",
-            title: _l("Permissions"),
-            icon: "fa-user",
-            tooltip: _l("Permissions"),
-            $el: this.forms.permission.$el,
-        });
-        return tabs;
-    },
-
-    /** edit main attributes form */
-    _getAttribute: function () {
-        var self = this;
-        var form = new Form({
-            title: _l("Edit attributes"),
-            operations: {
-                submit_attributes: new Ui.Button({
-                    tooltip: _l("Save attributes of the dataset."),
-                    icon: "fa-floppy-o",
-                    title: _l("Save"),
-                    onclick: function () {
-                        self._submit("attributes", form);
-                    },
-                }),
-                submit_autodetect: new Ui.Button({
-                    tooltip:
-                        "This will inspect the dataset and attempt to correct the values of fields if they are not accurate.",
-                    icon: "fa-redo",
-                    title: "Auto-detect",
-                    onclick: function () {
-                        self._submit("autodetect", form);
-                    },
-                }),
-            },
-        });
-        return form;
-    },
-
-    /** datatype conversion form */
-    _getConversion: function () {
-        var self = this;
-        var form = new Form({
-            title: _l("Convert to new format"),
-            operations: {
-                submit_conversion: new Ui.Button({
-                    tooltip: _l("Convert the datatype to a new format."),
-                    title: _l("Convert datatype"),
-                    icon: "fa-exchange",
-                    onclick: function () {
-                        self._submit("conversion", form);
-                    },
-                }),
-            },
-        });
-        return form;
-    },
-
-    /** change datatype form */
-    _getDatatype: function () {
-        var self = this;
-        var form = new Form({
-            title: _l("Change datatype"),
-            operations: {
-                submit_datatype: new Ui.Button({
-                    tooltip: _l("Change the datatype to a new type."),
-                    title: _l("Change datatype"),
-                    icon: "fa-exchange",
-                    onclick: function () {
-                        self._submit("datatype", form);
-                    },
-                }),
-                submit_datatype_detect: new Ui.Button({
-                    tooltip: _l("Detect the datatype and change it."),
-                    title: _l("Detect datatype"),
-                    icon: "fa-redo",
-                    onclick: function () {
-                        self._submit("datatype_detect", form);
-                    },
-                }),
-            },
-        });
-        return form;
-    },
-
-    /** dataset permissions form */
-    _getPermission: function () {
-        var self = this;
-        var form = new Form({
-            title: _l("Manage dataset permissions"),
-            operations: {
-                submit_permission: new Ui.Button({
-                    tooltip: _l("Save permissions."),
-                    title: _l("Save permissions"),
-                    icon: "fa-floppy-o ",
-                    onclick: function () {
-                        self._submit("permission", form);
-                    },
-                }),
-            },
-        });
-        return form;
-    },
-
-    /** reload Galaxy's history after updating dataset's attributes */
-    _reloadHistory: function () {
-        const Galaxy = getGalaxyInstance();
-        if (Galaxy) {
-            Galaxy.currHistoryPanel.loadCurrentHistory();
-        }
-    },
-});
-
-export default {
-    View: View,
-};

From 0c14ef3ae47ac9b39a7c58da1dc9e6d2e6f123d9 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 4 Aug 2021 17:57:25 -0400
Subject: [PATCH 281/452] Add error handling to dataset attributes, fix v-slot
 tag

---
 .../DatasetInformation/DatasetAttributes.vue  | 32 +++++++++++--------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index 602ec36e225..cb1dbe70f64 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -4,11 +4,13 @@
         <b-alert v-if="messageText" :variant="messageVariant" show>
             {{ messageText | l }}
         </b-alert>
-        <DatasetAttributesProvider :id="datasetId" v-slot="{ result, loading }">
+        <DatasetAttributesProvider :id="datasetId" v-slot="{ result, loading }" @error="onError">
             <div v-if="!loading" class="mt-3">
                 <b-tabs>
                     <b-tab v-if="!result['attribute_disable']">
-                        <template #title><font-awesome-icon icon="bars" class="mr-1" />{{ "Attributes" | l }}</template>
+                        <template v-slot:title>
+                            <font-awesome-icon icon="bars" class="mr-1" />{{ "Attributes" | l }}
+                        </template>
                         <FormDisplay :inputs="result['attribute_inputs']" @onChange="onAttribute" />
                         <div class="mt-2">
                             <b-button @click="submit('attribute', 'attributes')" variant="primary" class="mr-1">
@@ -20,7 +22,9 @@
                         </div>
                     </b-tab>
                     <b-tab v-if="!result['conversion_disable']">
-                        <template #title><font-awesome-icon icon="cog" class="mr-1" />{{ "Convert" | l }}</template>
+                        <template v-slot:title>
+                            <font-awesome-icon icon="cog" class="mr-1" />{{ "Convert" | l }}
+                        </template>
                         <FormDisplay :inputs="result['conversion_inputs']" @onChange="onConversion" />
                         <div class="mt-2">
                             <b-button @click="submit('conversion', 'conversion')" variant="primary">
@@ -29,9 +33,9 @@
                         </div>
                     </b-tab>
                     <b-tab v-if="!result['datatype_disable']">
-                        <template #title
-                            ><font-awesome-icon icon="database" class="mr-1" />{{ "Datatypes" | l }}</template
-                        >
+                        <template v-slot:title>
+                            <font-awesome-icon icon="database" class="mr-1" />{{ "Datatypes" | l }}
+                        </template>
                         <FormDisplay :inputs="result['datatype_inputs']" @onChange="onDatatype" />
                         <div class="mt-2">
                             <b-button @click="submit('datatype', 'datatype')" variant="primary" class="mr-1">
@@ -43,9 +47,9 @@
                         </div>
                     </b-tab>
                     <b-tab v-if="!result['permission_disable']">
-                        <template #title
-                            ><font-awesome-icon icon="user" class="mr-1" />{{ "Permissions" | l }}</template
-                        >
+                        <template v-slot:title>
+                            <font-awesome-icon icon="user" class="mr-1" />{{ "Permissions" | l }}
+                        </template>
                         <FormDisplay :inputs="result['permission_inputs']" @onChange="onPermission" />
                         <div class="mt-2">
                             <b-button @click="submit('permission', 'permission')" variant="primary">
@@ -108,6 +112,10 @@ export default {
         onPermission(data) {
             this.formData["permission"] = data;
         },
+        onError(messageText) {
+            this.messageText = messageText;
+            this.messageVariant = "danger";
+        },
         submit(key, operation) {
             setAttributes(this.datasetId, this.formData[key], operation).then(
                 (response) => {
@@ -115,13 +123,9 @@ export default {
                     this.messageVariant = response.status;
                     this._reloadHistory();
                 },
-                (messageText) => {
-                    this.messageText = messageText;
-                    this.messageVariant = "danger";
-                }
+                this.onError
             );
         },
-
         /** reload Galaxy's history after updating dataset's attributes */
         _reloadHistory: function () {
             const Galaxy = getGalaxyInstance();

From 0b5688a600ee74279d803233f36e4fa2dbd7413e Mon Sep 17 00:00:00 2001
From: Dannon <dannon.baker@gmail.com>
Date: Thu, 5 Aug 2021 08:02:40 -0700
Subject: [PATCH 282/452] Update
 client/src/components/DatasetInformation/DatasetAttributes.vue

---
 .../components/DatasetInformation/DatasetAttributes.vue   | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index cb1dbe70f64..90ba8a46333 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -72,13 +72,7 @@ import { library } from "@fortawesome/fontawesome-svg-core";
 import { faBars, faCog, faDatabase, faExchangeAlt, faRedo, faSave, faUser } from "@fortawesome/free-solid-svg-icons";
 import { setAttributes } from "./services";
 
-library.add(faBars);
-library.add(faCog);
-library.add(faDatabase);
-library.add(faExchangeAlt);
-library.add(faRedo);
-library.add(faSave);
-library.add(faUser);
+library.add(faBars, faCog, faDatabase, faExchangeAlt, faRedo, faSave, faUser);
 
 export default {
     components: {

From 2af3ef5ca6ca836843cc59b5833e5cd81cf683dc Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 5 Aug 2021 19:04:03 -0400
Subject: [PATCH 283/452] Format client

---
 .../DatasetInformation/DatasetAttributes.vue        | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index 90ba8a46333..ac267d191df 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -111,14 +111,11 @@ export default {
             this.messageVariant = "danger";
         },
         submit(key, operation) {
-            setAttributes(this.datasetId, this.formData[key], operation).then(
-                (response) => {
-                    this.messageText = response.message;
-                    this.messageVariant = response.status;
-                    this._reloadHistory();
-                },
-                this.onError
-            );
+            setAttributes(this.datasetId, this.formData[key], operation).then((response) => {
+                this.messageText = response.message;
+                this.messageVariant = response.status;
+                this._reloadHistory();
+            }, this.onError);
         },
         /** reload Galaxy's history after updating dataset's attributes */
         _reloadHistory: function () {

From a7cea39d5ea3bf072092220118c26cc2a8a8a342 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 5 Aug 2021 21:35:15 -0400
Subject: [PATCH 284/452] Fix button identifier used in selenium test

---
 .../components/DatasetInformation/DatasetAttributes.vue    | 7 ++++++-
 lib/galaxy/selenium/navigation.yml                         | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index ac267d191df..f740347dcbe 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -13,7 +13,12 @@
                         </template>
                         <FormDisplay :inputs="result['attribute_inputs']" @onChange="onAttribute" />
                         <div class="mt-2">
-                            <b-button @click="submit('attribute', 'attributes')" variant="primary" class="mr-1">
+                            <b-button
+                                id="dataset-attributes-default-save"
+                                @click="submit('attribute', 'attributes')"
+                                variant="primary"
+                                class="mr-1"
+                            >
                                 <font-awesome-icon icon="save" class="mr-1" />{{ "Save" | l }}
                             </b-button>
                             <b-button @click="submit('attribute', 'autodetect')">
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 11fc8a5c98c..8e69f39e21c 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -234,7 +234,7 @@ history_structure:
 edit_dataset_attributes:
   selectors:
     database_build_dropdown: '[data-label="Database/Build"]'
-    save_btn: '#submit_attributes'
+    save_btn: '#dataset-attributes-default-save'
 
   dbkey_dropdown_results:
     selectors:

From 7fb5e5c9c42f8dd0baae094873a555012cf493be Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 5 Aug 2021 22:25:38 -0400
Subject: [PATCH 285/452] Fix hash-key generation in single query provider

---
 client/src/components/providers/SingleQueryProvider.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/providers/SingleQueryProvider.js b/client/src/components/providers/SingleQueryProvider.js
index dff317ba5d4..ba5c3677467 100644
--- a/client/src/components/providers/SingleQueryProvider.js
+++ b/client/src/components/providers/SingleQueryProvider.js
@@ -23,7 +23,7 @@ export const SingleQueryProvider = (lookup) => {
                 return this.result === undefined;
             },
             cacheKey() {
-                hash(this.$attrs || {});
+                return hash(this.$attrs || {});
             },
         },
         mounted() {

From b020fdb9cd7a47e3f8ba622382da8e6d7069eafb Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 6 Aug 2021 17:49:32 -0400
Subject: [PATCH 286/452] Add vue test cases for dataset attributes component

---
 .../DatasetAttributes.test.js                 | 62 +++++++++++++++++++
 .../DatasetInformation/DatasetAttributes.vue  |  1 +
 2 files changed, 63 insertions(+)
 create mode 100644 client/src/components/DatasetInformation/DatasetAttributes.test.js

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.test.js b/client/src/components/DatasetInformation/DatasetAttributes.test.js
new file mode 100644
index 00000000000..a5533ecbe4e
--- /dev/null
+++ b/client/src/components/DatasetInformation/DatasetAttributes.test.js
@@ -0,0 +1,62 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "jest/helpers";
+import DatasetAttributes from "./DatasetAttributes";
+import MockProvider from "../providers/MockProvider";
+import axios from "axios";
+import MockAdapter from "axios-mock-adapter";
+import Vue from "vue";
+
+const localVue = getLocalVue();
+const flushPromises = () => new Promise((resolve) => setTimeout(resolve));
+
+function buildWrapper(conversion_disable = false) {
+    return mount(DatasetAttributes, {
+        propsData: {
+            datasetId: "dataset_id",
+        },
+        localVue,
+        stubs: {
+            DatasetAttributesProvider: MockProvider({
+                result: {
+                    attribute_inputs: [{ name: "attribute_text", type: "text" }],
+                    conversion_inputs: [{ name: "conversion_text", type: "text" }],
+                    conversion_disable: conversion_disable,
+                    datatype_inputs: [{ name: "datatype_text", type: "text" }],
+                    permission_inputs: [{ name: "permission_text", type: "text" }],
+                },
+            }),
+            FontAwesomeIcon: false,
+            FormElement: false,
+        },
+    });
+}
+
+describe("DatasetAttributes", () => {
+    it("check rendering", async () => {
+        const axiosMock = new MockAdapter(axios);
+        axiosMock.onPut(`/dataset/set_edit`).reply(200, { message: "success", status: "success" });
+        const wrapper = buildWrapper();
+        await Vue.nextTick();
+        expect(wrapper.findAll("button").length).toBe(6);
+        expect(wrapper.findAll("[tour_id=attribute_text]").length).toBe(1);
+        expect(wrapper.findAll("[tour_id=conversion_text]").length).toBe(1);
+        expect(wrapper.findAll("[tour_id=datatype_text]").length).toBe(1);
+        expect(wrapper.findAll("[tour_id=permission_text]").length).toBe(1);
+        expect(wrapper.findAll(".tab-pane").length).toBe(4);
+        const $button = wrapper.find("#dataset-attributes-default-save");
+        await $button.trigger("click");
+        await flushPromises();
+        expect(wrapper.find("[role=alert]").text()).toBe("success");
+    });
+
+    it("check rendering without conversion option", async () => {
+        const wrapper = buildWrapper(true);
+        await Vue.nextTick();
+        expect(wrapper.findAll("button").length).toBe(5);
+        expect(wrapper.findAll("[tour_id=attribute_text]").length).toBe(1);
+        expect(wrapper.findAll("[tour_id=conversion_text]").length).toBe(0);
+        expect(wrapper.findAll("[tour_id=datatype_text]").length).toBe(1);
+        expect(wrapper.findAll("[tour_id=permission_text]").length).toBe(1);
+        expect(wrapper.findAll(".tab-pane").length).toBe(3);
+    });
+});
diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index f740347dcbe..824d0cc67bf 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -120,6 +120,7 @@ export default {
                 this.messageText = response.message;
                 this.messageVariant = response.status;
                 this._reloadHistory();
+                this.$emit("onSubmit", this.messageText);
             }, this.onError);
         },
         /** reload Galaxy's history after updating dataset's attributes */

From 36114459af01c8e17a114dbd0819d7b03f7f078c Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 6 Aug 2021 17:51:23 -0400
Subject: [PATCH 287/452] Remove unused submission event emission

---
 .../components/DatasetInformation/DatasetAttributes.test.js   | 4 ++--
 .../src/components/DatasetInformation/DatasetAttributes.vue   | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.test.js b/client/src/components/DatasetInformation/DatasetAttributes.test.js
index a5533ecbe4e..7f9d5de41a6 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.test.js
+++ b/client/src/components/DatasetInformation/DatasetAttributes.test.js
@@ -1,10 +1,10 @@
+import Vue from "vue";
+import axios from "axios";
 import { mount } from "@vue/test-utils";
 import { getLocalVue } from "jest/helpers";
 import DatasetAttributes from "./DatasetAttributes";
 import MockProvider from "../providers/MockProvider";
-import axios from "axios";
 import MockAdapter from "axios-mock-adapter";
-import Vue from "vue";
 
 const localVue = getLocalVue();
 const flushPromises = () => new Promise((resolve) => setTimeout(resolve));
diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index 824d0cc67bf..f740347dcbe 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -120,7 +120,6 @@ export default {
                 this.messageText = response.message;
                 this.messageVariant = response.status;
                 this._reloadHistory();
-                this.$emit("onSubmit", this.messageText);
             }, this.onError);
         },
         /** reload Galaxy's history after updating dataset's attributes */

From 861ebf182b7337494577c28d4fd9751c72c967c9 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 6 Aug 2021 18:31:24 -0400
Subject: [PATCH 288/452] Augment SingleQueryProvider, remove unnecessary
 result cache, add non-caching option

---
 .../DatasetInformation/DatasetAttributes.vue  |  2 +-
 .../providers/SingleQueryProvider.js          | 39 +++++++++++--------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index f740347dcbe..a63485f4223 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -4,7 +4,7 @@
         <b-alert v-if="messageText" :variant="messageVariant" show>
             {{ messageText | l }}
         </b-alert>
-        <DatasetAttributesProvider :id="datasetId" v-slot="{ result, loading }" @error="onError">
+        <DatasetAttributesProvider :id="datasetId" :useCache="false" v-slot="{ result, loading }" @error="onError">
             <div v-if="!loading" class="mt-3">
                 <b-tabs>
                     <b-tab v-if="!result['attribute_disable']">
diff --git a/client/src/components/providers/SingleQueryProvider.js b/client/src/components/providers/SingleQueryProvider.js
index ba5c3677467..bee613787e0 100644
--- a/client/src/components/providers/SingleQueryProvider.js
+++ b/client/src/components/providers/SingleQueryProvider.js
@@ -10,9 +10,14 @@ import hash from "object-hash";
  */
 export const SingleQueryProvider = (lookup) => {
     const promiseCache = new Map();
-    const resultCache = new Map();
 
     return {
+        props: {
+            useCache: {
+                type: Boolean,
+                default: true,
+            },
+        },
         data() {
             return {
                 result: undefined,
@@ -27,24 +32,24 @@ export const SingleQueryProvider = (lookup) => {
             },
         },
         mounted() {
-            const cachedResult = resultCache.get(this.cacheKey);
-            if (cachedResult) {
-                this.result = cachedResult;
-                return;
+            let lookupPromise;
+            if (this.useCache) {
+                lookupPromise = promiseCache.get(this.cacheKey);
+                if (!lookupPromise) {
+                    lookupPromise = lookup(this.$attrs);
+                    promiseCache.set(this.cacheKey, lookupPromise);
+                }
+            } else {
+                lookupPromise = lookup(this.$attrs);
             }
-
-            let lookupPromise = promiseCache.get(this.cacheKey);
-            if (!lookupPromise) {
-                lookupPromise = lookup(this.$attrs).catch((err) => {
+            lookupPromise.then(
+                (result) => {
+                    this.result = result;
+                },
+                (err) => {
                     this.$emit("error", err);
-                });
-                promiseCache.set(this.cacheKey, lookupPromise);
-            }
-
-            lookupPromise.then((result) => {
-                resultCache.set(this.cacheKey, result);
-                this.result = result;
-            });
+                }
+            );
         },
         render() {
             return this.$scopedSlots.default({

From 62ee25091479f1c2044e853420f3ee21843ed5c8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 6 Aug 2021 18:44:42 -0400
Subject: [PATCH 289/452] Fix lint issue, hyphenate use cache prop

---
 client/src/components/DatasetInformation/DatasetAttributes.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.vue b/client/src/components/DatasetInformation/DatasetAttributes.vue
index a63485f4223..4685833941a 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.vue
+++ b/client/src/components/DatasetInformation/DatasetAttributes.vue
@@ -4,7 +4,7 @@
         <b-alert v-if="messageText" :variant="messageVariant" show>
             {{ messageText | l }}
         </b-alert>
-        <DatasetAttributesProvider :id="datasetId" :useCache="false" v-slot="{ result, loading }" @error="onError">
+        <DatasetAttributesProvider :id="datasetId" :use-cache="false" v-slot="{ result, loading }" @error="onError">
             <div v-if="!loading" class="mt-3">
                 <b-tabs>
                     <b-tab v-if="!result['attribute_disable']">

From ce312452c69f6213e786e0943bb3688e6a2c0892 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 6 Aug 2021 15:08:14 -0400
Subject: [PATCH 290/452] per host configuration parameters

---
 lib/galaxy/config/__init__.py                 | 13 +++++++++++++
 lib/galaxy/config/config_manage.py            |  2 +-
 lib/galaxy/config/schema.py                   |  7 +++++++
 lib/galaxy/managers/configuration.py          | 10 ++++++----
 lib/galaxy/webapps/base/controller.py         |  2 +-
 lib/galaxy/webapps/base/webapp.py             |  1 +
 lib/galaxy/webapps/galaxy/api/__init__.py     |  2 +-
 lib/galaxy/webapps/galaxy/config_schema.yml   | 18 ++++++++++++++++++
 lib/galaxy/webapps/galaxy/controllers/root.py |  2 +-
 lib/galaxy/work/context.py                    |  5 +++++
 10 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 0eb7403b1ae..d50bb2d4683 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -564,6 +564,19 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
         if string_as_bool(kwargs.get("override_tempdir", "True")):
             tempfile.tempdir = self.new_file_path
 
+    def config_value_for_host(self, config_option, host):
+        val = getattr(self, config_option)
+        if config_option in self.schema.per_host_options:
+            per_host_option = f"{config_option}_by_host"
+            if per_host_option in self.config_dict:
+                per_host = self.config_dict[per_host_option] or {}
+                for host_key, host_val in per_host.items():
+                    if host_key in host:
+                        val = host_val
+                        break
+
+        return val
+
     def _process_config(self, kwargs):
         # Backwards compatibility for names used in too many places to fix
         self.datatypes_config = self.datatypes_config_file
diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py
index 309cebc5200..4a2251e4429 100644
--- a/lib/galaxy/config/config_manage.py
+++ b/lib/galaxy/config/config_manage.py
@@ -534,7 +534,7 @@ def _validate(args, app_desc):
         ordered_dump(raw_config, config_p)
 
     def _clean(p, k, v):
-        return k not in ['reloadable', 'path_resolves_to']
+        return k not in ['reloadable', 'path_resolves_to', 'per_host']
 
     clean_schema = remap(app_desc.schema.raw_schema, _clean)
     with tempfile.NamedTemporaryFile('w', suffix=".yml") as fp:
diff --git a/lib/galaxy/config/schema.py b/lib/galaxy/config/schema.py
index c357f06a350..ebd88e05c74 100644
--- a/lib/galaxy/config/schema.py
+++ b/lib/galaxy/config/schema.py
@@ -53,10 +53,13 @@ class AppSchema(Schema):
         self._defaults = {}  # {config option: default value or null}
         self._reloadable_options = set()  # config options we can reload at runtime
         self._paths_to_resolve = {}  # {config option: referenced config option}
+        self._per_host_options = set()  # config options that can be set using a per_host config parameter
         for key, data in app_schema.items():
             self._defaults[key] = data.get('default')
             if data.get('reloadable'):
                 self._reloadable_options.add(key)
+            if data.get('per_host'):
+                self._per_host_options.add(key)
             if data.get('path_resolves_to'):
                 self._paths_to_resolve[key] = data.get('path_resolves_to')
 
@@ -72,6 +75,10 @@ class AppSchema(Schema):
     def reloadable_options(self):
         return self._reloadable_options
 
+    @property
+    def per_host_options(self):
+        return self._per_host_options
+
     def validate_path_resolution_graph(self):
         """This method is for tests only: we SHOULD validate the schema's path resolution graph
            as part of automated testing; but we should NOT validate it at runtime.
diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py
index 3e43f4320c4..7516de0bde8 100644
--- a/lib/galaxy/managers/configuration.py
+++ b/lib/galaxy/managers/configuration.py
@@ -39,8 +39,10 @@ class ConfigurationManager:
         serialization_params: SerializationParams
     ) -> Dict[str, Any]:
         is_admin = trans.user_is_admin
-        serializer = AdminConfigSerializer(self._app) if is_admin else ConfigSerializer(self._app)
-        return serializer.serialize_to_view(self._app.config, **serialization_params)
+        host = getattr(trans, "host", None)
+        serializer_class = AdminConfigSerializer if is_admin else ConfigSerializer
+        serializer = serializer_class(self._app)
+        return serializer.serialize_to_view(self._app.config, host=host, **serialization_params)
 
     def version(self) -> Dict[str, Any]:
         version_info = {
@@ -120,7 +122,7 @@ class ConfigSerializer(base.ModelSerializer):
         def _use_config(config, key, **context):
             """Let config object determine the value for key"""
             assert hasattr(config, key)
-            return getattr(config, key)
+            return config.config_value_for_host(key, context.get("host"))
 
         def _config_is_truthy(config, key, **context):
             return True if config.get(key) else False
@@ -217,7 +219,7 @@ class AdminConfigSerializer(ConfigSerializer):
 
         self.serializers.update({
             # TODO: this is available from user serialization: remove
-            'is_admin_user': lambda *a: True,
+            'is_admin_user': lambda *a, **context: True,
 
             'library_import_dir': _defaults_to(None),
             'user_library_import_dir': _defaults_to(None),
diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py
index e7780e5d424..e45ab020ab3 100644
--- a/lib/galaxy/webapps/base/controller.py
+++ b/lib/galaxy/webapps/base/controller.py
@@ -336,7 +336,7 @@ class JSAppLauncher(BaseUIController):
             serializer = self.config_serializer
             if self.user_manager.is_admin(trans.user, trans=trans):
                 serializer = self.admin_config_serializer
-            return serializer.serialize_to_view(self.app.config, view='all')
+            return serializer.serialize_to_view(self.app.config, view='all', host=trans.host)
         except Exception as exc:
             log.exception(exc)
             return {}
diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py
index 867ed69b501..e0b749ac20d 100644
--- a/lib/galaxy/webapps/base/webapp.py
+++ b/lib/galaxy/webapps/base/webapp.py
@@ -218,6 +218,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryCo
         self.__user = None
         self.galaxy_session = None
         self.error_message = None
+        self.host = self.request.host
 
         # set any cross origin resource sharing headers if configured to do so
         self.set_cors_headers()
diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py
index e10a08b8a02..d021fbaa5e4 100644
--- a/lib/galaxy/webapps/galaxy/api/__init__.py
+++ b/lib/galaxy/webapps/galaxy/api/__init__.py
@@ -140,7 +140,7 @@ def get_trans(request: Request, app: StructuredApp = DependsOnApp, user: Optiona
               galaxy_session: Optional[model.GalaxySession] = Depends(get_session),
               ) -> SessionRequestContext:
     qualified_url_builder = QualifiedUrlBuilder(request)
-    return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session, qualified_url_builder=qualified_url_builder)
+    return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session, qualified_url_builder=qualified_url_builder, host=request.client.host)
 
 
 DependsOnTrans = Depends(get_trans)
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index be85ddc24ef..14c9b869f39 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -1237,6 +1237,7 @@ mapping:
       brand:
         type: str
         required: false
+        per_host: true
         desc: |
           Append "{brand}" text to the masthead.
 
@@ -1244,6 +1245,7 @@ mapping:
         type: bool
         default: true
         required: false
+        per_host: true
         desc: |
           Display the "Galaxy" text in the masthead.
 
@@ -1327,6 +1329,7 @@ mapping:
         default: /static/welcome.html
         required: false
         reloadable: true
+        per_host: true
         desc: |
           The URL of the page to display in Galaxy's middle pane when loaded.  This can
           be an absolute or relative URL.
@@ -1335,6 +1338,7 @@ mapping:
         type: str
         default: /
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Galaxy/brand" text.
 
@@ -1342,18 +1346,21 @@ mapping:
         type: str
         default: /static/favicon.png
         required: false
+        per_host: true
         desc: |
           The brand image source.
 
       logo_src_secondary:
         type: str
         required: false
+        per_host: true
         desc: |
           The custom brand image source.
 
       helpsite_url:
         type: str
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Galaxy Help" link in the "Help" menu.
 
@@ -1361,6 +1368,7 @@ mapping:
         type: str
         default: https://galaxyproject.org/
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Wiki" link in the "Help" menu.
 
@@ -1375,6 +1383,7 @@ mapping:
         type: str
         default: https://galaxyproject.org/support/
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Support" link in the "Help" menu.
 
@@ -1382,6 +1391,7 @@ mapping:
         type: str
         default: https://galaxyproject.org/citing-galaxy
         required: false
+        per_host: true
         desc: |
           The URL linked by the "How to Cite Galaxy" link in the "Help" menu.
 
@@ -1396,6 +1406,7 @@ mapping:
         type: str
         default: https://galaxyproject.org/search/
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Search" link in the "Help" menu.
 
@@ -1403,6 +1414,7 @@ mapping:
         type: str
         default: https://galaxyproject.org/mailing-lists/
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Mailing Lists" link in the "Help" menu.
 
@@ -1410,12 +1422,14 @@ mapping:
         type: str
         default: https://vimeo.com/galaxyproject
         required: false
+        per_host: true
         desc: |
           The URL linked by the "Videos" link in the "Help" menu.
 
       terms_url:
         type: str
         required: false
+        per_host: true        
         desc: |
           The URL linked by the "Terms and Conditions" link in the "Help" menu, as well
           as on the user registration and login forms and in the activation emails.
@@ -2638,6 +2652,7 @@ mapping:
         default: 'prefer'
         enum: ['off', 'prefer']
         required: false
+        per_host: true
         desc: |
           If set to 'off' by default, always use the traditional workflow form that renders
           all steps in the GUI and serializes the tool state of all steps during
@@ -2652,6 +2667,7 @@ mapping:
         default: 'current'
         enum: ['current', 'new']
         required: false
+        per_host: true
         desc: |
           When the simplified workflow run form is rendered, should the invocation outputs
           be sent to the 'current' history or a 'new' history.
@@ -2661,6 +2677,7 @@ mapping:
         default: 'off'
         enum: ['on', 'off']
         required: false
+        per_host: true
         desc: |
           When the simplified workflow run form is rendered, should the invocation use job
           caching. This isn't a boolean so an option for 'show-selection' can be added later.
@@ -2675,6 +2692,7 @@ mapping:
       ftp_upload_site:
         type: str
         required: false
+        per_host: true
         desc: |
           Enable Galaxy's "Upload via FTP" interface.  You'll need to install and
           configure an FTP server (we've used ProFTPd since it can use Galaxy's
diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py
index a00448a7007..8300a8032f9 100644
--- a/lib/galaxy/webapps/galaxy/controllers/root.py
+++ b/lib/galaxy/webapps/galaxy/controllers/root.py
@@ -252,5 +252,5 @@ class RootController(controller.JSAppLauncher, UsesAnnotations):
 
     @web.expose
     def welcome(self, trans):
-        welcome_url = trans.app.config.welcome_url
+        welcome_url = trans.app.config.config_value_for_host("welcome_url", trans.host)
         return trans.response.send_redirect(web.url_for(welcome_url))
diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py
index 5fbcf79f16a..a53f4af6a19 100644
--- a/lib/galaxy/work/context.py
+++ b/lib/galaxy/work/context.py
@@ -62,8 +62,13 @@ class SessionRequestContext(WorkRequestContext):
     """Like WorkRequestContext, but provides access to galaxy session and session."""
     def __init__(self, **kwargs):
         self.galaxy_session = kwargs.pop('galaxy_session', None)
+        self._host = kwargs.pop("host")
         super().__init__(**kwargs)
 
+    @property
+    def host(self):
+        return self._host
+
     def get_galaxy_session(self):
         return self.galaxy_session
 

From d8e95aede7b3f1f741ff029e98e4fc4a325d647a Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 10 Aug 2021 09:50:23 +0200
Subject: [PATCH 291/452] Including column to order in distinct breaks tests

---
 lib/galaxy/model/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 66013884c7a..ae3d3fcbdee 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -4299,7 +4299,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
             q = q.add_entity(entity)
             if entity == DatasetCollectionElement:
                 q = q.filter(entity.id == dce.c.id)
-        return q.distinct(*order_by_columns).order_by(*order_by_columns)
+        return q.distinct().order_by(*order_by_columns)
 
     @property
     def dataset_states_and_extensions_summary(self):

From 760461a1c722990fbf9bc8f81c08eb272a57c5f6 Mon Sep 17 00:00:00 2001
From: selten <selten@users.noreply.github.com>
Date: Tue, 10 Aug 2021 15:21:03 +0200
Subject: [PATCH 292/452] Gitlab Error report - Remove previously required
 urllib

## Why these changes
The library has been updated whereby it no longer accepts urlencoded data that used to be required.

## Description on changes
Removal of previously required urllib for URL encoding. Combined with enforced minimum version of python-gitlab to fix the issue.
---
 .../tools/error_reports/plugins/gitlab.py     | 22 +++++--------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/lib/galaxy/tools/error_reports/plugins/gitlab.py b/lib/galaxy/tools/error_reports/plugins/gitlab.py
index ada1ff7462a..f3615060581 100644
--- a/lib/galaxy/tools/error_reports/plugins/gitlab.py
+++ b/lib/galaxy/tools/error_reports/plugins/gitlab.py
@@ -5,12 +5,6 @@ import os
 import sys
 
 import requests
-if sys.version_info[0] < 3:
-    import urllib as urllib
-    import urlparse as urlparse
-else:
-    import urllib.parse as urllib
-    urlparse = urllib
 
 try:
     import gitlab
@@ -115,21 +109,18 @@ class GitLabPlugin(BaseGitPlugin):
                     else "/".join((self.git_default_repo_owner, self.git_default_repo_name))
                 issue_cache_key = self._get_issue_cache_key(job, ts_repourl)
 
-                gitlab_urlencodedpath = urllib.quote_plus(gitlab_projecturl)
-
                 # Make sure we are always logged in, then retrieve the GitLab project if it isn't cached.
                 self.gitlab.auth()
                 try:
                     if gitlab_projecturl not in self.git_project_cache:
-                        self.git_project_cache[gitlab_projecturl] = self.gitlab.projects.get(gitlab_urlencodedpath)
+                        self.git_project_cache[gitlab_projecturl] = self.gitlab.projects.get(gitlab_projecturl)
                 except gitlab.GitlabGetError:
                     # Handle scenario where the repository doesn't exist so we can still continue
-                    log.warning(f"GitLab error reporting - Repository '{gitlab_urlencodedpath}' doesn't exist, using default repository.")
+                    log.warning(f"GitLab error reporting - Repository '{gitlab_projecturl}' doesn't exist, using default repository.")
                     # Redo some of the previous steps to recover from such an issue but continue issue creation
                     gitlab_projecturl = "/".join((self.git_default_repo_owner, self.git_default_repo_name))
-                    gitlab_urlencodedpath = urllib.quote_plus(gitlab_projecturl)
                     if gitlab_projecturl not in self.git_project_cache:
-                        self.git_project_cache[gitlab_projecturl] = self.gitlab.projects.get(gitlab_urlencodedpath)
+                        self.git_project_cache[gitlab_projecturl] = self.gitlab.projects.get(gitlab_projecturl)
                 gl_project = self.git_project_cache[gitlab_projecturl]
 
                 # Make sure we keep a cache of the issues, per tool in this case
@@ -175,7 +166,7 @@ class GitLabPlugin(BaseGitPlugin):
 
                         # Add a comment to an issue...
                         self._append_issue(issue_cache_key, error_title, error_message,
-                                           gitlab_urlencodedpath=gitlab_urlencodedpath)
+                                           gitlab_projecturl=gitlab_projecturl)
                     else:
                         self._open_issue(error_message, error_title, gitlab_projecturl, gl_project, gl_userid,
                                          issue_cache_key)
@@ -222,11 +213,10 @@ class GitLabPlugin(BaseGitPlugin):
         except (gitlab.GitlabOwnershipError, gitlab.GitlabGetError):
             # Create an issue in the default location
             gitlab_projecturl = "/".join((self.git_default_repo_owner, self.git_default_repo_name))
-            gitlab_urlencodedpath = urllib.quote_plus(gitlab_projecturl)
             # Make sure we are always logged in, then retrieve the GitLab project if it isn't cached.
             self.gitlab = self.gitlab_connect()
             if gitlab_projecturl not in self.git_project_cache:
-                self.git_project_cache[gitlab_projecturl] = self.gitlab.projects.get(gitlab_urlencodedpath)
+                self.git_project_cache[gitlab_projecturl] = self.gitlab.projects.get(gitlab_projecturl)
             gl_project = self.git_project_cache[gitlab_projecturl]
 
             # Submit issue to default project
@@ -262,7 +252,7 @@ class GitLabPlugin(BaseGitPlugin):
             "api",
             "v4",
             "projects",
-            kwargs.get('gitlab_urlencodedpath'),
+            kwargs.get('gitlab_projecturl'),
             "issues",
             str(self.issue_cache[issue_cache_key][error_title]),
             "notes"

From 0a686a46b43278746755c0d1a431967bdb4afc15 Mon Sep 17 00:00:00 2001
From: selten <selten@users.noreply.github.com>
Date: Tue, 10 Aug 2021 15:28:31 +0200
Subject: [PATCH 293/452] Update python-gitlab dependency

Update the dependency so it works with the updated GitLab error reporting system
---
 lib/galaxy/dependencies/conditional-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt
index e8076d51508..758b8607d14 100644
--- a/lib/galaxy/dependencies/conditional-requirements.txt
+++ b/lib/galaxy/dependencies/conditional-requirements.txt
@@ -33,7 +33,7 @@ kamaki
 watchdog
 
 # Error reporters optional modules
-python-gitlab
+python-gitlab>=2.10.0
 pygithub
 influxdb
 

From ece39129d2662c4924d4555be57263ebfe6630ba Mon Sep 17 00:00:00 2001
From: Aysam Guerler <aysam.guerler@gmail.com>
Date: Tue, 10 Aug 2021 10:47:39 -0400
Subject: [PATCH 294/452] Update
 client/src/components/DatasetInformation/DatasetAttributes.test.js

Co-authored-by: Dannon <dannon.baker@gmail.com>
---
 .../src/components/DatasetInformation/DatasetAttributes.test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/DatasetInformation/DatasetAttributes.test.js b/client/src/components/DatasetInformation/DatasetAttributes.test.js
index 7f9d5de41a6..524d2de08d3 100644
--- a/client/src/components/DatasetInformation/DatasetAttributes.test.js
+++ b/client/src/components/DatasetInformation/DatasetAttributes.test.js
@@ -5,9 +5,9 @@ import { getLocalVue } from "jest/helpers";
 import DatasetAttributes from "./DatasetAttributes";
 import MockProvider from "../providers/MockProvider";
 import MockAdapter from "axios-mock-adapter";
+import flushPromises from "flush-promises";
 
 const localVue = getLocalVue();
-const flushPromises = () => new Promise((resolve) => setTimeout(resolve));
 
 function buildWrapper(conversion_disable = false) {
     return mount(DatasetAttributes, {

From 48df4b910dff66578a00fa40aeb5aececdf99dcc Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 19 Jul 2021 15:17:59 -0400
Subject: [PATCH 295/452] Systematic handling of remotely required tool files.

- Allow tools to define "pulsar file action"-like paths for files to include and exclude during remote job setup.
- Package up a list of defaults for older tools.
---
 doc/schema_template.md                        |   3 +
 lib/galaxy/jobs/runners/pulsar.py             |   3 +-
 lib/galaxy/tool_util/parser/__init__.py       |   3 +-
 lib/galaxy/tool_util/parser/interface.py      |  58 ++++++
 lib/galaxy/tool_util/parser/xml.py            |  22 +++
 lib/galaxy/tool_util/xsd/galaxy.xsd           |  72 ++++++++
 lib/galaxy/tools/__init__.py                  |  21 +++
 packages/app/requirements.txt                 |   2 +-
 test/unit/tool_util/test_required_files.py    | 170 ++++++++++++++++++
 .../tool_directories/r-tool-dir/my_script.R   |   1 +
 .../r-tool-dir/other_script.R                 |   1 +
 11 files changed, 353 insertions(+), 3 deletions(-)
 create mode 100644 test/unit/tool_util/test_required_files.py
 create mode 100644 test/unit/tool_util/tool_directories/r-tool-dir/my_script.R
 create mode 100644 test/unit/tool_util/tool_directories/r-tool-dir/other_script.R

diff --git a/doc/schema_template.md b/doc/schema_template.md
index 2f98d31a22d..a78b21180bf 100644
--- a/doc/schema_template.md
+++ b/doc/schema_template.md
@@ -26,6 +26,9 @@ $tag:tool|creator|organization://complexType[@name='Organization']
 $tag:tool|requirements://complexType[@name='Requirements']
 $tag:tool|requirements|requirement://complexType[@name='Requirement']
 $tag:tool|requirements|container://complexType[@name='Container']
+$tag:tool|required_files://complexType[@name='RequiredFiles']
+$tag:tool|required_files|include://complexType[@name='RequiredFileInclude']
+$tag:tool|required_files|exclude://complexType[@name='RequiredFileExclude']
 $tag:tool|code://complexType[@name='Code']
 $tag:tool|stdio://complexType[@name='Stdio']
 $tag:tool|stdio|exit_code://complexType[@name='ExitCode'] hide_attributes
diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index 5d7dfa19830..d84e3af48ee 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -395,7 +395,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
                 job_directory_path = tool_env.get("job_directory_path")
                 if job_directory_path:
                     config_files.append(job_directory_path)
-
+            tool_directory_required_files = job_wrapper.tool.required_files
             client_job_description = ClientJobDescription(
                 command_line=command_line,
                 input_files=input_files,
@@ -414,6 +414,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
                 job_directory_files=job_directory_files,
                 container=None if not remote_container else remote_container.container_id,
                 guest_ports=job_wrapper.guest_ports,
+                tool_directory_required_files=tool_directory_required_files,
             )
             job_id = pulsar_submit_job(client, client_job_description, remote_job_config)
             log.info(f"Pulsar job submitted with job_id {job_id}")
diff --git a/lib/galaxy/tool_util/parser/__init__.py b/lib/galaxy/tool_util/parser/__init__.py
index 771fbaf3202..2a215042946 100644
--- a/lib/galaxy/tool_util/parser/__init__.py
+++ b/lib/galaxy/tool_util/parser/__init__.py
@@ -1,7 +1,7 @@
 """ Package responsible for parsing tools from files/abstract tool sources.
 """
 from .factory import get_input_source, get_tool_source, get_tool_source_from_representation
-from .interface import ToolSource
+from .interface import RequiredFiles, ToolSource
 from .output_objects import (
     ToolOutputCollectionPart,
 )
@@ -10,6 +10,7 @@ __all__ = (
     "get_input_source",
     "get_tool_source",
     "get_tool_source_from_representation",
+    "RequiredFiles",
     "ToolOutputCollectionPart",
     "ToolSource",
 )
diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py
index a30040a19b2..8cfead61887 100644
--- a/lib/galaxy/tool_util/parser/interface.py
+++ b/lib/galaxy/tool_util/parser/interface.py
@@ -1,9 +1,14 @@
+import fnmatch
 import os
+import re
 from abc import (
     ABCMeta,
     abstractmethod
 )
+from os.path import join
+from typing import Dict, List, Optional
 
+from galaxy.util.path import safe_walk
 from .util import _parse_name
 
 NOT_IMPLEMENTED_MESSAGE = "Galaxy tool format does not yet support this tool feature."
@@ -167,6 +172,10 @@ class ToolSource(metaclass=ABCMeta):
         """
         return False
 
+    def parse_required_files(self) -> Optional['RequiredFiles']:
+        """ Parse explicit RequiredFiles object or return None to let Galaxy decide implicit logic."""
+        return None
+
     @abstractmethod
     def parse_requirements_and_containers(self):
         """ Return pair of ToolRequirement and ContainerDescription lists. """
@@ -452,6 +461,55 @@ class TestCollectionDef:
         return inputs
 
 
+class RequiredFiles:
+
+    def __init__(self, includes: List[Dict], excludes: List[Dict], extend_default_excludes: bool):
+        self.includes = includes
+        self.excludes = excludes
+        self.extend_default_excludes = extend_default_excludes
+
+    @staticmethod
+    def from_dict(as_dict):
+        extend_default_excludes: bool = as_dict.get("extend_default_excludes", True)
+        includes: List = as_dict.get("includes", [])
+        excludes: List = as_dict.get("excludes", [])
+        return RequiredFiles(includes, excludes, extend_default_excludes)
+
+    def find_required_files(self, tool_directory: str) -> List[str]:
+
+        def matches(ie_list: List, rel_path: str):
+            for ie_item in ie_list:
+                ie_item_path = ie_item["path"]
+                ie_item_type = ie_item.get("path_type", "literal")
+                if ie_item_type == "literal":
+                    if rel_path == ie_item_path:
+                        return True
+                elif ie_item_type == "prefix":
+                    if rel_path.startswith(ie_item_path):
+                        return True
+                elif ie_item_type == "glob":
+                    if fnmatch.fnmatch(rel_path, ie_item_path):
+                        return True
+                else:
+                    if re.match(ie_item_path, rel_path) is not None:
+                        return True
+            return False
+
+        excludes = self.excludes
+        if self.extend_default_excludes:
+            excludes.append({"path": "tool-data", "path_type": "prefix"})
+            excludes.append({"path": "test-data", "path_type": "prefix"})
+            excludes.append({"path": ".hg", "path_type": "prefix"})
+
+        files: List[str] = []
+        for (dirpath, _, filenames) in safe_walk(tool_directory):
+            for filename in filenames:
+                rel_path = join(dirpath, filename).replace(tool_directory + os.path.sep, '')
+                if matches(self.includes, rel_path) and not matches(self.excludes, rel_path):
+                    files.append(rel_path)
+        return files
+
+
 class TestCollectionOutputDef:
     __test__ = False  # Prevent pytest from discovering this class (issue #12071)
 
diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py
index 3af561ca65c..a3bde5e3f2d 100644
--- a/lib/galaxy/tool_util/parser/xml.py
+++ b/lib/galaxy/tool_util/parser/xml.py
@@ -3,6 +3,7 @@ import logging
 import re
 import uuid
 from math import isinf
+from typing import Optional
 
 import packaging.version
 
@@ -20,6 +21,7 @@ from .interface import (
     InputSource,
     PageSource,
     PagesSource,
+    RequiredFiles,
     TestCollectionDef,
     TestCollectionOutputDef,
     ToolSource,
@@ -261,6 +263,26 @@ class XmlToolSource(ToolSource):
             elem = self.root
         return string_as_bool(elem.get(attribute, default))
 
+    def parse_required_files(self) -> Optional[RequiredFiles]:
+        required_files = self.root.find("required_files")
+        if required_files is None:
+            return None
+
+        def parse_include_exclude_list(tag_name):
+            as_list = []
+            for ref in required_files.findall(tag_name):
+                path = ref.get("path")
+                assert path is not None, f'"path" must be specified in {tag_name}'
+                path_type = ref.get("type", "literal")
+                as_list.append({"path": path, "path_type": path_type})
+            return as_list
+
+        as_dict = {}
+        as_dict["extend_default_excludes"] = self._get_attribute_as_bool("extend_default_excludes", True, elem=required_files)
+        as_dict["includes"] = parse_include_exclude_list("include")
+        as_dict["excludes"] = parse_include_exclude_list("exclude")
+        return RequiredFiles.from_dict(as_dict)
+
     def parse_requirements_and_containers(self):
         return requirements.parse_requirements_from_xml(self.root)
 
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 719962630b6..2ecbbf6eec5 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -89,6 +89,7 @@ A ``data_source`` tool contains a few more relevant attributes.
         <xs:element name="xrefs" type="xrefs" minOccurs="0" />
         <xs:element name="creator" type="Creator" minOccurs="0" />
         <xs:element name="requirements" type="Requirements" minOccurs="0"/>
+        <xs:element name="required_files" type="RequiredFiles" minOccurs="0"/>
         <xs:element name="entry_points" type="EntryPoints" minOccurs="0" maxOccurs="1" />
         <xs:element name="description" type="xs:string" minOccurs="0">
           <xs:annotation gxdocs:best_practices="tool-descriptions">
@@ -522,6 +523,77 @@ Describes an organization. Tries to stay close to [schema.org/Organization](http
     </xs:sequence>
   </xs:complexType>
 
+  <xs:complexType name="RequiredFiles">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+This declaration is used to define files that must be shipped from the tool directory
+for the tool to function properly in remote environments where the tool directory
+is not available to the job.
+
+All includes should be list before excludes. By default, the exclude list includes
+the tool-data/**, test-data/**, and .hg/** glob patterns.
+
+Pulsar hacks to implicitly find referenced files from the tool directory will be disabled
+when this block is used. A future Galaxy tool profile version may disable these hacks
+altogether and specifying this block for all referenced files should be considered a best
+practice.
+]]></xs:documentation>
+    </xs:annotation>
+    <xs:sequence>
+      <xs:element name="include" type="RequiredFileInclude" minOccurs="0" maxOccurs="unbounded"/>
+      <xs:element name="exclude" type="RequiredFileExclude" minOccurs="0" maxOccurs="unbounded"/>
+    </xs:sequence>
+    <xs:attribute name="extend_default_excludes" type="xs:boolean" default="true">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Set this to `false` to override the default excludes for mercurial, reference, and test data.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:complexType>
+
+  <xs:simpleType name="RequiredFileReferenceType">
+    <xs:annotation>
+      <xs:documentation xml:lang="en">How are files referenced in RequiredFileIncludes and RequiredFileExcludes. Paths are matched relative to the tool directory. `literal` must match the filename exactly. `prefix` will match paths based on their start. `glob` and `regex` use patterns to match files.</xs:documentation>
+    </xs:annotation>
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="literal"/>
+      <xs:enumeration value="prefix"/>
+      <xs:enumeration value="glob"/>
+      <xs:enumeration value="regex"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <xs:complexType name="RequiredFileInclude">
+    <xs:annotation>
+      <xs:documentation xml:lang="en">Describe files to include when relocating tool directory for remote execution.</xs:documentation>
+    </xs:annotation>
+    <xs:attribute name="type" type="RequiredFileReferenceType">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Type of file reference `path` is.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="path" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Path to referenced files - this should be relative to the tool's directory (this is the file the tool is located in not the repository directory if these conflict).</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:complexType>
+
+  <xs:complexType name="RequiredFileExclude">
+    <xs:annotation>
+      <xs:documentation xml:lang="en">Describe files to exclude when relocating tool directory for remote execution.</xs:documentation>
+    </xs:annotation>
+    <xs:attribute name="type" type="RequiredFileReferenceType">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Type of file reference `path` is.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="path" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Path to referenced files - this should be relative to the tool's directory (this is the file the tool is located in not the repository directory if these conflict).</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:complexType>
+
   <xs:complexType name="Requirements">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index e4dcc7b6669..a1681fcb135 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -43,6 +43,7 @@ from galaxy.tool_util.output_checker import DETECTED_JOB_STATE
 from galaxy.tool_util.parser import (
     get_tool_source,
     get_tool_source_from_representation,
+    RequiredFiles,
     ToolOutputCollectionPart
 )
 from galaxy.tool_util.parser.xml import XmlPageSource
@@ -195,6 +196,16 @@ GALAXY_LIB_TOOLS_VERSIONED = {
 BIOTOOLS_MAPPING_CONTENT = resource_string(__name__, 'biotools_mappings.tsv').decode("UTF-8")
 BIOTOOLS_MAPPING: Dict[str, str] = dict([cast(Tuple[str, str], tuple(x.split("\t"))) for x in BIOTOOLS_MAPPING_CONTENT.splitlines() if not x.startswith("#")])
 
+REQUIRE_FULL_DIRECTORY = {
+    "includes": [{"path": "**", "path_type": "glob"}],
+}
+IMPLICITLY_REQUIRED_TOOL_FILES: Dict[str, Dict] = {
+    "deseq2": {"version": packaging.version.parse("2.11.40.6"), "required": {"includes": [{"path": "*.R", "path_type": "glob"}]}},
+    # minimum example:
+    # "foobar": {"required": REQUIRE_FULL_DIRECTORY}
+    # if no version is specified, all versions without explicit RequiredFiles will be selected
+}
+
 
 class safe_update(NamedTuple):
     min_version: Union[packaging.version.LegacyVersion, packaging.version.Version]
@@ -963,6 +974,16 @@ class Tool(Dictifiable):
         self.requirements = requirements
         self.containers = containers
 
+        required_files = tool_source.parse_required_files()
+        if required_files is None:
+            old_id = self.old_id
+            if old_id in IMPLICITLY_REQUIRED_TOOL_FILES:
+                lineage_requirement = IMPLICITLY_REQUIRED_TOOL_FILES[old_id]
+                lineage_requirement_until = lineage_requirement.get("version")
+                if lineage_requirement_until is None or self.version_object < lineage_requirement_until:
+                    required_files = RequiredFiles.from_dict(lineage_requirement["required"])
+        self.required_files = required_files
+
         self.citations = self._parse_citations(tool_source)
         xrefs = tool_source.parse_xrefs()
         has_biotools_reference = any(x["reftype"] == "bio.tools" for x in xrefs)
diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt
index 4faa725a7df..d5572846ea2 100644
--- a/packages/app/requirements.txt
+++ b/packages/app/requirements.txt
@@ -9,7 +9,7 @@ celery
 kombu
 Beaker
 pykwalify
-pulsar-galaxy-lib
+pulsar-galaxy-lib>=0.14.7
 gxformat2
 Mako
 sqlitedict
diff --git a/test/unit/tool_util/test_required_files.py b/test/unit/tool_util/test_required_files.py
new file mode 100644
index 00000000000..f65ceca32e1
--- /dev/null
+++ b/test/unit/tool_util/test_required_files.py
@@ -0,0 +1,170 @@
+import os
+from pathlib import Path
+
+from .test_parsing import BaseLoaderTestCase
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+TEST_TOOL_DIRECTORIES = os.path.join(SCRIPT_DIRECTORY, "tool_directories")
+
+
+TOOL_REQUIRED_FILES_XML_1 = """
+<tool name="required_files_1" id="required_files_1" version="0.1.0">
+    <command>foo</command>
+    <required_files>
+        <include path="my_script.R" />
+    </required_files>
+    <inputs>
+        <input type="text" label="Text to parse." name="input1" />
+    </inputs>
+    <outputs>
+        <output type="integer" name="out1" from="output" />
+    </outputs>
+</tool>
+"""
+
+
+TOOL_REQUIRED_FILES_XML_2 = """
+<tool name="required_files_1" id="required_files_1" version="0.1.0">
+    <command>foo</command>
+    <required_files>
+        <include path="*.R" type="glob" />
+        <exclude path="other_script.R" />
+    </required_files>
+    <inputs>
+        <input type="text" label="Text to parse." name="input1" />
+    </inputs>
+    <outputs>
+        <output type="integer" name="out1" from="output" />
+    </outputs>
+</tool>
+"""
+
+
+TOOL_REQUIRED_FILES_XML_3 = """
+<tool name="required_files_1" id="required_files_1" version="0.1.0">
+    <command>foo</command>
+    <required_files>
+        <include path="*.R" type="glob" />
+    </required_files>
+    <inputs>
+        <input type="text" label="Text to parse." name="input1" />
+    </inputs>
+    <outputs>
+        <output type="integer" name="out1" from="output" />
+    </outputs>
+</tool>
+"""
+
+
+TOOL_REQUIRED_FILES_XML_4 = """
+<tool name="required_files_1" id="required_files_1" version="0.1.0">
+    <command>foo</command>
+    <required_files>
+        <include path=".*R" type="regex" />
+        <exclude path="other_script*" type="glob" />
+    </required_files>
+    <inputs>
+        <input type="text" label="Text to parse." name="input1" />
+    </inputs>
+    <outputs>
+        <output type="integer" name="out1" from="output" />
+    </outputs>
+</tool>
+"""
+
+
+TOOL_REQUIRED_FILES_XML_DISABLED_DEFAULT_EXCLUSIONS = """
+<tool name="required_files_1" id="required_files_1" version="0.1.0">
+    <command>foo</command>
+    <required_files extend_default_excludes="false">
+        <include path="*.R" type="glob" />
+    </required_files>
+    <inputs>
+        <input type="text" label="Text to parse." name="input1" />
+    </inputs>
+    <outputs>
+        <output type="integer" name="out1" from="output" />
+    </outputs>
+</tool>
+"""
+
+
+class BaseRequiredFilesTestCase(BaseLoaderTestCase):
+    source_file_name = "required_files.xml"
+
+    def _required_files(self, tool_directory: str):
+        tool_source = self._tool_source
+        required_files = tool_source.parse_required_files()
+        return required_files.find_required_files(tool_directory)
+
+
+# directly include just one file that is there
+class RequiredFiles1TestCase(BaseRequiredFilesTestCase):
+    source_contents = TOOL_REQUIRED_FILES_XML_1
+
+    def test_expected_files(self):
+        files = self._required_files(os.path.join(TEST_TOOL_DIRECTORIES, "r-tool-dir"))
+        assert len(files) == 1
+        assert "my_script.R" in files
+
+
+# include a glob and exclude a file in the glob
+class RequiredFiles2TestCase(BaseRequiredFilesTestCase):
+    source_contents = TOOL_REQUIRED_FILES_XML_2
+
+    def test_expected_files(self):
+        files = self._required_files(os.path.join(TEST_TOOL_DIRECTORIES, "r-tool-dir"))
+        assert len(files) == 1
+        assert "my_script.R" in files
+
+
+# include a glob with multiple matches
+class RequiredFiles3TestCase(BaseRequiredFilesTestCase):
+    source_contents = TOOL_REQUIRED_FILES_XML_3
+
+    def test_expected_files(self):
+        files = self._required_files(os.path.join(TEST_TOOL_DIRECTORIES, "r-tool-dir"))
+        assert len(files) == 2
+        assert "my_script.R" in files
+        assert "other_script.R" in files
+
+
+# include a file with regex and exclude with glob
+class RequiredFiles4TestCase(BaseRequiredFilesTestCase):
+    source_contents = TOOL_REQUIRED_FILES_XML_4
+
+    def test_expected_files(self):
+        files = self._required_files(os.path.join(TEST_TOOL_DIRECTORIES, "r-tool-dir"))
+        assert len(files) == 1
+        assert "my_script.R" in files
+
+
+class HgExcludedByDefaultTestCase(BaseRequiredFilesTestCase):
+    source_contents = TOOL_REQUIRED_FILES_XML_3
+
+    def test_expected_files(self):
+        repo_dir = setup_dir_with_repo(self.temp_directory)
+        files = self._required_files(repo_dir)
+        assert len(files) == 1
+        assert "my_script.R" in files
+
+
+class HgExclusionDisabledTestCase(BaseRequiredFilesTestCase):
+    source_contents = TOOL_REQUIRED_FILES_XML_DISABLED_DEFAULT_EXCLUSIONS
+
+    def test_expected_files(self):
+        repo_dir = setup_dir_with_repo(self.temp_directory)
+        files = self._required_files(repo_dir)
+        assert len(files) == 2
+        assert "my_script.R" in files
+        assert ".hg/index.R" in files
+
+
+def setup_dir_with_repo(tmp_dir):
+    repo = os.path.join(tmp_dir, "repo")
+    os.makedirs(repo)
+    hg = os.path.join(repo, ".hg")
+    os.makedirs(hg)
+    Path(hg, "index.R").touch()
+    Path(repo, "my_script.R").touch()
+    return repo
diff --git a/test/unit/tool_util/tool_directories/r-tool-dir/my_script.R b/test/unit/tool_util/tool_directories/r-tool-dir/my_script.R
new file mode 100644
index 00000000000..fee7cb0d403
--- /dev/null
+++ b/test/unit/tool_util/tool_directories/r-tool-dir/my_script.R
@@ -0,0 +1 @@
+my cool R script
diff --git a/test/unit/tool_util/tool_directories/r-tool-dir/other_script.R b/test/unit/tool_util/tool_directories/r-tool-dir/other_script.R
new file mode 100644
index 00000000000..6e7a2cc1241
--- /dev/null
+++ b/test/unit/tool_util/tool_directories/r-tool-dir/other_script.R
@@ -0,0 +1 @@
+another R script

From 2ffa18d836b40bebab911e59ab7b33e00c22e0d2 Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Tue, 10 Aug 2021 14:13:55 -0400
Subject: [PATCH 296/452] Update lib/galaxy/jobs/runners/pulsar.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Björn Grüning <bjoern@gruenings.eu>
---
 lib/galaxy/jobs/runners/pulsar.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index 15a6aa3c977..0dac01bdea7 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -773,7 +773,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
             dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:]
             # grab discovered outputs...
             dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns)
-            # grab too provided metadata (galaxy.json) also...
+            # grab tool provided metadata (galaxy.json) also...
             dynamic_outputs.append(re.escape(tool_provided_metadata_file_path))
         dynamic_file_sources = [
             {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"}

From b6f75b7ea64c324cbe94767bfdcb7ca904c29ec5 Mon Sep 17 00:00:00 2001
From: setlen <peter.selten98@gmail.com>
Date: Tue, 10 Aug 2021 20:15:22 +0200
Subject: [PATCH 297/452] Fix unused dependencies

---
 lib/galaxy/tools/error_reports/plugins/gitlab.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/tools/error_reports/plugins/gitlab.py b/lib/galaxy/tools/error_reports/plugins/gitlab.py
index f3615060581..12bf02b2207 100644
--- a/lib/galaxy/tools/error_reports/plugins/gitlab.py
+++ b/lib/galaxy/tools/error_reports/plugins/gitlab.py
@@ -2,9 +2,9 @@
 
 import logging
 import os
-import sys
 
 import requests
+import urllib.parse
 
 try:
     import gitlab
@@ -39,7 +39,8 @@ class GitLabPlugin(BaseGitPlugin):
 
         try:
             if gitlab is None:
-                raise Exception("GitLab error reporting plugin is configured, but gitlab is not installed. Please install python-gitlab.")
+                raise Exception("GitLab error reporting plugin is configured, but gitlab is not installed. "
+                                "Please install python-gitlab v2.10.0 or higher.")
             self.gitlab = self.gitlab_connect()
             self.gitlab.auth()
 
@@ -93,7 +94,7 @@ class GitLabPlugin(BaseGitPlugin):
                 # this will be detected by requests and used further down the line. Also cache this so everything is
                 # as fast as possible
                 ts_url = self._determine_ts_url(tool)
-                log.info("GitLab error reporting - Determined ToolShed is %s", ts_url)
+                log.info("GitLab error reporting - Determined ToolShed is {ts_url}")
 
                 # Find the repo inside the ToolShed
                 ts_repourl = self._get_gitrepo_from_ts(job, ts_url)
@@ -102,10 +103,10 @@ class GitLabPlugin(BaseGitPlugin):
                 if ts_repourl is not None and ts_repourl.endswith(".git"):
                     ts_repourl = ts_repourl[:-4]
 
-                log.info("GitLab error reporting - Determine ToolShed Repository URL: %s", ts_repourl)
+                log.info(f"GitLab error reporting - Determine ToolShed Repository URL: {ts_repourl}")
 
                 # Determine the GitLab project URL and the issue cache key
-                gitlab_projecturl = urlparse.urlparse(ts_repourl).path[1:] if (ts_repourl and not self.git_default_repo_only)\
+                gitlab_projecturl = urllib.parse.urlparse(ts_repourl).path[1:] if (ts_repourl and not self.git_default_repo_only)\
                     else "/".join((self.git_default_repo_owner, self.git_default_repo_name))
                 issue_cache_key = self._get_issue_cache_key(job, ts_repourl)
 
@@ -143,8 +144,7 @@ class GitLabPlugin(BaseGitPlugin):
                         gl_emailquery = self.gitlab.users.list(search=gl_useremail)
                         log.debug(f"GitLab error reporting - User list: {gl_emailquery}")
                         if len(gl_emailquery) > 0:
-                            log.debug("GitLab error reporting - Last Committer user ID: %d" %
-                                      gl_emailquery[0].get_id())
+                            log.debug("GitLab error reporting - Last Committer user ID: " % gl_emailquery[0].get_id())
                             self.git_username_id_cache[gl_useremail] = gl_emailquery[0].get_id()
                     gl_userid = self.git_username_id_cache.get(gl_useremail, None)
 

From c98cf15fd5885d94c4c4d9065b25be667a430290 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Tue, 10 Aug 2021 11:20:37 -0700
Subject: [PATCH 298/452] restore ability to bulk-update history visibility

---
 lib/galaxy/schema/schema.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 8c31eb11ada..98eb6b47436 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -632,6 +632,13 @@ class UpdateHistoryContentsBatchPayload(Model):
             "otherwise cannot delete uploading files, so it will raise an error."
         ),
     )
+    visible: Optional[bool] = Field(
+        default=False,
+        title="Visible",
+        description=(
+            "Show or hide history contents"
+        ),
+    )
 
 
 class HistoryBase(BaseModel):

From c1f39f6e9fb62b8b881b9bf393b5b8c168eaf948 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 17:20:46 -0400
Subject: [PATCH 299/452] Add generic form component, and redirect permissions
 route

---
 client/src/components/Form/FormGeneric.vue    | 140 ++++++++++++++++++
 .../components/providers/UrlDataProvider.js   |  17 +++
 client/src/entry/analysis/AnalysisRouter.js   |  11 +-
 3 files changed, 162 insertions(+), 6 deletions(-)
 create mode 100644 client/src/components/Form/FormGeneric.vue
 create mode 100644 client/src/components/providers/UrlDataProvider.js

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
new file mode 100644
index 00000000000..4a269dd173c
--- /dev/null
+++ b/client/src/components/Form/FormGeneric.vue
@@ -0,0 +1,140 @@
+<template>
+    <UrlDataProvider :url="url" v-slot="{ result, loading }" @error="onError">
+        <div v-if="!loading">
+            <b-alert v-if="result.message" :variant="resultMessageVariant" show>
+                {{ result.message }}
+            </b-alert>
+            <b-alert v-if="messageText" :variant="messageVariant" show>
+                {{ messageText }}
+            </b-alert>
+            <FormCard :title="result.title" :icon="result.icon">
+                <template v-slot:body>
+                    <FormDisplay :inputs="result.inputs" @onChange="onChange" />
+                </template>
+            </FormCard>
+            <div class="mt-2">
+                <b-button
+                    variant="primary"
+                    class="mr-1"
+                    v-b-tooltip.hover
+                    :title="result.submit_tooltip"
+                    @click="onSubmit(result)"
+                >
+                    <span :class="submitIcon(result)" />{{ submitTitle(result) | l }}
+                </b-button>
+                <b-button
+                    v-if="result.cancel_redirect"
+                    v-b-tooltip.hover
+                    :title="result.cancel_tooltip"
+                    @click="onCancel(result)"
+                >
+                    <span :class="cancelIcon(result)" />{{ cancelTitle(result) | l }}
+                </b-button>
+            </div>
+        </div>
+    </UrlDataProvider>
+</template>
+
+<script>
+import { getAppRoot } from "onload/loadConfig";
+import { UrlDataProvider } from "components/providers/UrlDataProvider";
+import FormCard from "components/Form/FormCard";
+import FormDisplay from "components/Form/FormDisplay";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faSave, faTimes } from "@fortawesome/free-solid-svg-icons";
+
+library.add(faSave, faTimes);
+
+export default {
+    props: {
+        url: {
+            type: String,
+            required: true,
+        },
+        redirect: {
+            type: String,
+            required: false,
+        },
+    },
+    components: {
+        FontAwesomeIcon,
+        FormCard,
+        FormDisplay,
+        UrlDataProvider,
+    },
+    data() {
+        return {
+            messageText: null,
+            messageVariant: null,
+            formData: {},
+        };
+    },
+    methods: {
+        resultMessageVariant(options) {
+            return result.status || "warning";
+        },
+        cancelTitle(options) {
+            return options.cancel_title || "Cancel";
+        },
+        cancelIcon(options) {
+            const icon = options.cancel_icon || "fa-times";
+            return `mr-1 fa ${icon}`;
+        },
+        submitTitle(options) {
+            return options.submit_title || "Save";
+        },
+        submitIcon(options) {
+            const icon = options.submit_icon || "fa-save";
+            return `mr-1 fa ${icon}`;
+        },
+        onChange(formData) {
+            this.formData = formData;
+        },
+        onCancel(options) {
+            window.location = `${getAppRoot()}${options.cancel_redirect}`;
+        },
+        onSubmit(options) {
+            /*
+            var self = this;
+        $.ajax({
+            url: getAppRoot() + self.url,
+            data: JSON.stringify(form.data.create()),
+            type: "PUT",
+            contentType: "application/json",
+        })
+            .done((response) => {
+                var params = {};
+                if (response.id) {
+                    params.id = response.id;
+                } else {
+                    params = {
+                        message: response.message,
+                        status: "success",
+                        persistent: false,
+                    };
+                }
+                if (self.redirect) {
+                    window.location = `${getAppRoot()}${self.redirect}?${$.param(params)}`;
+                } else {
+                    form.data.matchModel(response, (input, input_id) => {
+                        form.field_list[input_id].value(input.value);
+                    });
+                    self._showMessage(form, response.message);
+                }
+            })
+            .fail((response) => {
+                self._showMessage(form, {
+                    message: response.responseJSON.err_msg,
+                    status: "danger",
+                    persistent: false,
+                });
+            });*/
+        },
+        onError(error) {
+            this.messageText = `Failed to load resource ${this.url}.`;
+            this.messageVariant = "danger";
+        },
+    },
+};
+</script>
diff --git a/client/src/components/providers/UrlDataProvider.js b/client/src/components/providers/UrlDataProvider.js
new file mode 100644
index 00000000000..00d96d0c51b
--- /dev/null
+++ b/client/src/components/providers/UrlDataProvider.js
@@ -0,0 +1,17 @@
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
+import { rethrowSimple } from "utils/simple-error";
+
+async function urlData({ url }) {
+    console.log(url);
+    try {
+        const { data } = await axios.get(`${getAppRoot()}${url}`);
+        console.log(data);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}
+
+export const UrlDataProvider = SingleQueryProvider(urlData);
diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index 9c67d51ce28..987afcf71ef 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -16,6 +16,7 @@ import { getAppRoot } from "onload";
 import decodeUriComponent from "decode-uri-component";
 import Router from "layout/router";
 import ToolForm from "components/Tool/ToolForm";
+import FormGeneric from "components/Form/FormGeneric";
 import FormWrapper from "mvc/form/form-wrapper";
 import Sharing from "components/Sharing.vue";
 import UserPreferences from "components/User/UserPreferences.vue";
@@ -268,12 +269,10 @@ export const getAnalysisRouter = (Galaxy) => {
         },
 
         show_histories_permissions: function () {
-            this.page.display(
-                new FormWrapper.View({
-                    url: `history/permissions?id=${QueryStringParsing.get("id")}`,
-                    redirect: "histories/list",
-                })
-            );
+            this._display_vue_helper(FormGeneric, {
+                url: `history/permissions?id=${QueryStringParsing.get("id")}`,
+                redirect: "histories/list",
+            });
         },
 
         show_datasets: function () {

From daad9ff94490177c7361fb9b7f08abc58960665f Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 17:22:03 -0400
Subject: [PATCH 300/452] Remove font awesome important, icons are dynamic

---
 client/src/components/Form/FormGeneric.vue | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 4a269dd173c..08788291ed0 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -40,11 +40,6 @@ import { getAppRoot } from "onload/loadConfig";
 import { UrlDataProvider } from "components/providers/UrlDataProvider";
 import FormCard from "components/Form/FormCard";
 import FormDisplay from "components/Form/FormDisplay";
-import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
-import { library } from "@fortawesome/fontawesome-svg-core";
-import { faSave, faTimes } from "@fortawesome/free-solid-svg-icons";
-
-library.add(faSave, faTimes);
 
 export default {
     props: {

From 7aedfc91bf339afcfce1c30ae1ecae513dbf4c6e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 17:42:53 -0400
Subject: [PATCH 301/452] Add event handler to generic forms submit operation

---
 client/src/components/Form/FormGeneric.vue    | 39 ++++++-------------
 .../components/providers/UrlDataProvider.js   |  2 -
 2 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 08788291ed0..80ba47a3441 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -18,7 +18,7 @@
                     class="mr-1"
                     v-b-tooltip.hover
                     :title="result.submit_tooltip"
-                    @click="onSubmit(result)"
+                    @click="onSubmit()"
                 >
                     <span :class="submitIcon(result)" />{{ submitTitle(result) | l }}
                 </b-button>
@@ -37,6 +37,7 @@
 
 <script>
 import { getAppRoot } from "onload/loadConfig";
+import { submitData } from "./services";
 import { UrlDataProvider } from "components/providers/UrlDataProvider";
 import FormCard from "components/Form/FormCard";
 import FormDisplay from "components/Form/FormDisplay";
@@ -53,7 +54,6 @@ export default {
         },
     },
     components: {
-        FontAwesomeIcon,
         FormCard,
         FormDisplay,
         UrlDataProvider,
@@ -89,17 +89,9 @@ export default {
         onCancel(options) {
             window.location = `${getAppRoot()}${options.cancel_redirect}`;
         },
-        onSubmit(options) {
-            /*
-            var self = this;
-        $.ajax({
-            url: getAppRoot() + self.url,
-            data: JSON.stringify(form.data.create()),
-            type: "PUT",
-            contentType: "application/json",
-        })
-            .done((response) => {
-                var params = {};
+        onSubmit() {
+            submitData(this.url, this.formData).then((response) => {
+                let params = {};
                 if (response.id) {
                     params.id = response.id;
                 } else {
@@ -109,22 +101,15 @@ export default {
                         persistent: false,
                     };
                 }
-                if (self.redirect) {
-                    window.location = `${getAppRoot()}${self.redirect}?${$.param(params)}`;
+                if (this.redirect) {
+                    window.location = `${getAppRoot()}${this.redirect}?${$.param(params)}`;
                 } else {
-                    form.data.matchModel(response, (input, input_id) => {
-                        form.field_list[input_id].value(input.value);
-                    });
-                    self._showMessage(form, response.message);
+                    /*form.data.matchModel(response, (input, input_id) => {
+                            form.field_list[input_id].value(input.value);
+                        });
+                        self._showMessage(form, response.message);*/
                 }
-            })
-            .fail((response) => {
-                self._showMessage(form, {
-                    message: response.responseJSON.err_msg,
-                    status: "danger",
-                    persistent: false,
-                });
-            });*/
+            }, this.onError);
         },
         onError(error) {
             this.messageText = `Failed to load resource ${this.url}.`;
diff --git a/client/src/components/providers/UrlDataProvider.js b/client/src/components/providers/UrlDataProvider.js
index 00d96d0c51b..a7051ceef71 100644
--- a/client/src/components/providers/UrlDataProvider.js
+++ b/client/src/components/providers/UrlDataProvider.js
@@ -4,10 +4,8 @@ import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
 import { rethrowSimple } from "utils/simple-error";
 
 async function urlData({ url }) {
-    console.log(url);
     try {
         const { data } = await axios.get(`${getAppRoot()}${url}`);
-        console.log(data);
         return data;
     } catch (e) {
         rethrowSimple(e);

From 6884674d3171ae1434ee2895ce13ef6e315acea0 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 17:52:21 -0400
Subject: [PATCH 302/452] Replace all form-wrapper instances with form generic
 vue component, remove legacy module

---
 client/src/components/Form/FormGeneric.vue  |   2 +-
 client/src/entry/admin/AdminRouter.js       |   4 +-
 client/src/entry/analysis/AnalysisRouter.js |  58 ++++-----
 client/src/mvc/form/form-wrapper.js         | 123 --------------------
 4 files changed, 27 insertions(+), 160 deletions(-)
 delete mode 100644 client/src/mvc/form/form-wrapper.js

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 80ba47a3441..21ad8aec281 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -112,7 +112,7 @@ export default {
             }, this.onError);
         },
         onError(error) {
-            this.messageText = `Failed to load resource ${this.url}.`;
+            this.messageText = error || `Failed to load resource ${this.url}.`;
             this.messageVariant = "danger";
         },
     },
diff --git a/client/src/entry/admin/AdminRouter.js b/client/src/entry/admin/AdminRouter.js
index 62a8147de41..2b55152cb84 100644
--- a/client/src/entry/admin/AdminRouter.js
+++ b/client/src/entry/admin/AdminRouter.js
@@ -1,7 +1,7 @@
 import { getGalaxyInstance } from "app";
 import { getAppRoot } from "onload";
 import _l from "utils/localization";
-import FormWrapper from "mvc/form/form-wrapper";
+import FormGeneric from "components/Form/FormGeneric";
 import GridView from "mvc/grid/grid-view";
 import QueryStringParsing from "utils/query-string-parsing";
 import Router from "layout/router";
@@ -225,7 +225,7 @@ export const getAdminRouter = (Galaxy, options) => {
                     redirect: "admin/forms",
                 },
             };
-            this.page.display(new FormWrapper.View(form_defs[form_id]));
+            this._display_vue_helper(FormGeneric, form_defs[form_id]);
         },
     });
 };
diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index 987afcf71ef..cf27c1a4124 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -17,7 +17,6 @@ import decodeUriComponent from "decode-uri-component";
 import Router from "layout/router";
 import ToolForm from "components/Tool/ToolForm";
 import FormGeneric from "components/Form/FormGeneric";
-import FormWrapper from "mvc/form/form-wrapper";
 import Sharing from "components/Sharing.vue";
 import UserPreferences from "components/User/UserPreferences.vue";
 import DatasetList from "components/Dataset/DatasetList.vue";
@@ -114,7 +113,6 @@ export const getAnalysisRouter = (Galaxy) => {
                 container.active_tab = active_tab;
             }
             this.page.display(container, noPadding);
-
             const mountFn = mountVueComponent(component);
             return mountFn(propsData, container);
         },
@@ -141,7 +139,7 @@ export const getAnalysisRouter = (Galaxy) => {
 
         show_user_form: function (form_id, params) {
             const model = getUserPreferencesModel(params.id);
-            this.page.display(new FormWrapper.View(_.extend(model[form_id], { active_tab: "user" })));
+            this._display_vue_helper(FormGeneric, _.extend(model[form_id], { active_tab: "user" }));
         },
 
         show_interactivetool_list: function () {
@@ -175,13 +173,11 @@ export const getAnalysisRouter = (Galaxy) => {
         },
 
         show_visualizations_edit: function () {
-            this.page.display(
-                new FormWrapper.View({
-                    url: `visualization/edit?id=${QueryStringParsing.get("id")}`,
-                    redirect: "visualizations/list",
-                    active_tab: "visualization",
-                })
-            );
+            this._display_vue_helper(FormGeneric, {
+                url: `visualization/edit?id=${QueryStringParsing.get("id")}`,
+                redirect: "visualizations/list",
+                active_tab: "visualization",
+            });
         },
 
         show_visualizations_sharing: function () {
@@ -297,23 +293,19 @@ export const getAnalysisRouter = (Galaxy) => {
             if (invocation_id) {
                 url += `?invocation_id=${invocation_id}`;
             }
-            this.page.display(
-                new FormWrapper.View({
-                    url: url,
-                    redirect: "pages/list",
-                    active_tab: "user",
-                })
-            );
+            this._display_vue_helper(FormGeneric, {
+                url: url,
+                redirect: "pages/list",
+                active_tab: "user",
+            });
         },
 
         show_pages_edit: function () {
-            this.page.display(
-                new FormWrapper.View({
-                    url: `page/edit?id=${QueryStringParsing.get("id")}`,
-                    redirect: "pages/list",
-                    active_tab: "user",
-                })
-            );
+            this._display_vue_helper(FormGeneric, {
+                url: `page/edit?id=${QueryStringParsing.get("id")}`,
+                redirect: "pages/list",
+                active_tab: "user",
+            });
         },
 
         show_pages_sharing: function () {
@@ -333,16 +325,14 @@ export const getAnalysisRouter = (Galaxy) => {
         },
 
         show_workflows_create: function () {
-            this.page.display(
-                new FormWrapper.View({
-                    url: "workflow/create",
-                    redirect: "workflow/editor",
-                    active_tab: "workflow",
-                    submit_title: "Create",
-                    submit_icon: "fa-check",
-                    cancel_redirect: "workflows/list",
-                })
-            );
+            this._display_vue_helper(FormGeneric, {
+                url: "workflow/create",
+                redirect: "workflow/editor",
+                active_tab: "workflow",
+                submit_title: "Create",
+                submit_icon: "fa-check",
+                cancel_redirect: "workflows/list",
+            });
         },
 
         show_workflows_run: function () {
diff --git a/client/src/mvc/form/form-wrapper.js b/client/src/mvc/form/form-wrapper.js
deleted file mode 100644
index 79612b2c54a..00000000000
--- a/client/src/mvc/form/form-wrapper.js
+++ /dev/null
@@ -1,123 +0,0 @@
-/** Generic form view */
-import $ from "jquery";
-import Backbone from "backbone";
-import { getAppRoot } from "onload/loadConfig";
-import Form from "mvc/form/form-view";
-import Ui from "mvc/ui/ui-misc";
-
-var View = Backbone.View.extend({
-    initialize: function (options) {
-        this.model = new Backbone.Model(options);
-        this.url = this.model.get("url");
-        this.redirect = this.model.get("redirect");
-        if (options && options.active_tab) {
-            this.active_tab = options.active_tab;
-        }
-        this.setElement("<div/>");
-        this.render();
-    },
-
-    render: function () {
-        var self = this;
-        $.ajax({
-            url: getAppRoot() + this.url,
-            type: "GET",
-        })
-            .done((response) => {
-                var options = $.extend({}, self.model.attributes, response);
-                var form = new Form({
-                    title: options.title,
-                    title_id: options.title_id,
-                    message: options.message,
-                    status: options.status || "warning",
-                    icon: options.icon,
-                    initial_errors: true,
-                    errors: options.errors,
-                    inputs: options.inputs,
-                    buttons: {
-                        cancel:
-                            options.cancel_redirect &&
-                            new Ui.Button({
-                                tooltip: options.cancel_tooltip,
-                                title: options.cancel_title || "Cancel",
-                                icon: options.cancel_icon || "fa-times",
-                                onclick: function () {
-                                    window.location = `${getAppRoot()}${options.cancel_redirect}`;
-                                },
-                            }),
-                        submit: new Ui.Button({
-                            tooltip: options.submit_tooltip,
-                            title: options.submit_title || "Save",
-                            icon: options.submit_icon || "fa-save",
-                            cls: "btn btn-primary",
-                            onclick: function () {
-                                self._submit(form);
-                            },
-                        }),
-                    },
-                });
-                self.$el.empty().append(form.$el);
-            })
-            .fail((response) => {
-                self.$el.empty().append(
-                    new Ui.Message({
-                        message: `Failed to load resource ${self.url}.`,
-                        status: "danger",
-                        persistent: true,
-                    }).$el
-                );
-            });
-    },
-
-    _submit: function (form) {
-        var self = this;
-        $.ajax({
-            url: getAppRoot() + self.url,
-            data: JSON.stringify(form.data.create()),
-            type: "PUT",
-            contentType: "application/json",
-        })
-            .done((response) => {
-                var params = {};
-                if (response.id) {
-                    params.id = response.id;
-                } else {
-                    params = {
-                        message: response.message,
-                        status: "success",
-                        persistent: false,
-                    };
-                }
-                if (self.redirect) {
-                    window.location = `${getAppRoot()}${self.redirect}?${$.param(params)}`;
-                } else {
-                    form.data.matchModel(response, (input, input_id) => {
-                        form.field_list[input_id].value(input.value);
-                    });
-                    self._showMessage(form, response.message);
-                }
-            })
-            .fail((response) => {
-                self._showMessage(form, {
-                    message: response.responseJSON.err_msg,
-                    status: "danger",
-                    persistent: false,
-                });
-            });
-    },
-
-    _showMessage: function (form, options) {
-        var $panel = form.$el
-            .parents()
-            .filter(function () {
-                return ["auto", "scroll"].indexOf($(this).css("overflow")) != -1;
-            })
-            .first();
-        $panel.animate({ scrollTop: 0 }, 500);
-        form.message.update(options);
-    },
-});
-
-export default {
-    View: View,
-};

From 520eca9228580ec8fd786bdb833e851c497c5aac Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 18:02:54 -0400
Subject: [PATCH 303/452] Remove unused legacy condition for specific error
 highlighting in generic wrapper

---
 client/src/components/Form/FormGeneric.vue | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 21ad8aec281..0ce95fd3f6c 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -1,7 +1,7 @@
 <template>
     <UrlDataProvider :url="url" v-slot="{ result, loading }" @error="onError">
         <div v-if="!loading">
-            <b-alert v-if="result.message" :variant="resultMessageVariant" show>
+            <b-alert v-if="result.message" :variant="resultMessageVariant(result)" show>
                 {{ result.message }}
             </b-alert>
             <b-alert v-if="messageText" :variant="messageVariant" show>
@@ -50,7 +50,7 @@ export default {
         },
         redirect: {
             type: String,
-            required: false,
+            required: true,
         },
     },
     components: {
@@ -67,7 +67,7 @@ export default {
     },
     methods: {
         resultMessageVariant(options) {
-            return result.status || "warning";
+            return options.status || "warning";
         },
         cancelTitle(options) {
             return options.cancel_title || "Cancel";
@@ -101,14 +101,7 @@ export default {
                         persistent: false,
                     };
                 }
-                if (this.redirect) {
-                    window.location = `${getAppRoot()}${this.redirect}?${$.param(params)}`;
-                } else {
-                    /*form.data.matchModel(response, (input, input_id) => {
-                            form.field_list[input_id].value(input.value);
-                        });
-                        self._showMessage(form, response.message);*/
-                }
+                window.location = `${getAppRoot()}${this.redirect}?${$.param(params)}`;
             }, this.onError);
         },
         onError(error) {

From 1c278327b626f1210e0372de9c94fde066382b53 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 18:16:50 -0400
Subject: [PATCH 304/452] Handle title and icon parsing from client in generic
 form component

---
 client/src/components/Form/FormGeneric.vue | 42 +++++++++++++++-------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 0ce95fd3f6c..c5cee06886f 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -1,15 +1,15 @@
 <template>
-    <UrlDataProvider :url="url" v-slot="{ result, loading }" @error="onError">
+    <UrlDataProvider :url="url" v-slot="{ result: config, loading }" @error="onError">
         <div v-if="!loading">
-            <b-alert v-if="result.message" :variant="resultMessageVariant(result)" show>
-                {{ result.message }}
+            <b-alert v-if="config.message" :variant="configMessageVariant(config)" show>
+                {{ config.message }}
             </b-alert>
             <b-alert v-if="messageText" :variant="messageVariant" show>
                 {{ messageText }}
             </b-alert>
-            <FormCard :title="result.title" :icon="result.icon">
+            <FormCard :title="configTitle(config)" :icon="configIcon(config)">
                 <template v-slot:body>
-                    <FormDisplay :inputs="result.inputs" @onChange="onChange" />
+                    <FormDisplay :inputs="config.inputs" @onChange="onChange" />
                 </template>
             </FormCard>
             <div class="mt-2">
@@ -17,18 +17,18 @@
                     variant="primary"
                     class="mr-1"
                     v-b-tooltip.hover
-                    :title="result.submit_tooltip"
+                    :title="config.submit_tooltip"
                     @click="onSubmit()"
                 >
-                    <span :class="submitIcon(result)" />{{ submitTitle(result) | l }}
+                    <span :class="submitIcon(config)" />{{ submitTitle(config) | l }}
                 </b-button>
                 <b-button
-                    v-if="result.cancel_redirect"
+                    v-if="config.cancel_redirect"
                     v-b-tooltip.hover
-                    :title="result.cancel_tooltip"
-                    @click="onCancel(result)"
+                    :title="config.cancel_tooltip"
+                    @click="onCancel(config)"
                 >
-                    <span :class="cancelIcon(result)" />{{ cancelTitle(result) | l }}
+                    <span :class="cancelIcon(config)" />{{ cancelTitle(config) | l }}
                 </b-button>
             </div>
         </div>
@@ -44,6 +44,18 @@ import FormDisplay from "components/Form/FormDisplay";
 
 export default {
     props: {
+        id: {
+            type: String,
+            required: false,
+        },
+        title: {
+            type: String,
+            required: false,
+        },
+        icon: {
+            type: String,
+            required: false,
+        },
         url: {
             type: String,
             required: true,
@@ -66,7 +78,13 @@ export default {
         };
     },
     methods: {
-        resultMessageVariant(options) {
+        configTitle(options) {
+            return this.title || options.title;
+        },
+        configIcon(options) {
+            return this.icon || options.icon;
+        },
+        configMessageVariant(options) {
             return options.status || "warning";
         },
         cancelTitle(options) {

From 89bef05915a51caf36a1e8bee2867267bb58df84 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 18:19:13 -0400
Subject: [PATCH 305/452] Remove unsued custom tooltip option from generic form
 component

---
 client/src/components/Form/FormGeneric.vue | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index c5cee06886f..2f121ae7fe5 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -13,21 +13,10 @@
                 </template>
             </FormCard>
             <div class="mt-2">
-                <b-button
-                    variant="primary"
-                    class="mr-1"
-                    v-b-tooltip.hover
-                    :title="config.submit_tooltip"
-                    @click="onSubmit()"
-                >
+                <b-button variant="primary" class="mr-1" @click="onSubmit()">
                     <span :class="submitIcon(config)" />{{ submitTitle(config) | l }}
                 </b-button>
-                <b-button
-                    v-if="config.cancel_redirect"
-                    v-b-tooltip.hover
-                    :title="config.cancel_tooltip"
-                    @click="onCancel(config)"
-                >
+                <b-button v-if="config.cancel_redirect" @click="onCancel(config)">
                     <span :class="cancelIcon(config)" />{{ cancelTitle(config) | l }}
                 </b-button>
             </div>

From d1d6c166445c0885b705233d842eb3e5460ac0cc Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 18:32:00 -0400
Subject: [PATCH 306/452] Remove unused portlet and message dependencies from
 form-view

---
 client/src/mvc/form/form-view.js | 30 +-----------------------------
 1 file changed, 1 insertion(+), 29 deletions(-)

diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index 93ac9c7956d..a6cd52efbca 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -4,7 +4,6 @@
 import $ from "jquery";
 import _ from "underscore";
 import Backbone from "backbone";
-import Portlet from "mvc/ui/ui-portlet";
 import Ui from "mvc/ui/ui-misc";
 import FormSection from "mvc/form/form-section";
 import FormData from "mvc/form/form-data";
@@ -13,11 +12,7 @@ export default Backbone.View.extend({
     initialize: function (options) {
         this.model = new Backbone.Model({
             initial_errors: false,
-            cls: "ui-portlet",
-            icon: null,
             always_refresh: true,
-            status: "warning",
-            hide_operations: false,
             onchange: function () {},
         }).set(options);
         this.setElement(options.el || "<div/>");
@@ -52,8 +47,6 @@ export default Backbone.View.extend({
         var input_element = this.element_list[input_id];
         if (input_element) {
             input_element.error(message || "Please verify this option.");
-            this.portlet.expand();
-            this.trigger("expand", input_id);
             if (!silent) {
                 var $panel = this.$el
                     .parents()
@@ -126,33 +119,12 @@ export default Backbone.View.extend({
     _renderForm: function () {
         $(".tooltip").remove();
         var options = this.model.attributes;
-        this.message = new Ui.UnescapedMessage();
         this.section = new FormSection.View(this, {
             inputs: options.inputs,
         });
-        this.portlet = new Portlet.View({
-            icon: options.icon,
-            title: options.title,
-            title_id: options.title_id,
-            operations: !options.hide_operations && options.operations,
-            cls: options.cls,
-            buttons: options.buttons,
-            collapsible: options.collapsible,
-            collapsed: options.collapsed,
-            onchange_title: options.onchange_title,
-        });
-        this.portlet.append(this.message.$el);
-        this.portlet.append(this.section.$el);
         this.$el.empty();
         if (options.inputs) {
-            this.$el.append(this.portlet.$el);
-        }
-        if (options.message) {
-            this.message.update({
-                persistent: true,
-                status: options.status,
-                message: options.message,
-            });
+            this.$el.append(this.section.$el);
         }
         console.debug("form-view::initialize()", "Completed");
     },

From 3bec81d3d1238c7219c8afc42364a1b4b9d38b4a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sat, 7 Aug 2021 19:41:51 -0400
Subject: [PATCH 307/452] Fix missing history rename form mounting

---
 client/src/entry/analysis/AnalysisRouter.js | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index cf27c1a4124..8e78eabc312 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -228,12 +228,10 @@ export const getAnalysisRouter = (Galaxy) => {
         },
 
         show_histories_rename: function () {
-            this.page.display(
-                new FormWrapper.View({
-                    url: `history/rename?id=${QueryStringParsing.get("id")}`,
-                    redirect: "histories/list",
-                })
-            );
+            this._display_vue_helper(FormGeneric, {
+                url: `history/rename?id=${QueryStringParsing.get("id")}`,
+                redirect: "histories/list",
+            });
         },
 
         show_histories_sharing: function () {

From bd74606468db66f692b28fb7ee7e597ea80ad000 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 00:18:57 -0400
Subject: [PATCH 308/452] Remove unused ui module import in form-view

---
 client/src/mvc/form/form-view.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index a6cd52efbca..6954afc9e6f 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -4,7 +4,6 @@
 import $ from "jquery";
 import _ from "underscore";
 import Backbone from "backbone";
-import Ui from "mvc/ui/ui-misc";
 import FormSection from "mvc/form/form-section";
 import FormData from "mvc/form/form-data";
 

From 28524c73f61f335b07d43586411a9b1dd0d3c732 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 00:21:03 -0400
Subject: [PATCH 309/452] Add service module to generic form component

---
 client/src/components/Form/services.js | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 client/src/components/Form/services.js

diff --git a/client/src/components/Form/services.js b/client/src/components/Form/services.js
new file mode 100644
index 00000000000..ca3a1fd24f7
--- /dev/null
+++ b/client/src/components/Form/services.js
@@ -0,0 +1,12 @@
+import axios from "axios";
+import { rethrowSimple } from "utils/simple-error";
+import { getAppRoot } from "onload/loadConfig";
+
+export async function submitData(url, payload) {
+    try {
+        const { data } = await axios.put(`${getAppRoot()}${url}`, payload);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}

From e95a106143f47cd06f7b2e35aa9f299d8b86a1a7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 00:55:27 -0400
Subject: [PATCH 310/452] Add scrolling, replace vue scroll with js-native
 instant scrolling

---
 client/package.json                        |  1 -
 client/src/components/Form/FormGeneric.vue |  1 +
 client/src/components/Tool/ToolForm.vue    |  3 +--
 client/src/mvc/form/form-view.js           |  1 -
 client/yarn.lock                           | 12 ------------
 5 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/client/package.json b/client/package.json
index 2c70f9711bf..0011ab19a15 100644
--- a/client/package.json
+++ b/client/package.json
@@ -82,7 +82,6 @@
     "vue-observe-visibility": "^1.0.0",
     "vue-router": "^3.3.1",
     "vue-rx": "^6.2.0",
-    "vue-scrollto": "^2.20.0",
     "vuedraggable": "2.24.1",
     "vueisotope": "^3.1.2",
     "vuex": "^3.4.0",
diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 2f121ae7fe5..bbc7e5a8b50 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -114,6 +114,7 @@ export default {
         onError(error) {
             this.messageText = error || `Failed to load resource ${this.url}.`;
             this.messageVariant = "danger";
+            document.querySelector(".center-panel").scrollTop = 0;
         },
     },
 };
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 33aadafb650..60ecf918f8e 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -89,7 +89,6 @@
 </template>
 
 <script>
-import Scroller from "vue-scrollto";
 import { getGalaxyInstance } from "app";
 import { getToolFormData, updateToolFormData, submitJob } from "./services";
 import { allowCachedJobs } from "./utilities";
@@ -287,7 +286,7 @@ export default {
                     if ([true, "true"].includes(config.enable_tool_recommendations)) {
                         this.showRecommendation = true;
                     }
-                    Scroller.scrollTo("body");
+                    document.querySelector(".center-panel").scrollTop = 0;
                 },
                 (e) => {
                     this.showExecuting = false;
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index 6954afc9e6f..a7590991442 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -125,6 +125,5 @@ export default Backbone.View.extend({
         if (options.inputs) {
             this.$el.append(this.section.$el);
         }
-        console.debug("form-view::initialize()", "Completed");
     },
 });
diff --git a/client/yarn.lock b/client/yarn.lock
index 829749ec72d..52d83e861ff 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -3248,11 +3248,6 @@ better-assert@~1.0.0:
   dependencies:
     callsite "1.0.0"
 
-bezier-easing@2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/bezier-easing/-/bezier-easing-2.1.0.tgz#c04dfe8b926d6ecaca1813d69ff179b7c2025d86"
-  integrity sha1-wE3+i5JtbsrKGBPWn/F5t8ICXYY=
-
 big.js@^3.1.3:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@@ -15179,13 +15174,6 @@ vue-rx@^6.2.0:
   resolved "https://registry.yarnpkg.com/vue-rx/-/vue-rx-6.2.0.tgz#c3b86462b252626edd16417385bb9054a3a23acb"
   integrity sha512-tpKUcqS5IUYS+HsdbR5TlE5LL9PK4zwlplEtmMeydnbqaUTa9ciLMplJXAtFSiQw1vuURoyEJmCXqMxaVEIloQ==
 
-vue-scrollto@^2.20.0:
-  version "2.20.0"
-  resolved "https://registry.yarnpkg.com/vue-scrollto/-/vue-scrollto-2.20.0.tgz#3ba52239a62710c97003d12d4393f1f3393cd5cc"
-  integrity sha512-7i+AGKJTThZnMEkhIPgrZjyAX+fXV7/rGdg+CV283uZZwCxwiMXaBLTmIc5RTA4uwGnT+E6eJle3mXQfM2OD3A==
-  dependencies:
-    bezier-easing "2.1.0"
-
 vue-style-loader@^4.1.0:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/vue-style-loader/-/vue-style-loader-4.1.2.tgz#dedf349806f25ceb4e64f3ad7c0a44fba735fcf8"

From 48ee8b699356288af53447242326d4a423531825 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 01:11:21 -0400
Subject: [PATCH 311/452] Scroll to top of right panel when active node in
 workflow editor changes

---
 client/src/components/Workflow/Editor/Index.vue | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue
index 5093697d162..102fbe755ba 100644
--- a/client/src/components/Workflow/Editor/Index.vue
+++ b/client/src/components/Workflow/Editor/Index.vue
@@ -113,7 +113,7 @@
                                 />
                             </div>
                         </div>
-                        <div class="unified-panel-body workflow-right">
+                        <div class="unified-panel-body workflow-right" ref="right-panel">
                             <div class="m-1">
                                 <FormTool
                                     v-if="hasActiveNodeTool"
@@ -335,6 +335,7 @@ export default {
                 node.makeActive();
                 this.activeNode = node;
                 this.canvasManager.drawOverview();
+                this.$refs["right-panel"].scrollTop = 0;
             }
         },
         onDeactivate() {

From b279170bab33174d829cede767a7cb608e86ea5a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 04:40:44 -0400
Subject: [PATCH 312/452] Assemble search string without jquery, add id to
 generic form

---
 client/src/components/Form/FormGeneric.vue | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index bbc7e5a8b50..f9c2f64ed3d 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -1,6 +1,6 @@
 <template>
     <UrlDataProvider :url="url" v-slot="{ result: config, loading }" @error="onError">
-        <div v-if="!loading">
+        <div :id="id" v-if="!loading">
             <b-alert v-if="config.message" :variant="configMessageVariant(config)" show>
                 {{ config.message }}
             </b-alert>
@@ -13,7 +13,7 @@
                 </template>
             </FormCard>
             <div class="mt-2">
-                <b-button variant="primary" class="mr-1" @click="onSubmit()">
+                <b-button id="submit" variant="primary" class="mr-1" @click="onSubmit()">
                     <span :class="submitIcon(config)" />{{ submitTitle(config) | l }}
                 </b-button>
                 <b-button v-if="config.cancel_redirect" @click="onCancel(config)">
@@ -108,7 +108,8 @@ export default {
                         persistent: false,
                     };
                 }
-                window.location = `${getAppRoot()}${this.redirect}?${$.param(params)}`;
+                const urlParams = new URLSearchParams(params);
+                window.location = `${getAppRoot()}${this.redirect}?${urlParams.toString()}`;
             }, this.onError);
         },
         onError(error) {

From 25725bf4642b3e1488fd8bba6cd56c2cccbea373 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 13:26:48 -0400
Subject: [PATCH 313/452] Fix selenium test cases, allow parameter replacement
 in generic forms

---
 client/src/components/Form/FormGeneric.vue    | 24 +++++++++++++++----
 .../mvc/visualization/chart/views/groups.js   |  4 ++--
 .../mvc/visualization/chart/views/settings.js |  4 ++--
 lib/galaxy/selenium/navigation.yml            |  2 +-
 .../webapps/galaxy/controllers/admin.py       |  6 +++--
 5 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index f9c2f64ed3d..80dc77f113e 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -9,7 +9,7 @@
             </b-alert>
             <FormCard :title="configTitle(config)" :icon="configIcon(config)">
                 <template v-slot:body>
-                    <FormDisplay :inputs="config.inputs" @onChange="onChange" />
+                    <FormDisplay :inputs="config.inputs" :replace-params="replaceParams" @onChange="onChange" />
                 </template>
             </FormCard>
             <div class="mt-2">
@@ -28,6 +28,7 @@
 import { getAppRoot } from "onload/loadConfig";
 import { submitData } from "./services";
 import { UrlDataProvider } from "components/providers/UrlDataProvider";
+import { visitInputs } from "components/Form/utilities";
 import FormCard from "components/Form/FormCard";
 import FormDisplay from "components/Form/FormDisplay";
 
@@ -64,6 +65,7 @@ export default {
             messageText: null,
             messageVariant: null,
             formData: {},
+            replaceParams: null,
         };
     },
     methods: {
@@ -108,13 +110,25 @@ export default {
                         persistent: false,
                     };
                 }
-                const urlParams = new URLSearchParams(params);
-                window.location = `${getAppRoot()}${this.redirect}?${urlParams.toString()}`;
+                if (this.redirect) {
+                    const urlParams = new URLSearchParams(params);
+                    window.location = `${getAppRoot()}${this.redirect}?${urlParams.toString()}`;
+                } else {
+                    const replaceParams = {};
+                    visitInputs(response.inputs, (input, name) => {
+                        replaceParams[name] = input.value;
+                    });
+                    this.replaceParams = replaceParams;
+                    this.showMessage(response.message);
+                }
             }, this.onError);
         },
         onError(error) {
-            this.messageText = error || `Failed to load resource ${this.url}.`;
-            this.messageVariant = "danger";
+            this.showMessage(error || `Failed to load resource ${this.url}.`, "danger");
+        },
+        showMessage(message, variant = "success") {
+            this.messageText = message;
+            this.messageVariant = variant;
             document.querySelector(".center-panel").scrollTop = 0;
         },
     },
diff --git a/client/src/mvc/visualization/chart/views/groups.js b/client/src/mvc/visualization/chart/views/groups.js
index d26259fcba8..9d9a543e5da 100644
--- a/client/src/mvc/visualization/chart/views/groups.js
+++ b/client/src/mvc/visualization/chart/views/groups.js
@@ -6,7 +6,7 @@ import Backbone from "backbone";
 import { getAppRoot } from "onload/loadConfig";
 import Utils from "utils/utils";
 import Repeat from "mvc/form/form-repeat";
-import FormData from "mvc/form/form-data";
+import { visitInputs } from "components/Form/utilities";
 import FormDisplay from "components/Form/FormDisplay";
 import { appendVueComponent } from "utils/mountVueComponent";
 
@@ -34,7 +34,7 @@ var GroupView = Backbone.View.extend({
                     cache: true,
                     success: function (dataset) {
                         var data_columns = {};
-                        FormData.visitInputs(inputs, function (input, prefixed) {
+                        visitInputs(inputs, function (input, prefixed) {
                             if (input.type == "data_column") {
                                 data_columns[prefixed] = Utils.clone(input);
                                 var columns = [];
diff --git a/client/src/mvc/visualization/chart/views/settings.js b/client/src/mvc/visualization/chart/views/settings.js
index e78404b223d..5bcc63ef044 100644
--- a/client/src/mvc/visualization/chart/views/settings.js
+++ b/client/src/mvc/visualization/chart/views/settings.js
@@ -2,7 +2,7 @@
 import _ from "underscore";
 import Backbone from "backbone";
 import Utils from "utils/utils";
-import FormData from "mvc/form/form-data";
+import { visitInputs } from "components/Form/utilities";
 import FormDisplay from "components/Form/FormDisplay";
 import { appendVueComponent } from "utils/mountVueComponent";
 
@@ -32,7 +32,7 @@ export default Backbone.View.extend({
         }
         this.$el.empty();
         if (_.size(inputs) > 0) {
-            FormData.visitInputs(inputs, function (input, name) {
+            visitInputs(inputs, function (input, name) {
                 var model_value = self.chart.settings.get(name);
                 if (model_value !== undefined && !input.hidden) {
                     input.value = model_value;
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 8e69f39e21c..ef60882e9b0 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -619,7 +619,7 @@ admin:
     registration_form: 'form#registration'
     groups_grid: '#groups-grid'
     roles_grid: '#roles-grid'
-    groups_create_view: '#create-group'
+    groups_create_view: '#submit'
 
 libraries:
 
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index 53545bd9446..1f429bd5c5e 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -957,7 +957,8 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
                     build_select_input('in_users', 'Users', all_users, []), {
                     'name': 'auto_create',
                     'label': 'Create a new group of the same name for this role:',
-                    'type': 'boolean'
+                    'type': 'boolean',
+                    'optional': True,
                 }]}
         else:
             name = util.restore_text(payload.get('name', ''))
@@ -1270,7 +1271,8 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
                     build_select_input('in_users', 'Users', all_users, []), {
                     'name': 'auto_create',
                     'label': 'Create a new role of the same name for this group:',
-                    'type': 'boolean'
+                    'type': 'boolean',
+                    'optional': True,
                 }]
             }
         else:

From 15f6082944ce738e3b2795b8944a7ccb0b7647b8 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 14:47:04 -0400
Subject: [PATCH 314/452] Align option parsing to generic form, remove unused
 props

---
 client/src/components/Form/FormGeneric.vue    | 34 ++++++++++---------
 .../components/User/UserPreferencesModel.js   | 16 ++++-----
 client/src/entry/admin/AdminRouter.js         |  2 +-
 client/src/entry/analysis/AnalysisRouter.js   |  6 ++--
 .../webapps/galaxy/controllers/forms.py       |  2 +-
 5 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 80dc77f113e..05fd226e14c 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -14,10 +14,10 @@
             </FormCard>
             <div class="mt-2">
                 <b-button id="submit" variant="primary" class="mr-1" @click="onSubmit()">
-                    <span :class="submitIcon(config)" />{{ submitTitle(config) | l }}
+                    <span :class="getSubmitIcon()" />{{ submitTitle | l }}
                 </b-button>
-                <b-button v-if="config.cancel_redirect" @click="onCancel(config)">
-                    <span :class="cancelIcon(config)" />{{ cancelTitle(config) | l }}
+                <b-button v-if="cancelRedirect" @click="onCancel(config)">
+                    <span class="mr-1 fa fa-times" />{{ "Cancel" | l }}
                 </b-button>
             </div>
         </div>
@@ -46,6 +46,18 @@ export default {
             type: String,
             required: false,
         },
+        submitIcon: {
+            type: String,
+            required: false,
+        },
+        submitTitle: {
+            type: String,
+            default: "Save",
+        },
+        cancelRedirect: {
+            type: String,
+            required: false,
+        },
         url: {
             type: String,
             required: true,
@@ -78,25 +90,15 @@ export default {
         configMessageVariant(options) {
             return options.status || "warning";
         },
-        cancelTitle(options) {
-            return options.cancel_title || "Cancel";
-        },
-        cancelIcon(options) {
-            const icon = options.cancel_icon || "fa-times";
-            return `mr-1 fa ${icon}`;
-        },
-        submitTitle(options) {
-            return options.submit_title || "Save";
-        },
-        submitIcon(options) {
-            const icon = options.submit_icon || "fa-save";
+        getSubmitIcon() {
+            const icon = this.submitIcon || "fa-save";
             return `mr-1 fa ${icon}`;
         },
         onChange(formData) {
             this.formData = formData;
         },
         onCancel(options) {
-            window.location = `${getAppRoot()}${options.cancel_redirect}`;
+            window.location = `${getAppRoot()}${this.cancelRedirect}`;
         },
         onSubmit() {
             submitData(this.url, this.formData).then((response) => {
diff --git a/client/src/components/User/UserPreferencesModel.js b/client/src/components/User/UserPreferencesModel.js
index ad33b25be74..9082a4f1a77 100644
--- a/client/src/components/User/UserPreferencesModel.js
+++ b/client/src/components/User/UserPreferencesModel.js
@@ -30,8 +30,8 @@ export const getUserPreferencesModel = (user_id) => {
             id: "manage-third-party-identities",
             description: _l("Connect or disconnect access to your third-party identities."),
             icon: "fa-id-card-o",
-            submit_title: "Disconnect identity",
-            submit_icon: "fa-trash",
+            submitTitle: "Disconnect identity",
+            submitIcon: "fa-trash",
             shouldRender: config.enable_oidc,
         },
         permissions: {
@@ -41,7 +41,7 @@ export const getUserPreferencesModel = (user_id) => {
                 "Grant others default access to newly created histories. Changes made here will only affect histories created after these settings have been stored.",
             url: `api/users/${user_id}/permissions/inputs`,
             icon: "fa-users",
-            submit_title: "Save Permissions",
+            submitTitle: "Save Permissions",
             redirect: "user",
             shouldRender: !config.single_user,
         },
@@ -58,16 +58,16 @@ export const getUserPreferencesModel = (user_id) => {
             description: _l("Access your current API key or create a new one."),
             url: `api/users/${user_id}/api_key/inputs`,
             icon: "fa-key",
-            submit_title: "Create a new Key",
-            submit_icon: "fa-check",
+            submitTitle: "Create a new Key",
+            submitIcon: "fa-check",
         },
         cloud_auth: {
             id: "edit-preferences-cloud-auth",
             title: _l("Manage Cloud Authorization"),
             description: _l("Add or modify the configuration that grants Galaxy to access your cloud-based resources."),
             icon: "fa-cloud",
-            submit_title: "Create a new Key",
-            submit_icon: "fa-check",
+            submitTitle: "Create a new Key",
+            submitIcon: "fa-check",
             shouldRender: config.enable_account_interface,
         },
         toolbox_filters: {
@@ -76,7 +76,7 @@ export const getUserPreferencesModel = (user_id) => {
             description: _l("Customize your Toolbox by displaying or omitting sets of Tools."),
             url: `api/users/${user_id}/toolbox_filters/inputs`,
             icon: "fa-filter",
-            submit_title: "Save Filters",
+            submitTitle: "Save Filters",
             redirect: "user",
             shouldRender: !!config.has_user_tool_filters,
         },
diff --git a/client/src/entry/admin/AdminRouter.js b/client/src/entry/admin/AdminRouter.js
index 2b55152cb84..fc45d4e3ecf 100644
--- a/client/src/entry/admin/AdminRouter.js
+++ b/client/src/entry/admin/AdminRouter.js
@@ -164,7 +164,7 @@ export const getAdminRouter = (Galaxy, options) => {
                     title: _l("Reset passwords"),
                     url: `admin/reset_user_password${id}`,
                     icon: "fa-user",
-                    submit_title: "Save new password",
+                    submitTitle: "Save new password",
                     redirect: "admin/users",
                 },
                 manage_roles_and_groups_for_user: {
diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js
index 8e78eabc312..e17bdcdd4f1 100644
--- a/client/src/entry/analysis/AnalysisRouter.js
+++ b/client/src/entry/analysis/AnalysisRouter.js
@@ -327,9 +327,9 @@ export const getAnalysisRouter = (Galaxy) => {
                 url: "workflow/create",
                 redirect: "workflow/editor",
                 active_tab: "workflow",
-                submit_title: "Create",
-                submit_icon: "fa-check",
-                cancel_redirect: "workflows/list",
+                submitTitle: "Create",
+                submitIcon: "fa-check",
+                cancelRedirect: "workflows/list",
             });
         },
 
diff --git a/lib/galaxy/webapps/galaxy/controllers/forms.py b/lib/galaxy/webapps/galaxy/controllers/forms.py
index 79364892f5b..df6d867f192 100644
--- a/lib/galaxy/webapps/galaxy/controllers/forms.py
+++ b/lib/galaxy/webapps/galaxy/controllers/forms.py
@@ -106,7 +106,7 @@ class Forms(BaseUIController):
             fd_types = sorted(trans.app.model.FormDefinition.types.__members__.items())
             return {
                 'title': 'Create new form',
-                'submit_title': 'Create',
+                'submitTitle': 'Create',
                 'inputs': [{
                     'name': 'name',
                     'label': 'Name'

From d54474d79633579297ef32cb15939a9552708771 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 14:50:22 -0400
Subject: [PATCH 315/452] Remove custom button title for form creator from
 backend

---
 lib/galaxy/webapps/galaxy/controllers/forms.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/controllers/forms.py b/lib/galaxy/webapps/galaxy/controllers/forms.py
index df6d867f192..fe9bda6d77c 100644
--- a/lib/galaxy/webapps/galaxy/controllers/forms.py
+++ b/lib/galaxy/webapps/galaxy/controllers/forms.py
@@ -106,7 +106,6 @@ class Forms(BaseUIController):
             fd_types = sorted(trans.app.model.FormDefinition.types.__members__.items())
             return {
                 'title': 'Create new form',
-                'submitTitle': 'Create',
                 'inputs': [{
                     'name': 'name',
                     'label': 'Name'

From 18926add1ed46a4ec96016e07c2517a0dd4c37f7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 14:53:20 -0400
Subject: [PATCH 316/452] Remove unused config parsing to redirect option in
 generic form component

---
 client/src/components/Form/FormGeneric.vue | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 05fd226e14c..ff1ba265a5f 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -16,7 +16,7 @@
                 <b-button id="submit" variant="primary" class="mr-1" @click="onSubmit()">
                     <span :class="getSubmitIcon()" />{{ submitTitle | l }}
                 </b-button>
-                <b-button v-if="cancelRedirect" @click="onCancel(config)">
+                <b-button v-if="cancelRedirect" @click="onCancel()">
                     <span class="mr-1 fa fa-times" />{{ "Cancel" | l }}
                 </b-button>
             </div>
@@ -97,7 +97,7 @@ export default {
         onChange(formData) {
             this.formData = formData;
         },
-        onCancel(options) {
+        onCancel() {
             window.location = `${getAppRoot()}${this.cancelRedirect}`;
         },
         onSubmit() {

From 3a2bf1bf70c220ea3cc08074e7fc58558ae59928 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 14:55:53 -0400
Subject: [PATCH 317/452] Increase spacing between generic form and buttons
 below

---
 client/src/components/Form/FormGeneric.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index ff1ba265a5f..71909de7246 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -12,7 +12,7 @@
                     <FormDisplay :inputs="config.inputs" :replace-params="replaceParams" @onChange="onChange" />
                 </template>
             </FormCard>
-            <div class="mt-2">
+            <div class="mt-3">
                 <b-button id="submit" variant="primary" class="mr-1" @click="onSubmit()">
                     <span :class="getSubmitIcon()" />{{ submitTitle | l }}
                 </b-button>

From fb0b4af71fe0b7eaa2a785cb38a2a6aa1f99c79e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 15:00:40 -0400
Subject: [PATCH 318/452] Move icon determination to computed values in generic
 form component

---
 client/src/components/Form/FormGeneric.vue | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/client/src/components/Form/FormGeneric.vue b/client/src/components/Form/FormGeneric.vue
index 71909de7246..30914db59c9 100644
--- a/client/src/components/Form/FormGeneric.vue
+++ b/client/src/components/Form/FormGeneric.vue
@@ -14,7 +14,7 @@
             </FormCard>
             <div class="mt-3">
                 <b-button id="submit" variant="primary" class="mr-1" @click="onSubmit()">
-                    <span :class="getSubmitIcon()" />{{ submitTitle | l }}
+                    <span :class="submitIconClass" />{{ submitTitle | l }}
                 </b-button>
                 <b-button v-if="cancelRedirect" @click="onCancel()">
                     <span class="mr-1 fa fa-times" />{{ "Cancel" | l }}
@@ -48,7 +48,7 @@ export default {
         },
         submitIcon: {
             type: String,
-            required: false,
+            default: "fa-save",
         },
         submitTitle: {
             type: String,
@@ -80,6 +80,11 @@ export default {
             replaceParams: null,
         };
     },
+    computed: {
+        submitIconClass() {
+            return `mr-1 fa ${this.submitIcon}`;
+        },
+    },
     methods: {
         configTitle(options) {
             return this.title || options.title;
@@ -90,10 +95,6 @@ export default {
         configMessageVariant(options) {
             return options.status || "warning";
         },
-        getSubmitIcon() {
-            const icon = this.submitIcon || "fa-save";
-            return `mr-1 fa ${icon}`;
-        },
         onChange(formData) {
             this.formData = formData;
         },

From b8c5c56d07041abf48277afdb5df463f495aa1bc Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Sun, 8 Aug 2021 21:21:32 -0400
Subject: [PATCH 319/452] Fix tour based selenium test for core galaxy ui

---
 config/plugins/tours/core.galaxy_ui.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/plugins/tours/core.galaxy_ui.yaml b/config/plugins/tours/core.galaxy_ui.yaml
index 45f0d9956d2..5a06f770b21 100644
--- a/config/plugins/tours/core.galaxy_ui.yaml
+++ b/config/plugins/tours/core.galaxy_ui.yaml
@@ -95,7 +95,7 @@ steps:
       #backdrop: true
 
     - title: "Tool parameters"
-      element: '.center-panel .ui-portlet'
+      element: '.center-panel .portlet-content'
       intro: "Here you can choose your tool parameters. Select your input dataset from your history and specify parameters for your analysis."
       position: "right"
 

From 0bd338a334c657067560a9473af6f24305cd70f8 Mon Sep 17 00:00:00 2001
From: setlen <peter.selten98@gmail.com>
Date: Tue, 10 Aug 2021 20:51:46 +0200
Subject: [PATCH 320/452] Fix flake8 errors for gitlab error reporting

---
 lib/galaxy/tools/error_reports/plugins/gitlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/error_reports/plugins/gitlab.py b/lib/galaxy/tools/error_reports/plugins/gitlab.py
index 12bf02b2207..62c4a2b5bbe 100644
--- a/lib/galaxy/tools/error_reports/plugins/gitlab.py
+++ b/lib/galaxy/tools/error_reports/plugins/gitlab.py
@@ -2,9 +2,9 @@
 
 import logging
 import os
+import urllib.parse
 
 import requests
-import urllib.parse
 
 try:
     import gitlab

From 1ac10c292248e8ca71066bec40c222612193566d Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Thu, 29 Jul 2021 07:33:30 -0700
Subject: [PATCH 321/452] updated history contents api to return changed
 collection tags

---
 lib/galaxy/managers/collections.py           | 11 ++++++++++-
 test/unit/managers/test_CollectionManager.py |  6 ++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py
index acd4fea2d8c..ebfe09185f9 100644
--- a/lib/galaxy/managers/collections.py
+++ b/lib/galaxy/managers/collections.py
@@ -309,17 +309,26 @@ class DatasetCollectionManager:
     def _set_from_dict(self, trans, dataset_collection_instance, new_data):
         # send what we can down into the model
         changed = dataset_collection_instance.set_from_dict(new_data)
+
         # the rest (often involving the trans) - do here
         if 'annotation' in new_data.keys() and trans.get_user():
             dataset_collection_instance.add_item_annotation(trans.sa_session, trans.get_user(), dataset_collection_instance, new_data['annotation'])
             changed['annotation'] = new_data['annotation']
+
+        # the api promises a list of changed fields, but tags are not marked as changed to avoid the
+        # flush, so we must handle changed tag responses manually
+        new_tags = None
         if 'tags' in new_data.keys() and trans.get_user():
             # set_tags_from_list will flush on its own, no need to add to 'changed' here and incur a second flush.
-            self.tag_handler.set_tags_from_list(trans.get_user(), dataset_collection_instance, new_data['tags'])
+            new_tags = self.tag_handler.set_tags_from_list(trans.get_user(), dataset_collection_instance, new_data['tags'])
 
         if changed.keys():
             trans.sa_session.flush()
 
+        # set client tag field response after the flush
+        if new_tags is not None:
+            changed['tags'] = dataset_collection_instance.make_tag_string_list()
+
         return changed
 
     def _validate_and_parse_update_payload(self, payload):
diff --git a/test/unit/managers/test_CollectionManager.py b/test/unit/managers/test_CollectionManager.py
index 410ea1c99cb..a6f6a7d47e0 100644
--- a/test/unit/managers/test_CollectionManager.py
+++ b/test/unit/managers/test_CollectionManager.py
@@ -100,14 +100,16 @@ class DatasetCollectionManagerTestCase(BaseTestCase, CreatesCollectionsMixin):
             'deleted': True,
             'visible': False,
             'name': 'New Name',
+            'tags': ['name:one', 'group:two', 'three']
             # TODO: doesn't work
-            # 'tags'      : [ 'one', 'two', 'three' ]
             # 'annotations'      : [?]
         })
         self.assertEqual(hdca.name, 'New Name')
         self.assertTrue(hdca.deleted)
         self.assertFalse(hdca.visible)
-        # self.assertEqual( hdca.tags, [ 'one', 'two', 'three' ] )
+
+        tag_names = hdca.make_tag_string_list()
+        self.assertEqual(tag_names, ['name:one', 'group:two', 'three'])
         # self.assertEqual( hdca.annotations, [ 'one', 'two', 'three' ] )
 
     # def test_validation( self ):

From 9d2cdb8fa97bd0287177306e08e40d6abfe2fc66 Mon Sep 17 00:00:00 2001
From: setlen <peter.selten98@gmail.com>
Date: Wed, 11 Aug 2021 00:02:47 +0200
Subject: [PATCH 322/452] Fix incorrect formatting of string

---
 lib/galaxy/tools/error_reports/plugins/gitlab.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tools/error_reports/plugins/gitlab.py b/lib/galaxy/tools/error_reports/plugins/gitlab.py
index 62c4a2b5bbe..ae40c51c520 100644
--- a/lib/galaxy/tools/error_reports/plugins/gitlab.py
+++ b/lib/galaxy/tools/error_reports/plugins/gitlab.py
@@ -94,7 +94,7 @@ class GitLabPlugin(BaseGitPlugin):
                 # this will be detected by requests and used further down the line. Also cache this so everything is
                 # as fast as possible
                 ts_url = self._determine_ts_url(tool)
-                log.info("GitLab error reporting - Determined ToolShed is {ts_url}")
+                log.info(f"GitLab error reporting - Determined ToolShed is {ts_url}")
 
                 # Find the repo inside the ToolShed
                 ts_repourl = self._get_gitrepo_from_ts(job, ts_url)
@@ -144,7 +144,7 @@ class GitLabPlugin(BaseGitPlugin):
                         gl_emailquery = self.gitlab.users.list(search=gl_useremail)
                         log.debug(f"GitLab error reporting - User list: {gl_emailquery}")
                         if len(gl_emailquery) > 0:
-                            log.debug("GitLab error reporting - Last Committer user ID: " % gl_emailquery[0].get_id())
+                            log.debug("GitLab error reporting - Last Committer user ID: %s" % gl_emailquery[0].get_id())
                             self.git_username_id_cache[gl_useremail] = gl_emailquery[0].get_id()
                     gl_userid = self.git_username_id_cache.get(gl_useremail, None)
 

From f1e122c0ba63874f73131bb0f97d687120e7cf4b Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Wed, 11 Aug 2021 03:13:58 +0200
Subject: [PATCH 323/452] Migrate group_roles endpoints

---
 lib/galaxy/managers/group_roles.py           |  6 ++-
 lib/galaxy/schema/schema.py                  | 10 ++++
 lib/galaxy/webapps/galaxy/api/group_roles.py | 55 +++++++++++++++++++-
 lib/galaxy/webapps/galaxy/fast_app.py        |  4 ++
 4 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index a6e488272d0..1991863f072 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -106,8 +106,12 @@ class GroupRolesManager:
 
     def _serialize_group_role(self, encoded_group_id: EncodedDatabaseIdField, role: model.Role):
         encoded_role_id = self._app.security.encode_id(role.id)
+        try:
+            url  = url_for('role', group_id=encoded_group_id, id=encoded_role_id)
+        except AttributeError:
+            url  = "*deprecated attribute not filled in by FastAPI server*"
         return {
             "id": encoded_role_id,
             "name": role.name,
-            "url": url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
+            "url": url
         }
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 8c31eb11ada..c1b2edfa187 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -1902,6 +1902,16 @@ class RoleListModel(BaseModel):
 # Keeping it as a Tuple for now for backward compatibility
 RoleNameIdTuple = Tuple[str, EncodedDatabaseIdField]
 
+# Group_Roles -----------------------------------------------------------------
+
+
+class GroupRoleModel(BaseModel):
+    id: EncodedDatabaseIdField = RoleIdField
+    name: str = RoleNameField
+    url: str = Field(title="URL", description="URL for the group role")
+    
+class GroupRoleListModel(BaseModel):
+    __root__: List[GroupRoleModel]
 
 # Libraries -----------------------------------------------------------------
 
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 2e2410bf3ae..d7e0ed51e2b 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -1,8 +1,12 @@
 """
 API operations on Group objects.
 """
+
+from lib.galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 import logging
 
+from fastapi import Path
+
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.managers.group_roles import GroupRolesManager
 from galaxy.schema.fields import EncodedDatabaseIdField
@@ -10,10 +14,59 @@ from galaxy.web import (
     expose_api,
     require_admin,
 )
-from . import BaseGalaxyAPIController, depends
+from . import (
+    BaseGalaxyAPIController, 
+    depends, 
+    Router, 
+    DependsOnTrans
+)
+
 
 log = logging.getLogger(__name__)
 
+router = Router(tags=['group_roles'])
+
+GroupIDParam: EncodedDatabaseIdField = Path(
+    ...,
+    title='GroupID',
+    description='The ID of the group'
+)
+
+RoleIDParam: EncodedDatabaseIdField = Path(
+    ...,
+    title='RoleID',
+    description='The ID of the role'
+)
+
+
+@router.cbv
+class FastAPIGroupRoles:
+    manager: GroupRolesManager = depends(GroupRolesManager)
+
+    @router.get('/api/groups/{group_id}/roles',
+                require_admin=True,
+                summary='Displays a collection (list) of groups.')
+    def index(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
+        return self.manager.index(trans, group_id)
+
+    @router.get('/api/groups/{group_id}/roles/{role_id}',
+                require_admin=True,
+                summary='Displays information about a group role.')
+    def show(self,  trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
+        return self.manager.show(trans, role_id, group_id)
+
+    @router.put('/api/groups/{group_id}/roles/{role_id}',
+                require_admin=True,
+                summary='Adds a role to a group')
+    def update(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam):
+        return self.manager.update(trans, role_id, group_id)
+
+    @router.delete('/api/groups/{group_id}/roles/{role_id}',
+                    require_admin=True,
+                    summary='Removes a role from a group')
+    def delete(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam):
+        return self.manager.delete(trans, role_id, group_id)
+
 
 class GroupRolesAPIController(BaseGalaxyAPIController):
     manager = depends(GroupRolesManager)
diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py
index 518a9dc2129..11bb6b16c55 100644
--- a/lib/galaxy/webapps/galaxy/fast_app.py
+++ b/lib/galaxy/webapps/galaxy/fast_app.py
@@ -25,6 +25,10 @@ api_tags_metadata = [
         "name": "genomes",
         "description": "Operations with genome data.",
     },
+    {
+        "name": "group_roles",
+        "description": "Operations with group roles.",
+    },
     {
         "name": "licenses",
         "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).",

From 7692e6203663bd3c3a711194a5d53c0526660cdc Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 10:01:19 -0400
Subject: [PATCH 324/452] Load EDAM from PyPI if not configured instead of
 unexpected download.

---
 lib/galaxy/dependencies/dev-requirements.txt  |  1 +
 .../dependencies/pinned-requirements.txt      |  1 +
 lib/galaxy/tool_util/edam_util.py             | 62 +++++++++++++++++++
 lib/galaxy/tool_util/toolbox/base.py          | 49 ++-------------
 lib/galaxy/webapps/galaxy/config_schema.yml   |  2 +-
 packages/test.sh                              |  2 +-
 packages/tool_util/setup.py                   |  3 +
 pyproject.toml                                |  1 +
 test/unit/tool_util/test_edam_util.py         | 28 +++++++++
 9 files changed, 102 insertions(+), 47 deletions(-)
 create mode 100644 lib/galaxy/tool_util/edam_util.py
 create mode 100644 test/unit/tool_util/test_edam_util.py

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index fa0c98b11c3..85b8fdd2ce9 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -60,6 +60,7 @@ docopt==0.6.2
 docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 dogpile.cache==1.1.2; python_version >= "3.6"
 ecdsa==0.14.1; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0"
+edam-ontology==1.25.2
 fabric3==1.14.post1
 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0"
 fastapi==0.65.2; python_version >= "3.6"
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index bd4d0587550..d88ba36986c 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -55,6 +55,7 @@ docopt==0.6.2
 docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 dogpile.cache==1.1.2; python_version >= "3.6"
 ecdsa==0.14.1; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0"
+edam-ontology==1.25.2
 fabric3==1.14.post1
 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0"
 fastapi==0.65.2; python_version >= "3.6"
diff --git a/lib/galaxy/tool_util/edam_util.py b/lib/galaxy/tool_util/edam_util.py
new file mode 100644
index 00000000000..fb8917e64b0
--- /dev/null
+++ b/lib/galaxy/tool_util/edam_util.py
@@ -0,0 +1,62 @@
+import io
+import os
+from typing import Dict, Optional, TextIO
+
+try:
+    from edam_ontology.streams import tabular_stream
+except ImportError:
+    tabular_stream = None
+
+EDAM_PREFIX = 'http://edamontology.org/'
+COLUMN_TERM = 0
+COLUMN_LABEL = 1
+COLUMN_PARENTS = 7
+
+
+def load_edam_tree(path: Optional[str] = None):
+    if path is not None:
+        assert os.path.exists(path), f"Failed to load EDAM tabular data at [{path}] path does not exist."
+        handle = io.open(path, "r")
+    else:
+        assert tabular_stream is not None, "Failed to load optional import from edam-onotology package, install using [pip install edam-ontology]."
+        handle = tabular_stream()
+    return load_edam_tree_from_tsv_stream(handle)
+
+
+def load_edam_tree_from_tsv_stream(tsv_stream: TextIO):
+    edam: Dict[str, Dict] = {}
+
+    def _recurse_edam_parents(term, path=None):
+        if edam[term]['parents'] and len(edam[term]['parents']) > 0:
+            for parent in edam[term]['parents']:
+                yield from _recurse_edam_parents(parent, path + [parent])
+        else:
+            yield path
+
+    for line in tsv_stream.readlines():
+        fields = line.split('\t')
+
+        term = fields[COLUMN_TERM]
+        if not term.startswith(EDAM_PREFIX):
+            continue
+
+        term_id = term[len(EDAM_PREFIX):]
+
+        # Only care about formats and operations
+        if not (term_id.startswith('operation_') or term_id.startswith('topic_')):
+            continue
+
+        parents = fields[COLUMN_PARENTS].split('|')
+        edam[term_id] = {
+            'label': fields[COLUMN_LABEL],  # preferred label
+            'parents': [x[len(EDAM_PREFIX):] for x in parents if x.startswith(EDAM_PREFIX)],
+        }
+
+    for term in sorted(edam.keys()):
+        tails = []
+        for x in _recurse_edam_parents(term, path=[]):
+            if x[-2:] not in tails:
+                tails.append(x[-2:])
+        edam[term]['path'] = tails
+
+    return edam
diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 4728139c2ff..a42b4208aee 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -3,7 +3,6 @@ import logging
 import os
 import string
 import time
-import urllib.request
 from collections import namedtuple
 from errno import ENOENT
 from urllib.parse import urlparse
@@ -19,14 +18,13 @@ from galaxy.tool_util.deps import (
     build_dependency_manager,
     NullDependencyManager
 )
+from galaxy.tool_util.edam_util import load_edam_tree
 from galaxy.tool_util.loader_directory import looks_like_a_tool
 from galaxy.util import (
     etree,
     ExecutionTimer,
-    get_charset_from_http_headers,
     listify,
     parse_xml,
-    stream_to_open_named_file,
     string_as_bool,
     unicodify,
 )
@@ -124,52 +122,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         if save_integrated_tool_panel:
             self._save_integrated_tool_panel()
 
-    def _recurse_edam_parents(self, term, path=None):
-        log.debug(f"term:{term} path:{path} parents:{self.edam[term]['parents']}")
-        if self.edam[term]['parents'] and len(self.edam[term]['parents']) > 0:
-            for parent in self.edam[term]['parents']:
-                yield from self._recurse_edam_parents(parent, path + [parent])
-        else:
-            yield path
-
     def _load_edam(self):
         if not self.app.config.enable_beta_edam_toolbox:
             return
         log.debug('Loading EDAM Terms')
-
-        if not os.path.exists(self.app.config.beta_edam_toolbox_ontology_path):
-            log.debug('EDAM ontology file not present, downloading')
-
-            page = urllib.request.urlopen('https://edamontology.org/EDAM.tsv')
-            stream_to_open_named_file(page, os.open(self.app.config.beta_edam_toolbox_ontology_path, os.O_WRONLY | os.O_CREAT), None, source_encoding=get_charset_from_http_headers(page.headers))
-
-        log.debug('Processing EDAM Terms')
-        with open(self.app.config.beta_edam_toolbox_ontology_path, 'r') as handle:
-            log.debug(f'Processing {handle}')
-            self.edam = {}
-            for line in handle.readlines():
-                fields = line.split('\t')
-                if not fields[0].startswith('http://edamontology.org/'):
-                    continue
-                term_id = fields[0][len('http://edamontology.org/'):]
-
-                # Only care about formats and operations
-                if not (term_id.startswith('operation_') or term_id.startswith('topic_')):
-                    continue
-
-                parents = fields[7].split('|')
-                self.edam[term_id] = {
-                    'label': fields[1],  # preferred label
-                    'parents': [x[len('http://edamontology.org/'):] for x in parents if x.startswith('http://edamontology.org/')],
-                }
-                log.debug(f'Loaded term {term_id} => {fields[1]}')
-
-        for term in sorted(self.edam.keys()):
-            tails = []
-            for x in self._recurse_edam_parents(term, path=[]):
-                if x[-2:] not in tails:
-                    tails.append(x[-2:])
-            self.edam[term]['path'] = tails
+        edam_path = self.app.config.beta_edam_toolbox_ontology_path
+        edam = load_edam_tree(None if not os.path.exists(edam_path) else edam_path)
+        self.edam = edam
 
     def create_tool(self, config_file, tool_shed_repository=None, guid=None, **kwds):
         raise NotImplementedError()
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index be85ddc24ef..4d2739e3351 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -2454,7 +2454,7 @@ mapping:
         path_resolves_to: data_dir
         required: false
         desc: |
-          Sets the path to EDAM ontology file.
+          Sets the path to EDAM ontology file - if the path doesn't exist PyPI package data will be loaded.
 
       default_workflow_export_format:
         type: str
diff --git a/packages/test.sh b/packages/test.sh
index 033e1d8d202..0c9ddfe5f2f 100755
--- a/packages/test.sh
+++ b/packages/test.sh
@@ -50,7 +50,7 @@ for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do
         pip install -e '.[template,jstree]'
     fi
     if [ "$package_dir" = "tool_util" ]; then
-        pip install -e '.[mulled]'
+        pip install -e '.[mulled,edam]'
     fi
 
     if [[ "$run_tests" == "1" ]]; then
diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py
index 2d3a6a349ad..d7863021cd3 100644
--- a/packages/tool_util/setup.py
+++ b/packages/tool_util/setup.py
@@ -104,6 +104,9 @@ setup(
             'jinja2',
             'Whoosh',
         ],
+        'edam': [
+            'edam-ontology',
+        ],
     },
     license="AFL",
     zip_safe=False,
diff --git a/pyproject.toml b/pyproject.toml
index e85ffdbee53..985e6947c69 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,6 +34,7 @@ circus = "*"
 cwltool = "==3.0.20201109103151"
 dictobj = "*"
 docutils = "!=0.17, !=0.17.1"
+edam-ontology = "*"
 Fabric3 = "*"
 fastapi = ">=0.65.2"
 fastapi-utils = "*"
diff --git a/test/unit/tool_util/test_edam_util.py b/test/unit/tool_util/test_edam_util.py
new file mode 100644
index 00000000000..7697f547728
--- /dev/null
+++ b/test/unit/tool_util/test_edam_util.py
@@ -0,0 +1,28 @@
+import tempfile
+from typing import Any, Dict
+
+from edam_ontology.streams import tabular_stream
+
+from galaxy.tool_util.edam_util import load_edam_tree
+
+
+def test_load_edam_tree():
+    tree = load_edam_tree()
+    _verify_tree(tree)
+
+
+def test_load_edam_tree_from_path():
+    tf = tempfile.NamedTemporaryFile("w")
+    tf.write(tabular_stream().read())
+    tf.flush()
+    tree = load_edam_tree(tf.name)
+    _verify_tree(tree)
+
+
+def _verify_tree(tree: Dict[str, Any]):
+    assert tree is not None
+    assert "operation_0004" in tree
+    assert "topic_3974" in tree
+    assert tree["topic_3974"]["label"] == "Epistasis"
+    assert tree["topic_3974"]["parents"] == ['topic_0622', 'topic_3295']
+    assert tree["topic_3974"]["path"] == [['topic_3391', 'topic_0003'], ['topic_3070', 'topic_0003']]

From 7b665d81679b033c471817a01c60aa33847f875a Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 10:50:16 -0400
Subject: [PATCH 325/452] test case for get_integrated_section_for_tool method

---
 test/unit/tools/test_toolbox.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py
index 13ef949bd0f..ab06eac419b 100644
--- a/test/unit/tools/test_toolbox.py
+++ b/test/unit/tools/test_toolbox.py
@@ -371,6 +371,10 @@ class ToolBoxTestCase(BaseToolBoxTestCase):
         assert len(section.elems) == 1
         assert next(iter(section.elems.values())).id == "github.com/galaxyproject/example/test_tool/0.2"
 
+        test_tool = self.toolbox.get_tool("test_tool", tool_version="0.1")
+        section_pair = self.toolbox.get_integrated_section_for_tool(test_tool)
+        assert section_pair == ("tid", "TID")
+
     def test_group_tools_out_of_section(self):
         self._init_tool()
         self._setup_two_versions_in_config(section=False)

From 7aba4216f910c314c002a77bf5ee5bc5cf107afa Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 10:53:30 -0400
Subject: [PATCH 326/452] Some initial typing for toolbox lineage stuff.

---
 lib/galaxy/tool_util/toolbox/lineages/factory.py  | 15 +++++++++------
 .../tool_util/toolbox/lineages/interface.py       | 12 ++++++------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/lineages/factory.py b/lib/galaxy/tool_util/toolbox/lineages/factory.py
index a9f4f98563c..e7053e5c6b3 100644
--- a/lib/galaxy/tool_util/toolbox/lineages/factory.py
+++ b/lib/galaxy/tool_util/toolbox/lineages/factory.py
@@ -1,3 +1,5 @@
+from typing import Dict, Optional
+
 from galaxy.util.tool_version import remove_version_from_guid
 from .interface import ToolLineage
 
@@ -7,16 +9,17 @@ class LineageMap:
     """
 
     def __init__(self, app):
-        self.lineage_map = {}
+        self.lineage_map: Dict[str, ToolLineage] = {}
         self.app = app
 
-    def register(self, tool):
+    def register(self, tool) -> ToolLineage:
         tool_id = tool.id
         versionless_tool_id = remove_version_from_guid(tool_id)
-        lineage = self.lineage_map.get(versionless_tool_id)
-        if not lineage:
+        lineage: ToolLineage
+        if versionless_tool_id not in self.lineage_map:
             lineage = ToolLineage.from_tool(tool)
         else:
+            lineage = self.lineage_map[versionless_tool_id]
             # A lineage for a tool with the same versionless_tool_id exists,
             # but this lineage may not have the current tools' version,
             # so we add tool.version to the lineage
@@ -27,7 +30,7 @@ class LineageMap:
             self.lineage_map[tool_id] = lineage
         return self.lineage_map[tool_id]
 
-    def get(self, tool_id):
+    def get(self, tool_id) -> Optional[ToolLineage]:
         """
         Get lineage for `tool_id`.
 
@@ -46,7 +49,7 @@ class LineageMap:
                 self.lineage_map[tool_id] = lineage
         return self.lineage_map.get(tool_id)
 
-    def _get_versionless(self, tool_id):
+    def _get_versionless(self, tool_id) -> Optional[ToolLineage]:
         versionless_tool_id = remove_version_from_guid(tool_id)
         return self.lineage_map.get(versionless_tool_id, None)
 
diff --git a/lib/galaxy/tool_util/toolbox/lineages/interface.py b/lib/galaxy/tool_util/toolbox/lineages/interface.py
index eb305a25d76..b8f48657bac 100644
--- a/lib/galaxy/tool_util/toolbox/lineages/interface.py
+++ b/lib/galaxy/tool_util/toolbox/lineages/interface.py
@@ -1,5 +1,5 @@
 import threading
-from typing import Dict
+from typing import Any, Dict, List
 
 import packaging.version
 from sortedcontainers import SortedSet
@@ -44,13 +44,13 @@ class ToolLineage:
         self.tool_versions = SortedSet(key=packaging.version.parse)
 
     @property
-    def tool_ids(self):
+    def tool_ids(self) -> List[str]:
         versionless_tool_id = remove_version_from_guid(self.tool_id)
         tool_id = versionless_tool_id or self.tool_id
         return [f"{tool_id}/{version}" for version in self.tool_versions]
 
     @staticmethod
-    def from_tool(tool):
+    def from_tool(tool) -> 'ToolLineage':
         tool_id = tool.id
         lineages_by_id = ToolLineage.lineages_by_id
         with ToolLineage.lock:
@@ -60,7 +60,7 @@ class ToolLineage:
         lineage.register_version(tool.version)
         return lineage
 
-    def register_version(self, tool_version):
+    def register_version(self, tool_version) -> None:
         assert tool_version is not None
         self.tool_versions.add(str(tool_version))
 
@@ -71,12 +71,12 @@ class ToolLineage:
         """
         return [ToolLineageVersion(tool_id, tool_version) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions)]
 
-    def get_version_ids(self, reverse=False):
+    def get_version_ids(self, reverse=False) -> List[str]:
         if reverse:
             return list(reversed(self.tool_ids))
         return self.tool_ids
 
-    def to_dict(self):
+    def to_dict(self) -> Dict[str, Any]:
         return dict(
             tool_id=self.tool_id,
             tool_versions=list(self.tool_versions),

From 45f7dcb10c9767ab76bb13ae3fbe90591d1f9ef8 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 10:57:50 -0400
Subject: [PATCH 327/452] Rename method for clarify it is a general view
 concern not a merged toolbox concern.

---
 lib/galaxy/tool_util/toolbox/base.py | 2 +-
 lib/galaxy/tools/__init__.py         | 2 +-
 test/unit/tools/test_toolbox.py      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index a42b4208aee..91381d84e45 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -329,7 +329,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         log.debug(f"Loading new tool panel section: {str(tool_section.name)}")
         return tool_section
 
-    def get_integrated_section_for_tool(self, tool):
+    def get_section_for_tool(self, tool):
         tool_id = tool.id
 
         if tool_id in self._integrated_section_by_tool:
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index e4dcc7b6669..efbb7950443 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -760,7 +760,7 @@ class Tool(Dictifiable):
         return self.app.job_config.get_destination(self.__get_job_tool_configuration(job_params=job_params).destination)
 
     def get_panel_section(self):
-        return self.app.toolbox.get_integrated_section_for_tool(self)
+        return self.app.toolbox.get_section_for_tool(self)
 
     def allow_user_access(self, user, attempting_access=True):
         """
diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py
index ab06eac419b..5728f20d99a 100644
--- a/test/unit/tools/test_toolbox.py
+++ b/test/unit/tools/test_toolbox.py
@@ -372,7 +372,7 @@ class ToolBoxTestCase(BaseToolBoxTestCase):
         assert next(iter(section.elems.values())).id == "github.com/galaxyproject/example/test_tool/0.2"
 
         test_tool = self.toolbox.get_tool("test_tool", tool_version="0.1")
-        section_pair = self.toolbox.get_integrated_section_for_tool(test_tool)
+        section_pair = self.toolbox.get_section_for_tool(test_tool)
         assert section_pair == ("tid", "TID")
 
     def test_group_tools_out_of_section(self):

From 7f07987175e83021e8ea80e1692b60531b4bd8f8 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 11:10:07 -0400
Subject: [PATCH 328/452] Refactor toolbox._integrated_section_by_tool to make
 it clear tool_panel is responsible for section info.

---
 lib/galaxy/tool_util/toolbox/base.py  | 19 +++++++------------
 lib/galaxy/tool_util/toolbox/panel.py | 11 ++++++++++-
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 91381d84e45..5a67a94ab82 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -86,7 +86,6 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         self._dynamic_tool_confs = []
         self._tools_by_id = {}
         self._tools_by_uuid = {}
-        self._integrated_section_by_tool = {}
         # Tool lineages can contain chains of related tools with different ids
         # so each will be present once in the above dictionary. The following
         # dictionary can instead hold multiple tools with different versions.
@@ -331,11 +330,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
 
     def get_section_for_tool(self, tool):
         tool_id = tool.id
-
-        if tool_id in self._integrated_section_by_tool:
-            return self._integrated_section_by_tool[tool_id]
-
-        return None, None
+        return self._tool_panel.get_section_for_tool_id(tool_id)
 
     def __resolve_tool_path(self, tool_path, config_filename):
         if not tool_path:
@@ -522,7 +517,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             for (term, tool_id, key, val, val_name) in operations[term].values():
                 section = self._get_section(term, self.edam[term]['label'])
                 self.__add_tool_to_tool_panel(val, section, section=True)
-                self._integrated_section_by_tool[tool_id] = key, val_name
+                self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
 
         for term in sorted(topics.keys(), key=lambda x: self._sort_edam_key(x)):
             if len(topics[term].keys()) == 0:
@@ -536,13 +531,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             for (term, tool_id, key, val, val_name) in topics[term].values():
                 section = self._get_section(term, self.edam[term]['label'])
                 self.__add_tool_to_tool_panel(val, section, section=True)
-                self._integrated_section_by_tool[tool_id] = key, val_name
+                self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
 
         section = self._get_section('uncategorized', 'Uncategorized')
         for (tool_id, key, val, val_name) in uncategorized:
             print(tool_id, key, val, val_name)
             self.__add_tool_to_tool_panel(val, section, section=True)
-            self._integrated_section_by_tool[tool_id] = key, val_name
+            self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
 
     def _sort_edam_key(self, x):
         if x in ('operation_0004', 'topic_0003'):
@@ -557,7 +552,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
                 tool_id = key.replace('tool_', '', 1)
                 if tool_id in self._tools_by_id:
                     self.__add_tool_to_tool_panel(val, self._tool_panel, section=False)
-                    self._integrated_section_by_tool[tool_id] = '', ''
+                    self._tool_panel.record_section_for_tool_id(tool_id, '', '')
             elif item_type == panel_item_types.WORKFLOW:
                 workflow_id = key.replace('workflow_', '', 1)
                 if workflow_id in self._workflows_by_id:
@@ -579,7 +574,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
                         tool_id = section_key.replace('tool_', '', 1)
                         if tool_id in self._tools_by_id:
                             self.__add_tool_to_tool_panel(section_val, section, section=True)
-                            self._integrated_section_by_tool[tool_id] = key, val.name
+                            self._tool_panel.record_section_for_tool_id(tool_id, key, val.name)
                     elif section_item_type == panel_item_types.WORKFLOW:
                         workflow_id = section_key.replace('workflow_', '', 1)
                         if workflow_id in self._workflows_by_id:
@@ -943,7 +938,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             if section_item_type == panel_item_types.TOOL:
                 if section_item:
                     tool_id = section_key.replace('tool_', '', 1)
-                    self._integrated_section_by_tool[tool_id] = integrated_section.id, integrated_section.name
+                    self._tool_panel.record_section_for_tool_id(tool_id, integrated_section.id, integrated_section.name)
 
         if load_panel_dict:
             self._tool_panel[key] = section
diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
index 71d67a78c5e..a33e164a8a2 100644
--- a/lib/galaxy/tool_util/toolbox/panel.py
+++ b/lib/galaxy/tool_util/toolbox/panel.py
@@ -1,5 +1,5 @@
 from abc import abstractmethod
-
+from typing import Dict, Optional, Tuple
 
 from galaxy.util import bunch
 from galaxy.util.dictifiable import Dictifiable
@@ -115,6 +115,15 @@ class ToolPanelElements(odict, HasPanelItems):
     """ Represents an ordered dictionary of tool entries - abstraction
     used both by tool panel itself (normal and integrated) and its sections.
     """
+    _section_by_tool: Dict[str, Tuple[str, str]] = {}
+
+    def record_section_for_tool_id(self, tool_id: str, key: str, val: str):
+        self._section_by_tool[tool_id] = (key, val)
+
+    def get_section_for_tool_id(self, tool_id: str) -> Tuple[Optional[str], Optional[str]]:
+        if tool_id in self._section_by_tool:
+            return self._section_by_tool[tool_id]
+        return (None, None)
 
     def update_or_append(self, index, key, value):
         if key in self or index is None:

From e5747b294ae5265da24680289b0734845b0973b5 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 11:39:16 -0400
Subject: [PATCH 329/452] Abstraction for replacing tools in tool panel.

---
 lib/galaxy/tool_util/toolbox/base.py  | 10 +---------
 lib/galaxy/tool_util/toolbox/panel.py | 11 +++++++++++
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 5a67a94ab82..5ce50f50636 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -1095,15 +1095,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             new_tool.installed_changeset_revision = old_tool.installed_changeset_revision
             new_tool.old_id = old_tool.old_id
             # Replace old_tool with new_tool in self._tool_panel
-            tool_key = f"tool_{tool_id}"
-            for key, val in self._tool_panel.items():
-                if key == tool_key:
-                    self._tool_panel[key] = new_tool
-                    break
-                elif key.startswith('section'):
-                    if tool_key in val.elems:
-                        self._tool_panel[key].elems[tool_key] = new_tool
-                        break
+            self._tool_panel.replace_tool_for_id(tool_id, new_tool)
             # (Re-)Register the reloaded tool, this will handle
             #  _tools_by_id and _tool_versions_by_id
             self.register_tool(new_tool)
diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
index a33e164a8a2..89075ad287a 100644
--- a/lib/galaxy/tool_util/toolbox/panel.py
+++ b/lib/galaxy/tool_util/toolbox/panel.py
@@ -125,6 +125,17 @@ class ToolPanelElements(odict, HasPanelItems):
             return self._section_by_tool[tool_id]
         return (None, None)
 
+    def replace_tool_for_id(self, tool_id: str, new_tool) -> None:
+        tool_key = f"tool_{tool_id}"
+        for key, val in self.items():
+            if key == tool_key:
+                self[key] = new_tool
+                break
+            elif key.startswith('section'):
+                if tool_key in val.elems:
+                    self[key].elems[tool_key] = new_tool
+                    break
+
     def update_or_append(self, index, key, value):
         if key in self or index is None:
             self[key] = value

From 2314058fe0c590d2724b12ceae3d6a606efdc2fd Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 12:41:09 -0400
Subject: [PATCH 330/452] Reduce duplication around loading EDAM panel, fix log
 message.

---
 lib/galaxy/tool_util/toolbox/base.py | 78 ++++++++++------------------
 1 file changed, 28 insertions(+), 50 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 5ce50f50636..e30bf81eb16 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -431,6 +431,19 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             if parentTerm in self.edam[term]['parents']:
                 yield term
 
+    def _walk_integrated_tools(self):
+        for key, item_type, val in self._integrated_tool_panel.panel_items_iter():
+            if item_type == panel_item_types.TOOL:
+                tool_id = key.replace('tool_', '', 1)
+                if tool_id in self._tools_by_id:
+                    yield (tool_id, key, val, val.name)
+            elif item_type == panel_item_types.SECTION:
+                for section_key, section_item_type, section_val in val.panel_items_iter():
+                    if section_item_type == panel_item_types.TOOL:
+                        tool_id = section_key.replace('tool_', '', 1)
+                        if tool_id in self._tools_by_id:
+                            yield (tool_id, key, section_val, val.name)
+
     def _load_tool_panel_edam(self):
         execution_timer = ExecutionTimer()
 
@@ -453,57 +466,21 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         }
         uncategorized = []
 
-        for key, item_type, val in self._integrated_tool_panel.panel_items_iter():
-            if item_type == panel_item_types.TOOL:
-                tool_id = key.replace('tool_', '', 1)
-                if tool_id in self._tools_by_id:
-                    if tool_id in self._tools_by_id:
-                        for term in self._get_edam_sec(val):
-                            if term == 'uncategorized':
-                                uncategorized.append((tool_id, key, val, val.name))
-                            else:
-                                for path in self.edam[term]['path']:
-                                    if len(path) == 1:
-                                        t = term
-                                    else:
-                                        t = path[0]
+        for tool_id, key, val, val_name in self._walk_integrated_tools():
+            for term in self._get_edam_sec(val):
+                if term == 'uncategorized':
+                    uncategorized.append((tool_id, key, val, val_name))
+                else:
+                    for path in self.edam[term]['path']:
+                        if len(path) == 1:
+                            t = term
+                        else:
+                            t = path[0]
 
-                                    if path[0].startswith('operation_'):
-                                        operations[t][tool_id] = (term, tool_id, key, val, val.name)
-                                    elif path[0].startswith('topic_'):
-                                        topics[t][tool_id] = (term, tool_id, key, val, val.name)
-            # elif item_type == panel_item_types.WORKFLOW:
-                # workflow_id = key.replace('workflow_', '', 1)
-                # if workflow_id in self._workflows_by_id:
-                    # workflow = self._workflows_by_id[workflow_id]
-                    # self._tool_panel[key] = workflow
-                    # log.debug(f"Loaded workflow: {workflow_id} {workflow.name}")
-            elif item_type == panel_item_types.SECTION:
-                for section_key, section_item_type, section_val in val.panel_items_iter():
-                    if section_item_type == panel_item_types.TOOL:
-                        tool_id = section_key.replace('tool_', '', 1)
-                        if tool_id in self._tools_by_id:
-                            for term in self._get_edam_sec(section_val):
-                                if term == 'uncategorized':
-                                    uncategorized.append((tool_id, key, section_val, val.name))
-                                else:
-                                    for path in self.edam[term]['path']:
-                                        if len(path) == 1:
-                                            t = term
-                                        else:
-                                            t = path[0]
-
-                                        if path[0].startswith('operation_'):
-                                            operations[t][tool_id] = (term, tool_id, key, section_val, val.name)
-                                        if path[0].startswith('topic_'):
-                                            topics[t][tool_id] = (term, tool_id, key, section_val, val.name)
-                    # elif section_item_type == panel_item_types.WORKFLOW:
-                        # workflow_id = section_key.replace('workflow_', '', 1)
-                        # if workflow_id in self._workflows_by_id:
-                            # workflow = self._workflows_by_id[workflow_id]
-                            # section.elems[section_key] = workflow
-                            # log.debug(f"Loaded workflow: {workflow_id} {workflow.name}")
-        log.debug("Loading tool panel finished %s", execution_timer)
+                        if path[0].startswith('operation_'):
+                            operations[t][tool_id] = (term, tool_id, key, val, val_name)
+                        elif path[0].startswith('topic_'):
+                            topics[t][tool_id] = (term, tool_id, key, val, val_name)
 
         for term in sorted(operations.keys(), key=lambda x: self._sort_edam_key(x)):
             if len(operations[term].keys()) == 0:
@@ -538,6 +515,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             print(tool_id, key, val, val_name)
             self.__add_tool_to_tool_panel(val, section, section=True)
             self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
+        log.debug("Loading EDAM tool panel finished %s", execution_timer)
 
     def _sort_edam_key(self, x):
         if x in ('operation_0004', 'topic_0003'):

From cc187a38f334defe155eeb4f314e02aa73622983 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 12:41:37 -0400
Subject: [PATCH 331/452] Abstraction for removing tool from tool panel.

---
 lib/galaxy/tool_util/toolbox/base.py  | 10 +---------
 lib/galaxy/tool_util/toolbox/panel.py | 11 +++++++++++
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index e30bf81eb16..09ca91e8f6d 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -1099,15 +1099,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             if tool_cache:
                 tool_cache.expire_tool(tool_id)
             if remove_from_panel:
-                tool_key = f"tool_{tool_id}"
-                for key, val in self._tool_panel.items():
-                    if key == tool_key:
-                        del self._tool_panel[key]
-                        break
-                    elif key.startswith('section'):
-                        if tool_key in val.elems:
-                            del self._tool_panel[key].elems[tool_key]
-                            break
+                self._tool_panel.remove_tool(tool_id)
                 if tool_id in self.data_manager_tools:
                     del self.data_manager_tools[tool_id]
             # TODO: do we need to manually remove from the integrated panel here?
diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
index 89075ad287a..5f01e6e5478 100644
--- a/lib/galaxy/tool_util/toolbox/panel.py
+++ b/lib/galaxy/tool_util/toolbox/panel.py
@@ -136,6 +136,17 @@ class ToolPanelElements(odict, HasPanelItems):
                     self[key].elems[tool_key] = new_tool
                     break
 
+    def remove_tool(self, tool_id: str) -> None:
+        tool_key = f"tool_{tool_id}"
+        for key, val in self.items():
+            if key == tool_key:
+                del self[key]
+                break
+            elif key.startswith('section'):
+                if tool_key in val.elems:
+                    del self[key].elems[tool_key]
+                    break
+
     def update_or_append(self, index, key, value):
         if key in self or index is None:
             self[key] = value

From 5b87813620d51b1539440f82ed7abc14f302141b Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 12:44:00 -0400
Subject: [PATCH 332/452] Abstractions for root topic and operation in edam.

---
 lib/galaxy/tool_util/edam_util.py    |  6 ++++++
 lib/galaxy/tool_util/toolbox/base.py | 12 ++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/tool_util/edam_util.py b/lib/galaxy/tool_util/edam_util.py
index fb8917e64b0..6f6f26059ec 100644
--- a/lib/galaxy/tool_util/edam_util.py
+++ b/lib/galaxy/tool_util/edam_util.py
@@ -12,6 +12,9 @@ COLUMN_TERM = 0
 COLUMN_LABEL = 1
 COLUMN_PARENTS = 7
 
+ROOT_OPERATION = 'operation_0004'
+ROOT_TOPIC = 'topic_0003'
+
 
 def load_edam_tree(path: Optional[str] = None):
     if path is not None:
@@ -60,3 +63,6 @@ def load_edam_tree_from_tsv_stream(tsv_stream: TextIO):
         edam[term]['path'] = tails
 
     return edam
+
+
+__all__ = ('load_edam_tree', 'ROOT_OPERATION', 'ROOT_TOPIC')
diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 09ca91e8f6d..2c589ce0cd2 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -18,7 +18,11 @@ from galaxy.tool_util.deps import (
     build_dependency_manager,
     NullDependencyManager
 )
-from galaxy.tool_util.edam_util import load_edam_tree
+from galaxy.tool_util.edam_util import (
+    load_edam_tree,
+    ROOT_OPERATION,
+    ROOT_TOPIC,
+)
 from galaxy.tool_util.loader_directory import looks_like_a_tool
 from galaxy.util import (
     etree,
@@ -448,8 +452,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         execution_timer = ExecutionTimer()
 
         # Find the children of the top level topics
-        operations = ['operation_0004'] + list(self._edam_children_of('operation_0004'))
-        topics = ['topic_0003'] + list(self._edam_children_of('topic_0003'))
+        operations = [ROOT_OPERATION] + list(self._edam_children_of(ROOT_OPERATION))
+        topics = [ROOT_TOPIC] + list(self._edam_children_of(ROOT_TOPIC))
 
         # Sort them (by english label)
         # operations = sorted(operations, key=lambda x: self.edam[x]['label'])
@@ -518,7 +522,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         log.debug("Loading EDAM tool panel finished %s", execution_timer)
 
     def _sort_edam_key(self, x):
-        if x in ('operation_0004', 'topic_0003'):
+        if x in (ROOT_OPERATION, ROOT_TOPIC):
             return f"!{x}"
         else:
             return self.edam[x]['label']

From a554fdd4b6d2b006184cd8148249a94d2c48d5f9 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 13:10:37 -0400
Subject: [PATCH 333/452] Toolbox docstring.

---
 lib/galaxy/tool_util/toolbox/base.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 2c589ce0cd2..50a75ff820d 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -1115,6 +1115,10 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         return message, status
 
     def get_sections(self):
+        """Return section id and name pairs.
+
+        Only used by build_tool_panel_section_select_field in tool shed client code.
+        """
         for v in self._tool_panel.values():
             if isinstance(v, ToolSection):
                 yield (v.id, v.name)

From e4d7cb520820b2d88abb42dc9934dec25d1d59ce Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 13:10:51 -0400
Subject: [PATCH 334/452] EDAM info on test tools.

---
 test/functional/tools/for_workflows/mapper.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/functional/tools/for_workflows/mapper.xml b/test/functional/tools/for_workflows/mapper.xml
index 3a6f325d475..8160dd89300 100644
--- a/test/functional/tools/for_workflows/mapper.xml
+++ b/test/functional/tools/for_workflows/mapper.xml
@@ -1,4 +1,10 @@
 <tool id="mapper" name="Mapper" version="0.1.0">
+    <edam_topics>
+        <edam_topic>topic_0102</edam_topic><!-- CompBio -> SeqAnalysis -> Mapping  -->
+    </edam_topics>
+    <edam_operations>
+        <edam_operation>operation_3198</edam_operation> <!-- Analysis -> SeqAnalysis -> Nucleic acid sequence analysis -> Sequence read processing -> Read Mapping -->
+    </edam_operations>
     <command><![CDATA[
 cp '$__tool_directory__/1.bam' '$out_file1'
     ]]></command>

From 618fa7713be510440fd7332d31fb45489c2ff678 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 13:55:35 -0400
Subject: [PATCH 335/452] Integration tests for EDAM toolbox.

---
 lib/galaxy/tool_util/toolbox/base.py  |  3 +++
 test/integration/test_edam_toolbox.py | 24 ++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 test/integration/test_edam_toolbox.py

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 50a75ff820d..59aea84b652 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -122,6 +122,9 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
                 self._load_tool_panel_edam()
             else:
                 self._load_tool_panel()
+        elif self.app.name == 'galaxy' and self.app.config.enable_beta_edam_toolbox:
+            # This would have been ignored in the integrated tool panel loading right?
+            self._load_tool_panel_edam()
         if save_integrated_tool_panel:
             self._save_integrated_tool_panel()
 
diff --git a/test/integration/test_edam_toolbox.py b/test/integration/test_edam_toolbox.py
new file mode 100644
index 00000000000..66b0ab7503c
--- /dev/null
+++ b/test/integration/test_edam_toolbox.py
@@ -0,0 +1,24 @@
+from galaxy_test.driver import integration_util
+
+
+class EdamToolboxIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    framework_tool_and_types = True
+
+    def setUp(self):
+        super().setUp()
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["enable_beta_edam_toolbox"] = True
+
+    def test_edam_toolbox(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True))
+        index_as_list = index.json()
+        sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
+        section_names = [s["name"] for s in sections]
+        assert "Mapping" in section_names
+        mapping_section = [s for s in sections if s["name"] == "Mapping"][0]
+        mapping_section_elems = mapping_section["elems"]
+        # make sure our mapper tool was mapped using Edam correctly...
+        assert [x for x in mapping_section_elems if x["id"] == "mapper"]

From 45df5c6c36b9c3be5cd468fcc58d216c83b3cf92 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 4 Aug 2021 15:57:09 -0400
Subject: [PATCH 336/452] Typing for toolbox panel.

---
 lib/galaxy/tool_util/toolbox/panel.py | 40 +++++++++++++--------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
index 5f01e6e5478..e8cdbf6e733 100644
--- a/lib/galaxy/tool_util/toolbox/panel.py
+++ b/lib/galaxy/tool_util/toolbox/panel.py
@@ -1,18 +1,17 @@
 from abc import abstractmethod
+from enum import Enum
 from typing import Dict, Optional, Tuple
 
-from galaxy.util import bunch
 from galaxy.util.dictifiable import Dictifiable
 from galaxy.util.odict import odict
 from .parser import ensure_tool_conf_item
 
 
-panel_item_types = bunch.Bunch(
-    TOOL="TOOL",
-    WORKFLOW="WORKFLOW",
-    SECTION="SECTION",
-    LABEL="LABEL",
-)
+class panel_item_types(str, Enum):
+    TOOL = "TOOL"
+    WORKFLOW = "WORKFLOW"
+    SECTION = "SECTION"
+    LABEL = "LABEL"
 
 
 class HasPanelItems:
@@ -147,61 +146,62 @@ class ToolPanelElements(odict, HasPanelItems):
                     del self[key].elems[tool_key]
                     break
 
-    def update_or_append(self, index, key, value):
+    def update_or_append(self, index: int, key: str, value) -> None:
         if key in self or index is None:
             self[key] = value
         else:
             self.insert(index, key, value)
 
-    def get_label(self, label):
+    def get_label(self, label: str) -> Optional[ToolSection]:
         for element in self.values():
             if isinstance(element, ToolSection) and element.name == label:
                 return element
+        return None
 
-    def has_tool_with_id(self, tool_id):
+    def has_tool_with_id(self, tool_id: str) -> bool:
         key = f'tool_{tool_id}'
         return key in self
 
-    def replace_tool(self, previous_tool_id, new_tool_id, tool):
+    def replace_tool(self, previous_tool_id: str, new_tool_id: str, tool) -> None:
         previous_key = f'tool_{previous_tool_id}'
         new_key = f'tool_{new_tool_id}'
         index = self.keys().index(previous_key)
         del self[previous_key]
         self.insert(index, new_key, tool)
 
-    def index_of_tool_id(self, tool_id):
+    def index_of_tool_id(self, tool_id: str) -> Optional[int]:
         query_key = f'tool_{tool_id}'
         for index, target_key in enumerate(self.keys()):
             if query_key == target_key:
                 return index
         return None
 
-    def insert_tool(self, index, tool):
+    def insert_tool(self, index: int, tool) -> None:
         key = f"tool_{tool.id}"
         self.insert(index, key, tool)
 
-    def get_tool_with_id(self, tool_id):
+    def get_tool_with_id(self, tool_id: str):
         key = f"tool_{tool_id}"
         return self[key]
 
-    def append_tool(self, tool):
+    def append_tool(self, tool) -> None:
         key = f"tool_{tool.id}"
         self[key] = tool
 
-    def stub_tool(self, key):
+    def stub_tool(self, key: str) -> None:
         key = f"tool_{key}"
         self[key] = None
 
-    def stub_workflow(self, key):
+    def stub_workflow(self, key: str) -> None:
         key = f'workflow_{key}'
         self[key] = None
 
-    def stub_label(self, key):
+    def stub_label(self, key: str) -> None:
         key = f'label_{key}'
         self[key] = None
 
-    def append_section(self, key, section_elems):
-        self[key] = section_elems
+    def append_section(self, key: str, section: ToolSection) -> None:
+        self[key] = section
 
     def panel_items(self):
         return self

From b036d3a81c0950d7f1cb529d3d12704df29de8da Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 5 Aug 2021 13:12:28 -0400
Subject: [PATCH 337/452] typing for galaxy.tools.search

---
 lib/galaxy/tools/search/__init__.py | 40 +++++++++++++++++++----------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py
index 3aeb835b984..849139b6d31 100644
--- a/lib/galaxy/tools/search/__init__.py
+++ b/lib/galaxy/tools/search/__init__.py
@@ -7,6 +7,7 @@ through the library docs at https://whoosh.readthedocs.io.
 import logging
 import os
 import re
+from typing import Dict, List, Tuple, Union
 
 from whoosh import (
     analysis,
@@ -34,8 +35,11 @@ from galaxy.web.framework.helpers import to_unicode
 
 log = logging.getLogger(__name__)
 
+CanConvertToFloat = Union[str, int, float]
+CanConvertToInt = Union[str, int, float]
 
-def get_or_create_index(index_dir, schema):
+
+def get_or_create_index(index_dir: str, schema: Schema) -> index.Index:
     if not os.path.exists(index_dir):
         os.makedirs(index_dir)
     if index.exists_in(index_dir):
@@ -54,7 +58,7 @@ class ToolBoxSearch:
     the Whoosh search library.
     """
 
-    def __init__(self, toolbox, index_dir=None, index_help=True):
+    def __init__(self, toolbox, index_dir: str, index_help: bool = True):
         self.schema = Schema(id=ID(stored=True, unique=True),
                              old_id=ID,
                              stub=KEYWORD,
@@ -73,10 +77,10 @@ class ToolBoxSearch:
         # reindexing if the index count is equal to the toolbox reload count.
         self.index_count = -1
 
-    def _index_setup(self):
+    def _index_setup(self) -> index.Index:
         return get_or_create_index(index_dir=self.index_dir, schema=self.schema)
 
-    def build_index(self, tool_cache, index_help=True):
+    def build_index(self, tool_cache, index_help: bool = True) -> None:
         """
         Prepare search index for tools loaded in toolbox.
         Use `tool_cache` to determine which tools need indexing and which tools should be expired.
@@ -119,7 +123,7 @@ class ToolBoxSearch:
                     writer.update_document(**add_doc_kwds)
         log.debug("Toolbox index finished %s", execution_timer)
 
-    def _create_doc(self, tool_id, tool, index_help=True):
+    def _create_doc(self, tool_id: str, tool, index_help: bool = True) -> Dict[str, str]:
         #  Do not add data managers to the public index
         if tool.tool_type == 'manage_data':
             return {}
@@ -153,10 +157,20 @@ class ToolBoxSearch:
                     pass
         return add_doc_kwds
 
-    def search(self, q, tool_name_boost, tool_id_boost, tool_section_boost,
-            tool_description_boost, tool_label_boost, tool_stub_boost,
-            tool_help_boost, tool_search_limit, tool_enable_ngram_search,
-            tool_ngram_minsize, tool_ngram_maxsize):
+    def search(
+        self, q: str,
+        tool_name_boost: CanConvertToFloat,
+        tool_id_boost: CanConvertToFloat,
+        tool_section_boost: CanConvertToFloat,
+        tool_description_boost: CanConvertToFloat,
+        tool_label_boost: CanConvertToFloat,
+        tool_stub_boost: CanConvertToFloat,
+        tool_help_boost: CanConvertToFloat,
+        tool_search_limit: CanConvertToFloat,
+        tool_enable_ngram_search: bool,
+        tool_ngram_minsize: CanConvertToInt,
+        tool_ngram_maxsize: CanConvertToInt
+    ) -> List[str]:
         """
         Perform search on the in-memory index. Weight in the given boosts.
         """
@@ -194,13 +208,13 @@ class ToolBoxSearch:
             hits = self.searcher.search(parsed_query, limit=float(tool_search_limit), sortedby='')
             return [hit['id'] for hit in hits]
 
-    def _search_ngrams(self, cleaned_query, tool_ngram_minsize, tool_ngram_maxsize, tool_search_limit):
+    def _search_ngrams(self, cleaned_query: str, tool_ngram_minsize: CanConvertToInt, tool_ngram_maxsize: CanConvertToInt, tool_search_limit: CanConvertToFloat) -> List[str]:
         """
         Break tokens into ngrams and search on those instead.
         This should make searching more resistant to typos and unfinished words.
         See docs at https://whoosh.readthedocs.io/en/latest/ngrams.html
         """
-        hits_with_score = {}
+        hits_with_score: Dict[str, float] = {}
         token_analyzer = StandardAnalyzer() | analysis.NgramFilter(minsize=int(tool_ngram_minsize), maxsize=int(tool_ngram_maxsize))
         ngrams = [token.text for token in token_analyzer(cleaned_query)]
         for query in ngrams:
@@ -219,6 +233,6 @@ class ToolBoxSearch:
                 if not is_present:
                     hits_with_score[curr_hit['id']] = curr_hits.score(i)
         # Sort the results based on aggregated BM25 score in decreasing order of scores
-        hits_with_score = sorted(hits_with_score.items(), key=lambda x: x[1], reverse=True)
+        hits_with_score_list: List[Tuple[str, float]] = sorted(hits_with_score.items(), key=lambda x: x[1], reverse=True)
         # Return the tool ids
-        return [item[0] for item in hits_with_score[0:int(tool_search_limit)]]
+        return [item[0] for item in hits_with_score_list[0:int(tool_search_limit)]]

From 8f05a055f26aafba9d7e5bb5ad63c056c2d4caba Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 15:16:43 -0400
Subject: [PATCH 338/452] get_or_create_section abstraction in toolbox panel.

---
 lib/galaxy/tool_util/toolbox/base.py  | 9 ++-------
 lib/galaxy/tool_util/toolbox/panel.py | 8 ++++++++
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 59aea84b652..82fac6d8518 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -425,13 +425,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         else:
             yield 'uncategorized'
 
-    def _get_section(self, sec_id, sec_nm):
-        if sec_id not in self._tool_panel:
-            section = ToolSection({'id': sec_id, 'name': sec_nm, 'version': ''})
-            self._tool_panel[sec_id] = section
-        else:
-            section = self._tool_panel[sec_id]
-        return section
+    def _get_section(self, sec_id: str, sec_nm: str) -> ToolSection:
+        return self._tool_panel.get_or_create_section(sec_id, sec_nm)
 
     def _edam_children_of(self, parentTerm):
         for term in self.edam.keys():
diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
index e8cdbf6e733..044ab8aea10 100644
--- a/lib/galaxy/tool_util/toolbox/panel.py
+++ b/lib/galaxy/tool_util/toolbox/panel.py
@@ -135,6 +135,14 @@ class ToolPanelElements(odict, HasPanelItems):
                     self[key].elems[tool_key] = new_tool
                     break
 
+    def get_or_create_section(self, sec_id: str, sec_nm: str) -> ToolSection:
+        if sec_id not in self:
+            section = ToolSection({'id': sec_id, 'name': sec_nm, 'version': ''})
+            self[sec_id] = section
+        else:
+            section = self[sec_id]
+        return section
+
     def remove_tool(self, tool_id: str) -> None:
         tool_key = f"tool_{tool_id}"
         for key, val in self.items():

From 2dddcac8c86043e451939453154aa760745b77d6 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Aug 2021 15:15:03 -0400
Subject: [PATCH 339/452] Implement tool panel views - config, toolbox, search,
 client.

---
 .../Panels/Buttons/PanelViewButton.vue        | 103 ++++++++
 .../Panels/Buttons/PanelViewMenuItem.vue      |  52 ++++
 .../components/Panels/Common/ToolSearch.vue   |   6 +-
 .../Panels/ProviderAwareToolBox.vue           |  52 ++++
 .../Panels/ProviderAwareToolBoxWorkflow.vue   |  72 +++++
 client/src/components/Panels/ToolBox.vue      |  26 +-
 .../src/components/Panels/ToolBoxWorkflow.vue |  26 +-
 .../src/components/Workflow/Editor/Index.vue  |  11 +-
 .../providers/ToolPanelViewProvider.js        |  25 ++
 client/src/entry/panels/tool-panel.js         |   6 +-
 client/src/store/index.js                     |   2 +
 client/src/store/panelStore.js                |  55 ++++
 lib/galaxy/managers/configuration.py          |   2 +
 lib/galaxy/tool_util/toolbox/base.py          | 250 ++++++++----------
 lib/galaxy/tool_util/toolbox/panel.py         |  43 ++-
 lib/galaxy/tool_util/toolbox/parser.py        |   6 +
 .../tool_util/toolbox/views/__init__.py       |   0
 .../tool_util/toolbox/views/definitions.py    | 155 +++++++++++
 lib/galaxy/tool_util/toolbox/views/edam.py    | 168 ++++++++++++
 .../tool_util/toolbox/views/interface.py      |  75 ++++++
 lib/galaxy/tool_util/toolbox/views/sources.py |  52 ++++
 lib/galaxy/tool_util/toolbox/views/static.py  | 168 ++++++++++++
 lib/galaxy/tools/__init__.py                  |   8 +
 lib/galaxy/tools/search/__init__.py           |  47 +++-
 lib/galaxy/webapps/galaxy/api/tools.py        |  10 +-
 lib/galaxy/webapps/galaxy/config_schema.yml   |  41 ++-
 lib/galaxy_test/driver/driver_util.py         |  14 +-
 packages/tool_util/requirements.txt           |   2 +
 packages/tool_util/setup.py                   |   1 +
 test/integration/panel_views_1/custom_1.yml   |  14 +
 test/integration/panel_views_1/custom_10.yml  |   4 +
 test/integration/panel_views_1/custom_2.yml   |   7 +
 test/integration/panel_views_1/custom_3.yml   |   8 +
 test/integration/panel_views_1/custom_4.yml   |  15 ++
 test/integration/panel_views_1/custom_5.yml   |   9 +
 test/integration/panel_views_1/custom_6.yml   |  13 +
 test/integration/panel_views_1/custom_7.yml   |  13 +
 test/integration/panel_views_1/custom_8.yml   |   9 +
 test/integration/panel_views_1/custom_9.yml   |   7 +
 test/integration/panel_views_1/filter.yml     |   5 +
 test/integration/test_edam_toolbox.py         |  46 +++-
 test/integration/test_panel_views.py          | 178 +++++++++++++
 test/unit/config/test_config_values.py        |   2 +-
 .../tool_util/test_toolbox_view_parsing.py    | 101 +++++++
 .../tool_util/test_toolbox_view_sources.py    |  45 ++++
 test/unit/unittest_utils/galaxy_mock.py       |   5 +
 46 files changed, 1774 insertions(+), 185 deletions(-)
 create mode 100644 client/src/components/Panels/Buttons/PanelViewButton.vue
 create mode 100644 client/src/components/Panels/Buttons/PanelViewMenuItem.vue
 create mode 100644 client/src/components/Panels/ProviderAwareToolBox.vue
 create mode 100644 client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue
 create mode 100644 client/src/components/providers/ToolPanelViewProvider.js
 create mode 100644 client/src/store/panelStore.js
 create mode 100644 lib/galaxy/tool_util/toolbox/views/__init__.py
 create mode 100644 lib/galaxy/tool_util/toolbox/views/definitions.py
 create mode 100644 lib/galaxy/tool_util/toolbox/views/edam.py
 create mode 100644 lib/galaxy/tool_util/toolbox/views/interface.py
 create mode 100644 lib/galaxy/tool_util/toolbox/views/sources.py
 create mode 100644 lib/galaxy/tool_util/toolbox/views/static.py
 create mode 100644 test/integration/panel_views_1/custom_1.yml
 create mode 100644 test/integration/panel_views_1/custom_10.yml
 create mode 100644 test/integration/panel_views_1/custom_2.yml
 create mode 100644 test/integration/panel_views_1/custom_3.yml
 create mode 100644 test/integration/panel_views_1/custom_4.yml
 create mode 100644 test/integration/panel_views_1/custom_5.yml
 create mode 100644 test/integration/panel_views_1/custom_6.yml
 create mode 100644 test/integration/panel_views_1/custom_7.yml
 create mode 100644 test/integration/panel_views_1/custom_8.yml
 create mode 100644 test/integration/panel_views_1/custom_9.yml
 create mode 100644 test/integration/panel_views_1/filter.yml
 create mode 100644 test/integration/test_panel_views.py
 create mode 100644 test/unit/tool_util/test_toolbox_view_parsing.py
 create mode 100644 test/unit/tool_util/test_toolbox_view_sources.py

diff --git a/client/src/components/Panels/Buttons/PanelViewButton.vue b/client/src/components/Panels/Buttons/PanelViewButton.vue
new file mode 100644
index 00000000000..84fc12ddc2e
--- /dev/null
+++ b/client/src/components/Panels/Buttons/PanelViewButton.vue
@@ -0,0 +1,103 @@
+<template>
+    <b-dropdown
+        no-caret
+        right
+        role="button"
+        title="Panel Views"
+        variant="link"
+        aria-label="View all tool panel configurations"
+        class="tool-panel-dropdown float-right"
+        size="sm"
+        v-b-tooltip.hover
+    >
+        <template v-slot:button-content>
+            <span class="fa fa-list" />
+            <!-- Maybe cog? -->
+        </template>
+        <PanelViewMenuItem
+            :current-panel-view="currentPanelView"
+            :panel-view="defaultPanelView"
+            @onSelect="updatePanelView"
+        />
+        <b-dropdown-divider></b-dropdown-divider>
+        <b-dropdown-group v-for="group in groupedPanelViews" :key="group.type" :id="group.type" :header="group.title">
+            <PanelViewMenuItem
+                v-for="(panelView, key) in group.panelViews"
+                :key="key"
+                :current-panel-view="currentPanelView"
+                :panel-view="panelView"
+                @onSelect="updatePanelView"
+            />
+        </b-dropdown-group>
+        <b-dropdown-divider v-if="groupedPanelViews.length > 0"></b-dropdown-divider>
+        <PanelViewMenuItem
+            v-if="ungroupedPanelViews.length > 0"
+            v-for="(panelView, key) in ungroupedPanelViews"
+            :key="key"
+            :current-panel-view="currentPanelView"
+            :panel-view="panelView"
+            @onSelect="updatePanelView"
+        />
+    </b-dropdown>
+</template>
+
+<script>
+import PanelViewMenuItem from "./PanelViewMenuItem";
+
+const groupsDefinitions = [
+    { type: "ontology", title: "... by Ontology" },
+    { type: "activity", title: "... for Activity" },
+    { type: "publication", title: "... from Publication" },
+    { type: "training", title: "... for Training" },
+];
+
+export default {
+    components: { PanelViewMenuItem },
+    props: {
+        panelViews: {
+            type: Object,
+        },
+        currentPanelView: {
+            type: String,
+        },
+    },
+    computed: {
+        defaultPanelView() {
+            return this.panelViewsOfType("default")[0];
+        },
+        groupedPanelViews() {
+            const groups = [];
+            for (const group of groupsDefinitions) {
+                const viewType = group.type;
+                const panelViews = this.panelViewsOfType(viewType);
+                if (panelViews.length > 0) {
+                    groups.push({
+                        type: viewType,
+                        title: group.title,
+                        panelViews: panelViews,
+                    });
+                }
+            }
+            return groups;
+        },
+        ungroupedPanelViews() {
+            return this.panelViewsOfType("generic");
+        },
+    },
+    methods: {
+        updatePanelView(panelView) {
+            this.$emit("updatePanelView", panelView.id);
+        },
+        panelViewsOfType(panelViewType) {
+            const panelViews = [];
+            for (const panelViewId in this.panelViews) {
+                const panelView = this.panelViews[panelViewId];
+                if (panelView.view_type == panelViewType) {
+                    panelViews.push(panelView);
+                }
+            }
+            return panelViews;
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Panels/Buttons/PanelViewMenuItem.vue b/client/src/components/Panels/Buttons/PanelViewMenuItem.vue
new file mode 100644
index 00000000000..7e03eb20efc
--- /dev/null
+++ b/client/src/components/Panels/Buttons/PanelViewMenuItem.vue
@@ -0,0 +1,52 @@
+<template>
+    <b-dropdown-item
+        :active="currentPanelView == panelView.id"
+        :disabled="currentPanelView == panelView.id"
+        @click="onClick"
+    >
+        <span :class="icon" />
+        <span :title="title" v-localize>{{ name }}</span>
+    </b-dropdown-item>
+</template>
+
+<script>
+const types_to_icons = {
+    default: "fa-backspace",
+    generic: "fa-filter",
+    ontology: "fa-sitemap",
+    activity: "fa-project-diagram",
+    publication: "fa-newspaper",
+    training: "fa-graduation-cap",
+};
+
+export default {
+    props: {
+        currentPanelView: {
+            type: String,
+            required: true,
+        },
+        panelView: {
+            type: Object,
+            required: true,
+        },
+    },
+    computed: {
+        title() {
+            return this.panelView.description;
+        },
+        icon() {
+            const viewType = this.panelView.view_type;
+            const faIcon = types_to_icons[viewType];
+            return ["fa", faIcon];
+        },
+        name() {
+            return this.panelView.name;
+        },
+    },
+    methods: {
+        onClick() {
+            this.$emit("onSelect", this.panelView);
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Panels/Common/ToolSearch.vue b/client/src/components/Panels/Common/ToolSearch.vue
index 11fe95f8f77..67c12215669 100644
--- a/client/src/components/Panels/Common/ToolSearch.vue
+++ b/client/src/components/Panels/Common/ToolSearch.vue
@@ -14,6 +14,10 @@ export default {
         DelayedInput,
     },
     props: {
+        currentPanelView: {
+            type: String,
+            required: true,
+        },
         query: {
             type: String,
             default: "",
@@ -46,7 +50,7 @@ export default {
                     this.loading = true;
                     axios
                         .get(`${getAppRoot()}api/tools`, {
-                            params: { q },
+                            params: { q, view: this.currentPanelView },
                         })
                         .then((response) => {
                             this.loading = false;
diff --git a/client/src/components/Panels/ProviderAwareToolBox.vue b/client/src/components/Panels/ProviderAwareToolBox.vue
new file mode 100644
index 00000000000..ad61547a267
--- /dev/null
+++ b/client/src/components/Panels/ProviderAwareToolBox.vue
@@ -0,0 +1,52 @@
+<template>
+    <ConfigProvider class="d-flex flex-column" v-slot="{ config }">
+        <ToolPanelViewProvider
+            v-slot="{ currentPanel, currentPanelView }"
+            :site-default-panel-view="config.default_panel_view"
+            v-if="config.default_panel_view"
+        >
+            <ToolBox
+                :toolbox="currentPanel"
+                :panel-views="config.panel_views"
+                :current-panel-view="currentPanelView"
+                :stored-workflow-menu-entries="storedWorkflowMenuEntries"
+                :workflow-title="workflowTitle"
+                @updatePanelView="updatePanelView"
+                v-if="currentPanelView"
+            >
+            </ToolBox>
+        </ToolPanelViewProvider>
+    </ConfigProvider>
+</template>
+
+<script>
+import ToolBox from "./ToolBox";
+import ConfigProvider from "components/providers/ConfigProvider";
+import ToolPanelViewProvider from "components/providers/ToolPanelViewProvider";
+import _l from "utils/localization";
+import { mapActions } from "vuex";
+
+export default {
+    components: {
+        ConfigProvider,
+        ToolBox,
+        ToolPanelViewProvider,
+    },
+    props: {
+        storedWorkflowMenuEntries: {
+            type: Array,
+            required: true,
+        },
+        workflowTitle: {
+            type: String,
+            default: _l("Workflows"),
+        },
+    },
+    methods: {
+        updatePanelView(panelView) {
+            this.setCurrentPanelView(panelView);
+        },
+        ...mapActions("panels", ["setCurrentPanelView"]),
+    },
+};
+</script>
diff --git a/client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue b/client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue
new file mode 100644
index 00000000000..2ab248a6097
--- /dev/null
+++ b/client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue
@@ -0,0 +1,72 @@
+<template>
+    <ConfigProvider class="d-flex flex-column" v-slot="{ config }">
+        <ToolPanelViewProvider
+            v-slot="{ currentPanel, currentPanelView }"
+            :site-default-panel-view="config.default_panel_view"
+            v-if="config.default_panel_view"
+        >
+            <ToolBoxWorkflow
+                :toolbox="currentPanel"
+                :panel-views="config.panel_views"
+                :current-panel-view="currentPanelView"
+                :workflows="workflows"
+                :data-managers="dataManagers"
+                :module-sections="moduleSections"
+                @updatePanelView="updatePanelView"
+                @onInsertTool="onInsertTool"
+                @onInsertModule="onInsertModule"
+                @onInsertWorkflow="onInsertWorkflow"
+                @onInsertWorkflowSteps="onInsertWorkflowSteps"
+                v-if="currentPanelView"
+            >
+            </ToolBoxWorkflow>
+        </ToolPanelViewProvider>
+    </ConfigProvider>
+</template>
+
+<script>
+import ToolBoxWorkflow from "./ToolBoxWorkflow";
+import ConfigProvider from "components/providers/ConfigProvider";
+import ToolPanelViewProvider from "components/providers/ToolPanelViewProvider";
+import { mapActions } from "vuex";
+
+export default {
+    components: {
+        ConfigProvider,
+        ToolBoxWorkflow,
+        ToolPanelViewProvider,
+    },
+    props: {
+        workflows: {
+            type: Array,
+            required: true,
+        },
+        dataManagers: {
+            type: Array,
+            required: true,
+        },
+        moduleSections: {
+            type: Array,
+            required: true,
+        },
+    },
+    methods: {
+        updatePanelView(panelView) {
+            this.setCurrentPanelView(panelView);
+        },
+        ...mapActions("panels", ["setCurrentPanelView"]),
+        onInsertTool(toolId, toolName) {
+            this.$emit("onInsertTool", toolId, toolName);
+        },
+        onInsertModule(moduleName, moduleTitle) {
+            this.$emit("onInsertModule", moduleName, moduleTitle);
+        },
+        onInsertWorkflow(workflowId, workflowName) {
+            this.$emit("onInsertWorkflow", workflowId, workflowName);
+        },
+        onInsertWorkflowSteps(workflowId, workflowStepCount) {
+            this.$emit("onInsertWorkflowSteps", workflowId, workflowStepCount);
+        },
+    },
+};
+</script>
diff --git a/client/src/components/Panels/ToolBox.vue b/client/src/components/Panels/ToolBox.vue
index fca9bd812ce..d57a03c7b64 100644
--- a/client/src/components/Panels/ToolBox.vue
+++ b/client/src/components/Panels/ToolBox.vue
@@ -4,12 +4,24 @@
             <div class="unified-panel-header-inner">
                 <div class="panel-header-buttons">
                     <favorites-button :query="query" @onFavorites="onFavorites" v-if="isUser" />
+                    <panel-view-button
+                        :panel-views="panelViews"
+                        :current-panel-view="currentPanelView"
+                        @updatePanelView="updatePanelView"
+                        v-if="panelViews && Object.keys(panelViews).length > 1"
+                    />
                 </div>
                 <div class="panel-header-text">Tools</div>
             </div>
         </div>
         <div class="unified-panel-controls">
-            <tool-search :query="query" placeholder="search tools" @onQuery="onQuery" @onResults="onResults" />
+            <tool-search
+                :current-panel-view="currentPanelView"
+                :query="query"
+                placeholder="search tools"
+                @onQuery="onQuery"
+                @onResults="onResults"
+            />
             <upload-button />
             <div class="py-2" v-if="hasResults">
                 <b-button @click="onToggle" size="sm" class="w-100">
@@ -52,16 +64,17 @@ import ToolSection from "./Common/ToolSection";
 import ToolSearch from "./Common/ToolSearch";
 import { UploadButton, openGlobalUploadModal } from "components/Upload";
 import FavoritesButton from "./Buttons/FavoritesButton";
+import PanelViewButton from "./Buttons/PanelViewButton";
 import { filterToolSections, filterTools } from "./utilities";
 import { getGalaxyInstance } from "app";
 import { getAppRoot } from "onload";
 import _l from "utils/localization";
 
 export default {
-    name: "ToolBox",
     components: {
         UploadButton,
         FavoritesButton,
+        PanelViewButton,
         ToolSection,
         ToolSearch,
     },
@@ -81,6 +94,12 @@ export default {
             type: Array,
             required: true,
         },
+        panelViews: {
+            type: Object,
+        },
+        currentPanelView: {
+            type: String,
+        },
         storedWorkflowMenuEntries: {
             type: Array,
             required: true,
@@ -163,6 +182,9 @@ export default {
             this.buttonText = this.showSections ? "Hide Sections" : "Show Sections";
             this.buttonIcon = this.showSections ? "fa fa-eye-slash" : "fa fa-eye";
         },
+        updatePanelView(panelView) {
+            this.$emit("updatePanelView", panelView);
+        },
     },
 };
 </script>
diff --git a/client/src/components/Panels/ToolBoxWorkflow.vue b/client/src/components/Panels/ToolBoxWorkflow.vue
index 64e98721273..8847b9d1958 100644
--- a/client/src/components/Panels/ToolBoxWorkflow.vue
+++ b/client/src/components/Panels/ToolBoxWorkflow.vue
@@ -2,11 +2,24 @@
     <div class="unified-panel">
         <div class="unified-panel-header" unselectable="on">
             <div class="unified-panel-header-inner">
+                <div class="panel-header-buttons">
+                    <panel-view-button
+                        :panel-views="panelViews"
+                        :current-panel-view="currentPanelView"
+                        @updatePanelView="updatePanelView"
+                        v-if="panelViews && Object.keys(panelViews).length > 1"
+                    />
+                </div>
                 <div class="panel-header-text">Tools</div>
             </div>
         </div>
         <div class="unified-panel-controls">
-            <tool-search placeholder="search tools" @onQuery="onQuery" @onResults="onResults" />
+            <tool-search
+                :current-panel-view="currentPanelView"
+                placeholder="search tools"
+                @onQuery="onQuery"
+                @onResults="onResults"
+            />
         </div>
         <div class="unified-panel-body">
             <div class="toolMenuContainer">
@@ -58,12 +71,14 @@ import _l from "utils/localization";
 import ToolSection from "./Common/ToolSection";
 import ToolSearch from "./Common/ToolSearch";
 import { filterToolSections } from "./utilities";
+import PanelViewButton from "./Buttons/PanelViewButton";
 
 export default {
     name: "ToolBox",
     components: {
         ToolSection,
         ToolSearch,
+        PanelViewButton,
     },
     data() {
         return {
@@ -76,6 +91,12 @@ export default {
             type: Array,
             required: true,
         },
+        panelViews: {
+            type: Object,
+        },
+        currentPanelView: {
+            type: String,
+        },
         workflows: {
             type: Array,
             required: true,
@@ -147,6 +168,9 @@ export default {
         onInsertWorkflowSteps(workflow) {
             this.$emit("onInsertWorkflowSteps", workflow.id, workflow.step_count);
         },
+        updatePanelView(panelView) {
+            this.$emit("updatePanelView", panelView);
+        },
     },
 };
 </script>
diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue
index 5093697d162..3789b594661 100644
--- a/client/src/components/Workflow/Editor/Index.vue
+++ b/client/src/components/Workflow/Editor/Index.vue
@@ -39,8 +39,7 @@
         <div v-show="isCanvas">
             <SidePanel id="left" side="left">
                 <template v-slot:panel>
-                    <ToolBoxWorkflow
-                        :toolbox="toolbox"
+                    <ProviderAwareToolBoxWorkflow
                         :module-sections="moduleSections"
                         :data-managers="dataManagers"
                         :workflows="workflows"
@@ -186,7 +185,7 @@ import WorkflowOptions from "./Options";
 import FormDefault from "./Forms/FormDefault";
 import FormTool from "./Forms/FormTool";
 import MarkdownEditor from "components/Markdown/MarkdownEditor";
-import ToolBoxWorkflow from "components/Panels/ToolBoxWorkflow";
+import ProviderAwareToolBoxWorkflow from "components/Panels/ProviderAwareToolBoxWorkflow";
 import SidePanel from "components/Panels/SidePanel";
 import { getAppRoot } from "onload/loadConfig";
 import reportDefault from "./reportDefault";
@@ -205,7 +204,7 @@ export default {
         MarkdownEditor,
         SidePanel,
         StateUpgradeModal,
-        ToolBoxWorkflow,
+        ProviderAwareToolBoxWorkflow,
         FormDefault,
         FormTool,
         WorkflowOptions,
@@ -241,10 +240,6 @@ export default {
             type: Array,
             required: true,
         },
-        toolbox: {
-            type: Array,
-            required: true,
-        },
     },
     data() {
         return {
diff --git a/client/src/components/providers/ToolPanelViewProvider.js b/client/src/components/providers/ToolPanelViewProvider.js
new file mode 100644
index 00000000000..11c09cdcc50
--- /dev/null
+++ b/client/src/components/providers/ToolPanelViewProvider.js
@@ -0,0 +1,25 @@
+import { mapGetters, mapActions } from "vuex";
+
+export default {
+    props: {
+        siteDefaultPanelView: {
+            type: String,
+            required: true,
+        },
+    },
+    computed: {
+        ...mapGetters("panels", ["currentPanel", "currentPanelView"]),
+    },
+    methods: {
+        ...mapActions("panels", ["initCurrentPanelView"]),
+    },
+    created() {
+        this.initCurrentPanelView(this.siteDefaultPanelView);
+    },
+    render() {
+        return this.$scopedSlots.default({
+            currentPanel: this.currentPanel,
+            currentPanelView: this.currentPanelView,
+        });
+    },
+};
diff --git a/client/src/entry/panels/tool-panel.js b/client/src/entry/panels/tool-panel.js
index c37f8f6dfa7..ee71792ef67 100644
--- a/client/src/entry/panels/tool-panel.js
+++ b/client/src/entry/panels/tool-panel.js
@@ -1,7 +1,7 @@
 import Backbone from "backbone";
 import _l from "utils/localization";
 import { getGalaxyInstance } from "app";
-import ToolBox from "../../components/Panels/ToolBox";
+import ToolBox from "../../components/Panels/ProviderAwareToolBox";
 import SidePanel from "../../components/Panels/SidePanel";
 import { mountVueComponent } from "../../utils/mountVueComponent";
 
@@ -23,7 +23,9 @@ const ToolPanel = Backbone.View.extend({
                 currentPanel: ToolBox,
                 currentPanelProperties: {
                     storedWorkflowMenuEntries: Galaxy.config.stored_workflow_menu_entries,
-                    toolbox: Galaxy.config.toolbox,
+                    // TODO: find a way to seed to Vuex store's default panel view with data
+                    // or just stop including it in the response.
+                    // toolbox: Galaxy.config.toolbox,
                 },
             },
             el
diff --git a/client/src/store/index.js b/client/src/store/index.js
index a85b06a82be..4dcb0320444 100644
--- a/client/src/store/index.js
+++ b/client/src/store/index.js
@@ -20,6 +20,7 @@ import { toolStore } from "./toolStore";
 import { datasetPathDestinationStore } from "./datasetPathDestinationStore";
 import { datasetExtFilesStore } from "./datasetExtFilesStore";
 import { jobStore } from "./jobStore";
+import { panelStore } from "./panelStore";
 
 // beta features
 import { historyStore as betaHistoryStore } from "components/History/model/historyStore";
@@ -36,6 +37,7 @@ export function createStore() {
             user: userStore,
             config: configStore,
             betaHistory: betaHistoryStore,
+            panels: panelStore,
 
             // TODO: please namespace all store modules
             gridSearch: gridSearchStore,
diff --git a/client/src/store/panelStore.js b/client/src/store/panelStore.js
new file mode 100644
index 00000000000..d34cfd58f88
--- /dev/null
+++ b/client/src/store/panelStore.js
@@ -0,0 +1,55 @@
+export const state = {
+    panel: {},
+    currentPanelView: null,
+};
+
+import Vue from "vue";
+import { getAppRoot } from "onload/loadConfig";
+import axios from "axios";
+
+const getters = {
+    panel: (state) => (panelView) => {
+        return state.panel[panelView] || [];
+    },
+    currentPanel: (state) => {
+        const effectiveView = state.currentPanelView || "default";
+        const val = state.panel[effectiveView] || [];
+        return val;
+    },
+    currentPanelView: (state) => {
+        return state.currentPanelView;
+    },
+};
+
+const actions = {
+    initCurrentPanelView: async ({ commit, state }, siteDefaultPanelView) => {
+        const panelView = state.currentPanelView || siteDefaultPanelView;
+        if (state.currentPanelView == null) {
+            commit("setCurrentPanelView", { panelView });
+        }
+        const { data } = await axios.get(`${getAppRoot()}api/tools?in_panel=true&view=${panelView}`);
+        commit("savePanelView", { panelView, panel: data });
+    },
+    setCurrentPanelView: async ({ commit }, panelView) => {
+        commit("setCurrentPanelView", { panelView });
+        const { data } = await axios.get(`${getAppRoot()}api/tools?in_panel=true&view=${panelView}`);
+        commit("savePanelView", { panelView, panel: data });
+    },
+};
+
+const mutations = {
+    savePanelView: (state, { panelView, panel }) => {
+        Vue.set(state.panel, panelView, panel);
+    },
+    setCurrentPanelView: (state, { panelView }) => {
+        state.currentPanelView = panelView;
+    },
+};
+
+export const panelStore = {
+    namespaced: true,
+    state,
+    getters,
+    actions,
+    mutations,
+};
diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py
index 3e43f4320c4..9f9385d6880 100644
--- a/lib/galaxy/managers/configuration.py
+++ b/lib/galaxy/managers/configuration.py
@@ -200,6 +200,8 @@ class ConfigSerializer(base.ModelSerializer):
             'python': _defaults_to((sys.version_info.major, sys.version_info.minor)),
             'select_type_workflow_threshold': _use_config,
             'file_sources_configured': lambda config, key, **context: self.app.file_sources.custom_sources_configured,
+            'panel_views': lambda config, key, **content: self.app.toolbox.panel_view_dicts(),
+            'default_panel_view': _use_config,
             'upload_from_form_button': _use_config,
             'release_doc_base_url': _use_config,
             'user_library_import_dir_available': lambda config, key, **context: bool(config.get('user_library_import_dir')),
diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py
index 82fac6d8518..ed58ec431e3 100644
--- a/lib/galaxy/tool_util/toolbox/base.py
+++ b/lib/galaxy/tool_util/toolbox/base.py
@@ -5,6 +5,7 @@ import string
 import time
 from collections import namedtuple
 from errno import ENOENT
+from typing import Dict, List
 from urllib.parse import urlparse
 
 from markupsafe import escape
@@ -13,16 +14,12 @@ from galaxy.exceptions import (
     ConfigurationError,
     MessageException,
     ObjectNotFound,
+    RequestParameterInvalidException,
 )
 from galaxy.tool_util.deps import (
     build_dependency_manager,
     NullDependencyManager
 )
-from galaxy.tool_util.edam_util import (
-    load_edam_tree,
-    ROOT_OPERATION,
-    ROOT_TOPIC,
-)
 from galaxy.tool_util.loader_directory import looks_like_a_tool
 from galaxy.util import (
     etree,
@@ -48,6 +45,14 @@ from .parser import (
     get_toolbox_parser
 )
 from .tags import tool_tag_manager
+from .views.edam import EdamPanelMode, EdamToolPanelView
+from .views.interface import (
+    ToolBoxRegistry,
+    ToolPanelView,
+    ToolPanelViewModel,
+    ToolPanelViewModelType,
+)
+from .views.static import StaticToolPanelView
 
 log = logging.getLogger(__name__)
 
@@ -73,17 +78,37 @@ class ToolConfRepository(_ToolConfRepository):
         return self.tool_path, self.repository_path
 
 
+class ToolBoxRegistryImpl(ToolBoxRegistry):
+    """View of ToolBox provided to ToolPanelView to reason about tools loaded."""
+
+    def __init__(self, toolbox: 'AbstractToolBox'):
+        self.__toolbox = toolbox
+
+    def has_tool(self, tool_id: str) -> bool:
+        return tool_id in self.__toolbox._tools_by_id
+
+    def get_tool(self, tool_id: str):
+        return self.__toolbox.get_tool(tool_id)
+
+    def get_workflow(self, id: str):
+        return self.__toolbox._workflows_by_id[id]
+
+    def add_tool_to_tool_panel_view(self, tool, tool_panel_component) -> None:
+        self.__toolbox.add_tool_to_tool_panel_view(tool, tool_panel_component)
+
+
 class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
     """
     Abstract container for managing a ToolPanel - containing tools and
     workflows optionally in labelled sections.
     """
 
-    def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True):
+    def __init__(self, config_filenames, tool_root_dir, app, view_sources=None, default_panel_view="default", save_integrated_tool_panel=True):
         """
         Create a toolbox from the config files named by `config_filenames`, using
         `tool_root_dir` as the base directory for finding individual tool config files.
         """
+        self._default_panel_view = default_panel_view
         # The _dynamic_tool_confs list contains dictionaries storing
         # information about the tools defined in each shed-related
         # shed_tool_conf.xml file.
@@ -114,28 +139,53 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         self._filter_factory = FilterFactory(self)
         self._tool_tag_manager = tool_tag_manager(app)
         self._init_tools_from_configs(config_filenames)
-        self._load_edam()
 
         if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents:
-            # Load self._tool_panel based on the order in self._integrated_tool_panel.
-            if self.app.config.enable_beta_edam_toolbox:
-                self._load_tool_panel_edam()
-            else:
-                self._load_tool_panel()
-        elif self.app.name == 'galaxy' and self.app.config.enable_beta_edam_toolbox:
-            # This would have been ignored in the integrated tool panel loading right?
-            self._load_tool_panel_edam()
+            self._load_tool_panel()
+
+        toolbox = self
+
+        class DefaultToolPanelView(ToolPanelView):
+
+            def apply_view(self, base_tool_panel: ToolPanelElements, toolbox_registry: ToolBoxRegistry) -> ToolPanelElements:
+                return toolbox._tool_panel
+
+            def to_model(self) -> ToolPanelViewModel:
+                model_id = "default"
+                name = "Full Tool Panel"
+                description = "Galaxy's fully configured toolbox panel with all sections, tools, and configured workflows loaded."
+                view_type = ToolPanelViewModelType.default_type
+                model_class = DefaultToolPanelView.__class__.__name__
+                return ToolPanelViewModel(
+                    id=model_id,
+                    name=name,
+                    description=description,
+                    model_class=model_class,
+                    view_type=view_type,
+                    searchable=True,
+                )
+
+        tool_panel_views_list = [
+            DefaultToolPanelView(),
+        ]
+
+        for edam_view in listify(self.app.config.edam_panel_views):
+            mode = EdamPanelMode[edam_view]
+            tool_panel_views_list.append(EdamToolPanelView(self.app.config.edam_toolbox_ontology_path, mode=mode))
+
+        if view_sources is not None:
+            for definition in view_sources.get_definitions():
+                tool_panel_views_list.append(StaticToolPanelView(definition))
+
+        self._tool_panel_views = {}
+        for tool_panel_view in tool_panel_views_list:
+            self._tool_panel_views[tool_panel_view.to_model().id] = tool_panel_view
+
+        if self.app.name == 'galaxy':
+            self._load_tool_panel_views()
         if save_integrated_tool_panel:
             self._save_integrated_tool_panel()
 
-    def _load_edam(self):
-        if not self.app.config.enable_beta_edam_toolbox:
-            return
-        log.debug('Loading EDAM Terms')
-        edam_path = self.app.config.beta_edam_toolbox_ontology_path
-        edam = load_edam_tree(None if not os.path.exists(edam_path) else edam_path)
-        self.edam = edam
-
     def create_tool(self, config_file, tool_shed_repository=None, guid=None, **kwds):
         raise NotImplementedError()
 
@@ -254,6 +304,16 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
 
         return None
 
+    def panel_views(self) -> List[ToolPanelViewModel]:
+        return [v.to_model() for v in self._tool_panel_views.values()]
+
+    def panel_view_dicts(self) -> Dict[str, Dict]:
+        return {m.id: m.dict() for m in self.panel_views()}
+
+    def panel_has_tool(self, tool, panel_view_id):
+        panel_view_rendered = self._tool_panel_view_rendered[panel_view_id]
+        return panel_view_rendered.has_item_recursive(tool)
+
     def load_dynamic_tool(self, dynamic_tool):
         if not dynamic_tool.active:
             return None
@@ -350,7 +410,12 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
             tool_path = string.Template(tool_path).safe_substitute(tool_path_vars)
         return tool_path
 
-    def __add_tool_to_tool_panel(self, tool, panel_component, section=False):
+    def add_tool_to_tool_panel_view(self, tool, view_panel_component):
+        self.__add_tool_to_tool_panel(tool, view_panel_component)
+
+    def __add_tool_to_tool_panel(self, tool, panel_component, section=None):
+        if section is None:
+            section = isinstance(panel_component, ToolSection)
         # See if a version of this tool is already loaded into the tool panel.
         # The value of panel_component will be a ToolSection (if the value of
         # section=True) or self._tool_panel (if section=False).
@@ -417,113 +482,11 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         if log_msg and (not hasattr(self.app, 'tool_cache') or tool_id in self.app.tool_cache._new_tool_ids):
             log.debug(log_msg)
 
-    def _get_edam_sec(self, tool):
-        edam = tool.edam_operations + tool.edam_topics
-        if len(edam) > 0:
-            for term in edam:
-                yield term
-        else:
-            yield 'uncategorized'
-
-    def _get_section(self, sec_id: str, sec_nm: str) -> ToolSection:
-        return self._tool_panel.get_or_create_section(sec_id, sec_nm)
-
-    def _edam_children_of(self, parentTerm):
-        for term in self.edam.keys():
-            if parentTerm in self.edam[term]['parents']:
-                yield term
-
-    def _walk_integrated_tools(self):
-        for key, item_type, val in self._integrated_tool_panel.panel_items_iter():
-            if item_type == panel_item_types.TOOL:
-                tool_id = key.replace('tool_', '', 1)
-                if tool_id in self._tools_by_id:
-                    yield (tool_id, key, val, val.name)
-            elif item_type == panel_item_types.SECTION:
-                for section_key, section_item_type, section_val in val.panel_items_iter():
-                    if section_item_type == panel_item_types.TOOL:
-                        tool_id = section_key.replace('tool_', '', 1)
-                        if tool_id in self._tools_by_id:
-                            yield (tool_id, key, section_val, val.name)
-
-    def _load_tool_panel_edam(self):
-        execution_timer = ExecutionTimer()
-
-        # Find the children of the top level topics
-        operations = [ROOT_OPERATION] + list(self._edam_children_of(ROOT_OPERATION))
-        topics = [ROOT_TOPIC] + list(self._edam_children_of(ROOT_TOPIC))
-
-        # Sort them (by english label)
-        # operations = sorted(operations, key=lambda x: self.edam[x]['label'])
-        # topics = sorted(topics, key=lambda x: self.edam[x]['label'])
-
-        # Convert these to list of dicts, wherein we'll add our tools/etc.
-        operations = {
-            x: {}
-            for x in operations
-        }
-        topics = {
-            x: {}
-            for x in topics
-        }
-        uncategorized = []
-
-        for tool_id, key, val, val_name in self._walk_integrated_tools():
-            for term in self._get_edam_sec(val):
-                if term == 'uncategorized':
-                    uncategorized.append((tool_id, key, val, val_name))
-                else:
-                    for path in self.edam[term]['path']:
-                        if len(path) == 1:
-                            t = term
-                        else:
-                            t = path[0]
-
-                        if path[0].startswith('operation_'):
-                            operations[t][tool_id] = (term, tool_id, key, val, val_name)
-                        elif path[0].startswith('topic_'):
-                            topics[t][tool_id] = (term, tool_id, key, val, val_name)
-
-        for term in sorted(operations.keys(), key=lambda x: self._sort_edam_key(x)):
-            if len(operations[term].keys()) == 0:
-                continue
-
-            elem = etree.Element('label')
-            elem.attrib['text'] = self.edam[term]['label']
-            elem.attrib['id'] = term
-            self._tool_panel[f"label_{term}"] = ToolSectionLabel(elem)
-
-            for (term, tool_id, key, val, val_name) in operations[term].values():
-                section = self._get_section(term, self.edam[term]['label'])
-                self.__add_tool_to_tool_panel(val, section, section=True)
-                self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
-
-        for term in sorted(topics.keys(), key=lambda x: self._sort_edam_key(x)):
-            if len(topics[term].keys()) == 0:
-                continue
-
-            elem = etree.Element('label')
-            elem.attrib['text'] = self.edam[term]['label']
-            elem.attrib['id'] = term
-            self._tool_panel[f"label_{term}"] = ToolSectionLabel(elem)
-
-            for (term, tool_id, key, val, val_name) in topics[term].values():
-                section = self._get_section(term, self.edam[term]['label'])
-                self.__add_tool_to_tool_panel(val, section, section=True)
-                self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
-
-        section = self._get_section('uncategorized', 'Uncategorized')
-        for (tool_id, key, val, val_name) in uncategorized:
-            print(tool_id, key, val, val_name)
-            self.__add_tool_to_tool_panel(val, section, section=True)
-            self._tool_panel.record_section_for_tool_id(tool_id, key, val_name)
-        log.debug("Loading EDAM tool panel finished %s", execution_timer)
-
-    def _sort_edam_key(self, x):
-        if x in (ROOT_OPERATION, ROOT_TOPIC):
-            return f"!{x}"
-        else:
-            return self.edam[x]['label']
+    def _load_tool_panel_views(self):
+        self._tool_panel_view_rendered = {}
+        registry = ToolBoxRegistryImpl(self)
+        for key, view in self._tool_panel_views.items():
+            self._tool_panel_view_rendered[key] = view.apply_view(self._integrated_tool_panel, registry)
 
     def _load_tool_panel(self):
         execution_timer = ExecutionTimer()
@@ -864,7 +827,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         tool._lineage = self._lineage_map.register(tool)
         self.register_tool(tool)
         if load_panel_dict:
-            self.__add_tool_to_tool_panel(tool, panel_dict, section=isinstance(panel_dict, ToolSection))
+            self.__add_tool_to_tool_panel(tool, panel_dict)
 
     def _load_workflow_tag_set(self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None):
         try:
@@ -941,10 +904,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
                 integrated_elems[key] = tool
 
                 if async_load:
-                    if self.app.config.enable_beta_edam_toolbox:
-                        self._load_tool_panel_edam()
-                    else:
-                        self._load_tool_panel()
+                    self._load_tool_panel()
+                    self._load_tool_panel_views()
                     self._save_integrated_tool_panel()
                 return tool.id
             except Exception:
@@ -1148,11 +1109,16 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         stored = self.app.model.context.query(self.app.model.StoredWorkflow).get(id)
         return stored.latest_workflow
 
-    def tool_panel_contents(self, trans, **kwds):
+    def tool_panel_contents(self, trans, view=None, **kwds):
         """ Filter tool_panel contents for displaying for user.
         """
+        if view is None:
+            view = self._default_panel_view
+        if view not in self._tool_panel_view_rendered:
+            raise RequestParameterInvalidException(f"No panel view {view} found.")
         filter_method = self._build_filter_method(trans)
-        for _, item_type, elt in self._tool_panel.panel_items_iter():
+        tool_panel_view = self._tool_panel_view_rendered[view]
+        for _, item_type, elt in tool_panel_view.panel_items_iter():
             elt = filter_method(elt, item_type)
             if elt:
                 yield elt
@@ -1179,14 +1145,14 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
                     self._tool_to_dict_cache_admin[tool.id] = to_dict
         return to_dict
 
-    def to_dict(self, trans, in_panel=True, tool_help=False, **kwds):
+    def to_dict(self, trans, in_panel=True, tool_help=False, view=None, **kwds):
         """
         Create a dictionary representation of the toolbox.
         Uses primitive cache for toolbox-specific tool 'to_dict's.
         """
         rval = []
         if in_panel:
-            panel_elts = list(self.tool_panel_contents(trans, **kwds))
+            panel_elts = list(self.tool_panel_contents(trans, view=view, **kwds))
             for elt in panel_elts:
                 # Only use cache for objects that are Tools.
                 if hasattr(elt, "tool_type"):
@@ -1304,8 +1270,8 @@ class BaseGalaxyToolBox(AbstractToolBox):
     shouldn't really depend on.
     """
 
-    def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True):
-        super().__init__(config_filenames, tool_root_dir, app, save_integrated_tool_panel)
+    def __init__(self, config_filenames, tool_root_dir, app, view_sources=None, default_panel_view=None, save_integrated_tool_panel=True):
+        super().__init__(config_filenames, tool_root_dir, app, view_sources, default_panel_view, save_integrated_tool_panel)
         old_toolbox = getattr(app, 'toolbox', None)
         if old_toolbox:
             self.dependency_manager = old_toolbox.dependency_manager
diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py
index 044ab8aea10..1846e18ce9f 100644
--- a/lib/galaxy/tool_util/toolbox/panel.py
+++ b/lib/galaxy/tool_util/toolbox/panel.py
@@ -64,7 +64,7 @@ class ToolSection(Dictifiable, HasPanelItems):
         copy.name = self.name
         copy.id = self.id
         copy.version = self.version
-        copy.elems = self.elems.copy()
+        copy.elems.update(self.elems)
         return copy
 
     def to_dict(self, trans, link_details=False, tool_help=False, toolbox=None):
@@ -213,3 +213,44 @@ class ToolPanelElements(odict, HasPanelItems):
 
     def panel_items(self):
         return self
+
+    def walk_sections(self):
+        for key, item in self.items():
+            if isinstance(item, ToolSection):
+                yield (key, item)
+
+    def closest_section(self, target_section_id: Optional[str], target_section_name: Optional[str]):
+        for (section_id, section) in self.walk_sections():
+            if section_id == target_section_id:
+                return section
+
+        for (_, section) in self.walk_sections():
+            if section.name == target_section_name:
+                return section
+
+        return None
+
+    def apply_filter(self, f):
+        to_remove = []
+
+        for key, item in self.items():
+            if not f(key, item):
+                to_remove.append(key)
+
+        for key in to_remove:
+            del self[key]
+
+    def copy(self) -> 'ToolPanelElements':
+        the_copy = ToolPanelElements()
+        the_copy.update(self)
+        return the_copy
+
+    def has_item_recursive(self, item):
+        """Check panel and section elements for supplied item."""
+        for value in self.values():
+            if value == item:
+                return True
+            if isinstance(value, ToolSection):
+                if item in value.elems.values():
+                    return True
+        return False
diff --git a/lib/galaxy/tool_util/toolbox/parser.py b/lib/galaxy/tool_util/toolbox/parser.py
index 787ac9fc876..e4df9ff5492 100644
--- a/lib/galaxy/tool_util/toolbox/parser.py
+++ b/lib/galaxy/tool_util/toolbox/parser.py
@@ -137,6 +137,12 @@ def ensure_tool_conf_item(xml_or_item):
         return None
     elif isinstance(xml_or_item, ToolConfItem):
         return xml_or_item
+    elif isinstance(xml_or_item, dict):
+        # TODO: handle sections...
+        as_dict = xml_or_item.copy()
+        type = as_dict.pop("type")
+        attributes = as_dict
+        return ToolConfItem(type, attributes, None)
     else:
         elem = xml_or_item
         type = elem.tag
diff --git a/lib/galaxy/tool_util/toolbox/views/__init__.py b/lib/galaxy/tool_util/toolbox/views/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/galaxy/tool_util/toolbox/views/definitions.py b/lib/galaxy/tool_util/toolbox/views/definitions.py
new file mode 100644
index 00000000000..8f10c548550
--- /dev/null
+++ b/lib/galaxy/tool_util/toolbox/views/definitions.py
@@ -0,0 +1,155 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field
+from typing_extensions import Literal
+
+
+class StaticToolBoxViewTypeEnum(str, Enum):
+    generic = 'generic'
+    activity = 'activity'
+    publication = 'publication'
+    training = 'training'
+
+
+class ExcludeTool(BaseModel):
+    tool_id: str
+
+
+class ExcludeToolRegex(BaseModel):
+    tool_id_regex: str
+
+
+class ExcludeTypes(BaseModel):
+    types: List[str]
+
+
+Exclusions = Union[
+    ExcludeTool,
+    ExcludeToolRegex,
+    ExcludeTypes,
+]
+OptionalExclusionList = Optional[List[Exclusions]]
+
+
+class Tool(BaseModel):
+    content_type: Literal['tool'] = Field(alias="type")
+    id: str
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Label(BaseModel):
+    content_type: Literal['label'] = Field(alias="type")
+    id: Optional[str]
+    text: str
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class LabelShortcut(BaseModel):
+    content_type = "simple_label"
+    label: str
+
+
+class Workflow(BaseModel):
+    content_type: Literal['workflow'] = Field(alias="type")
+    id: str
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class ItemsFrom(BaseModel):
+    content_type = "items_from"
+    items_from: str
+    excludes: OptionalExclusionList
+
+
+SectionContent = Union[
+    Tool,
+    Label,
+    LabelShortcut,
+    Workflow,
+    ItemsFrom,
+]
+
+
+class HasItems:
+
+    @property
+    def items_expanded(self):
+        # replace SectionAliases with individual SectionAlias objects
+        # replace LabelShortcuts with Labels
+        items = []
+        for item in self.items:
+            if isinstance(item, SectionAliases):
+                for section in item.sections:
+                    section_alias = SectionAlias(
+                        section=section,
+                        excludes=item.excludes,
+                    )
+                    items.append(section_alias)
+            elif isinstance(item, LabelShortcut):
+                new_item = Label(
+                    id=None,
+                    text=item.label,
+                    content_type="label",
+                )
+                items.append(new_item)
+            else:
+                items.append(item)
+        return items
+
+
+class Section(BaseModel, HasItems):
+    content_type: Literal['section'] = Field(alias="type")
+    id: Optional[str]
+    name: Optional[str]
+    items: Optional[List[SectionContent]]
+    excludes: OptionalExclusionList
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class SectionAlias(BaseModel):
+    content_type = "section_alias"
+    section: str
+    excludes: OptionalExclusionList
+
+
+class SectionAliases(BaseModel):
+    content_type = "section_aliases"
+    sections: List[str]
+    excludes: OptionalExclusionList
+
+
+RootContent = Union[
+    Section,
+    SectionAlias,
+    SectionAliases,
+    Tool,
+    Label,
+    LabelShortcut,
+    Workflow,
+    ItemsFrom,
+]
+
+
+class StaticToolBoxView(BaseModel, HasItems):
+    id: str
+    name: str
+    description: Optional[str]
+    view_type: StaticToolBoxViewTypeEnum = Field(alias="type")
+    items: List[RootContent]
+    excludes: Optional[List[Exclusions]]
+
+    @staticmethod
+    def from_dict(as_dict):
+        return StaticToolBoxView(**as_dict)
+
+    class Config:
+        allow_population_by_field_name = True
diff --git a/lib/galaxy/tool_util/toolbox/views/edam.py b/lib/galaxy/tool_util/toolbox/views/edam.py
new file mode 100644
index 00000000000..a5088711bd5
--- /dev/null
+++ b/lib/galaxy/tool_util/toolbox/views/edam.py
@@ -0,0 +1,168 @@
+import logging
+import os
+from enum import Enum
+from typing import Dict, List, Optional, Tuple
+
+from galaxy.tool_util.edam_util import (
+    load_edam_tree,
+    ROOT_OPERATION,
+    ROOT_TOPIC,
+)
+from galaxy.util import (
+    etree,
+    ExecutionTimer,
+)
+from .interface import (
+    ToolBoxRegistry,
+    ToolPanelView,
+    ToolPanelViewModel,
+    ToolPanelViewModelType,
+    walk_loaded_tools,
+)
+from ..panel import (
+    ToolPanelElements,
+    ToolSectionLabel,
+)
+
+log = logging.getLogger(__name__)
+
+
+class EdamPanelMode(str, Enum):
+    merged = "merged"
+    topics = "topics"
+    operations = "operations"
+
+
+class EdamToolPanelView(ToolPanelView):
+
+    def __init__(self, edam_ontology_path: Optional[str], mode: EdamPanelMode = EdamPanelMode.merged):
+        edam = load_edam_tree(None if not edam_ontology_path or not os.path.exists(edam_ontology_path) else edam_ontology_path)
+        self.edam = edam
+        self.mode = mode
+        self.include_topics = mode in [EdamPanelMode.merged, EdamPanelMode.topics]
+        self.include_operations = mode in [EdamPanelMode.merged, EdamPanelMode.operations]
+
+    def apply_view(self, base_tool_panel: ToolPanelElements, toolbox_registry: ToolBoxRegistry) -> ToolPanelElements:
+        execution_timer = ExecutionTimer()
+
+        # Find the children of the top level topics
+        operations_list = [ROOT_OPERATION] + list(self._edam_children_of(ROOT_OPERATION))
+        topics_list = [ROOT_TOPIC] + list(self._edam_children_of(ROOT_TOPIC))
+
+        # Sort them (by english label)
+        # operations = sorted(operations, key=lambda x: self.edam[x]['label'])
+        # topics = sorted(topics, key=lambda x: self.edam[x]['label'])
+
+        # Convert these to list of dicts, wherein we'll add our tools/etc.
+        operations: Dict[str, Dict] = {
+            x: {}
+            for x in operations_list
+        }
+        topics: Dict[str, Dict] = {
+            x: {}
+            for x in topics_list
+        }
+        uncategorized: List[Tuple] = []
+
+        for tool_id, key, val, val_name in walk_loaded_tools(base_tool_panel, toolbox_registry):
+            for term in self._get_edam_sec(val):
+                if term == 'uncategorized':
+                    uncategorized.append((tool_id, key, val, val_name))
+                else:
+                    for path in self.edam[term]['path']:
+                        if len(path) == 1:
+                            t = term
+                        else:
+                            t = path[0]
+
+                        if path[0].startswith('operation_'):
+                            operations[t][tool_id] = (term, tool_id, key, val, val_name)
+                        elif path[0].startswith('topic_'):
+                            topics[t][tool_id] = (term, tool_id, key, val, val_name)
+
+        new_panel = ToolPanelElements()
+        for term in sorted(operations.keys(), key=lambda x: self._sort_edam_key(x)):
+            if len(operations[term].keys()) == 0:
+                continue
+
+            elem = etree.Element('label')
+            elem.attrib['text'] = self.edam[term]['label']
+            elem.attrib['id'] = term
+            new_panel[f"label_{term}"] = ToolSectionLabel(elem)
+
+            for (term, tool_id, key, val, val_name) in operations[term].values():
+                section = new_panel.get_or_create_section(term, self.edam[term]['label'])
+
+                toolbox_registry.add_tool_to_tool_panel_view(val, section)
+                new_panel.record_section_for_tool_id(tool_id, key, val_name)
+
+        for term in sorted(topics.keys(), key=lambda x: self._sort_edam_key(x)):
+            if len(topics[term].keys()) == 0:
+                continue
+
+            elem = etree.Element('label')
+            elem.attrib['text'] = self.edam[term]['label']
+            elem.attrib['id'] = term
+            new_panel[f"label_{term}"] = ToolSectionLabel(elem)
+
+            for (term, tool_id, key, val, val_name) in topics[term].values():
+                section = new_panel.get_or_create_section(term, self.edam[term]['label'])
+                toolbox_registry.add_tool_to_tool_panel_view(val, section)
+                new_panel.record_section_for_tool_id(tool_id, key, val_name)
+
+        section = new_panel.get_or_create_section('uncategorized', 'Uncategorized')
+        for (tool_id, key, val, val_name) in uncategorized:
+            toolbox_registry.add_tool_to_tool_panel_view(val, section)
+            new_panel.record_section_for_tool_id(tool_id, key, val_name)
+        log.debug("Loading EDAM tool panel finished %s", execution_timer)
+        return new_panel
+
+    def _get_edam_sec(self, tool):
+        edam = []
+        if self.include_operations:
+            edam.extend(tool.edam_operations)
+        if self.include_topics:
+            edam.extend(tool.edam_topics)
+        if len(edam) > 0:
+            for term in edam:
+                yield term
+        else:
+            yield 'uncategorized'
+
+    def _sort_edam_key(self, x):
+        if x in (ROOT_OPERATION, ROOT_TOPIC):
+            return f"!{x}"
+        else:
+            return self.edam[x]['label']
+
+    def _edam_children_of(self, parentTerm):
+        for term in self.edam.keys():
+            if parentTerm in self.edam[term]['parents']:
+                yield term
+
+    def to_model(self) -> ToolPanelViewModel:
+        mode = self.mode
+        if mode == EdamPanelMode.merged:
+            model_id = "ontology:edam_merged"
+            name = "EDAM Operations and Topics"
+            description = "Tools are grouped using both annotated operation and topic information (if availabled)."
+        elif mode == EdamPanelMode.operations:
+            model_id = "ontology:edam_operations"
+            name = "EDAM Operations"
+            description = "Tools are grouped using annotated EDAM operation information (if availabled)."
+        elif mode == EdamPanelMode.topics:
+            model_id = "ontology:edam_topics"
+            name = "EDAM Topics"
+            description = "Tools are grouped using annotated EDAM topic information (if availabled)."
+        else:
+            raise AssertionError(f"Invalid EDAM mode encountered {mode}")
+        model_class = self.__class__.__name__
+        view_type = ToolPanelViewModelType.ontology
+        return ToolPanelViewModel(
+            id=model_id,
+            name=name,
+            description=description,
+            model_class=model_class,
+            view_type=view_type,
+            searchable=True,
+        )
diff --git a/lib/galaxy/tool_util/toolbox/views/interface.py b/lib/galaxy/tool_util/toolbox/views/interface.py
new file mode 100644
index 00000000000..e7bea38174e
--- /dev/null
+++ b/lib/galaxy/tool_util/toolbox/views/interface.py
@@ -0,0 +1,75 @@
+from abc import abstractmethod
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+from ..panel import (
+    HasPanelItems,
+    panel_item_types,
+    ToolPanelElements,
+)
+
+
+class ToolPanelViewModelType(str, Enum):
+    default_type = 'default'
+    generic = 'generic'
+    activity = 'activity'
+    ontology = 'ontology'
+    publication = 'publication'
+    training = 'training'
+
+
+class ToolPanelViewModel(BaseModel):
+    """A view of ToolPanelView objects serialized for the API."""
+    id: str
+    model_class: str
+    name: str
+    description: Optional[str]
+    view_type: ToolPanelViewModelType
+    searchable: bool  # Allow for more dynamic views that don't plug into fixed search indicies in the future...
+
+
+class ToolBoxRegistry:
+    """View of ToolBox provided to ToolPanelView to reason about tools loaded."""
+
+    @abstractmethod
+    def has_tool(self, tool_id: str) -> bool:
+        """Return bool indicating if tool with specified id is loaded."""
+
+    @abstractmethod
+    def get_tool(self, tool_id: str):
+        """Return tool with supplied tool id."""
+
+    @abstractmethod
+    def get_workflow(self, id: str):
+        """Return workflow from panel with supplied id."""
+
+    @abstractmethod
+    def add_tool_to_tool_panel_view(self, tool, tool_panel_component: HasPanelItems) -> None:
+        """Add tool to the tool panel view component (root or section)."""
+
+
+class ToolPanelView:
+
+    @abstractmethod
+    def apply_view(self, base_tool_panel: ToolPanelElements, toolbox_registry: ToolBoxRegistry) -> ToolPanelElements:
+        """Consume tool panel state and return custom tool panel view."""
+
+    @abstractmethod
+    def to_model(self) -> ToolPanelViewModel:
+        """Convert abstract description to dictionary description to emit via the API."""
+
+
+def walk_loaded_tools(tool_panel: ToolPanelElements, toolbox_registry: ToolBoxRegistry):
+    for key, item_type, val in tool_panel.panel_items_iter():
+        if item_type == panel_item_types.TOOL:
+            tool_id = key.replace('tool_', '', 1)
+            if toolbox_registry.has_tool(tool_id):
+                yield (tool_id, key, val, val.name)
+        elif item_type == panel_item_types.SECTION:
+            for section_key, section_item_type, section_val in val.panel_items_iter():
+                if section_item_type == panel_item_types.TOOL:
+                    tool_id = section_key.replace('tool_', '', 1)
+                    if toolbox_registry.has_tool(tool_id):
+                        yield (tool_id, key, section_val, val.name)
diff --git a/lib/galaxy/tool_util/toolbox/views/sources.py b/lib/galaxy/tool_util/toolbox/views/sources.py
new file mode 100644
index 00000000000..286a46a07ee
--- /dev/null
+++ b/lib/galaxy/tool_util/toolbox/views/sources.py
@@ -0,0 +1,52 @@
+import logging
+import os
+from typing import Dict, List
+
+import yaml
+
+from galaxy.util import config_directories_from_setting
+from .definitions import StaticToolBoxView
+
+log = logging.getLogger(__name__)
+
+EXTENSIONS = [".yml", ".yaml", ".json"]
+
+
+class StaticToolBoxViewSources:
+    view_directories: List[str]
+    view_dicts: List[Dict]
+
+    def __init__(self, view_directories=None, view_dicts=None):
+        self.view_directories = config_directories_from_setting(view_directories) or []
+        self.view_dicts = view_dicts or []
+
+    def get_definitions(self) -> List[StaticToolBoxView]:
+        view_definitions = []
+
+        for view_dict in self.view_dicts:
+            view_definitions.append(StaticToolBoxView.from_dict(view_dict))
+
+        for view_directory in self.view_directories:
+            if not os.path.exists(view_directory):
+                log.warning(f"Failed to find toolbox view directory {view_directory}")
+
+            for filename in os.listdir(view_directory):
+                if not looks_like_view_source_filename(filename):
+                    continue
+
+                view_path = os.path.join(view_directory, filename)
+                with open(view_path) as f:
+                    view_dict = yaml.safe_load(f)
+                if "id" not in view_dict:
+                    file_id = os.path.splitext(filename)[0]
+                    view_dict["id"] = file_id
+                view_definitions.append(StaticToolBoxView.from_dict(view_dict))
+
+        return view_definitions
+
+
+def looks_like_view_source_filename(filename: str) -> bool:
+    for ext in EXTENSIONS:
+        if filename.endswith(ext):
+            return True
+    return False
diff --git a/lib/galaxy/tool_util/toolbox/views/static.py b/lib/galaxy/tool_util/toolbox/views/static.py
new file mode 100644
index 00000000000..60c094d3ee8
--- /dev/null
+++ b/lib/galaxy/tool_util/toolbox/views/static.py
@@ -0,0 +1,168 @@
+import logging
+import re
+
+from .definitions import (
+    ExcludeTool,
+    ExcludeToolRegex,
+    ExcludeTypes,
+    Section,
+    StaticToolBoxView,
+    Workflow,
+)
+from .interface import (
+    ToolBoxRegistry,
+    ToolPanelView,
+    ToolPanelViewModel,
+    ToolPanelViewModelType,
+)
+from ..panel import (
+    ToolPanelElements,
+    ToolSection,
+    ToolSectionLabel,
+)
+
+
+log = logging.getLogger(__name__)
+
+
+def build_filter(excludes_):
+    excludes = excludes_ or []
+
+    def filter_function(panel_key, panel_value):
+        for exclude in excludes:
+            if panel_key.startswith("tool_"):
+                if isinstance(exclude, ExcludeTool):
+                    if panel_value.id == exclude.tool_id:
+                        return False
+                    if panel_value.old_id == exclude.tool_id:
+                        return False
+
+                if isinstance(exclude, ExcludeToolRegex):
+                    if re.match(exclude.tool_id_regex, panel_value.id):
+                        return False
+                    if re.match(exclude.tool_id_regex, panel_value.old_id):
+                        return False
+
+            if isinstance(exclude, ExcludeTypes):
+                if panel_key.startswith("label_") and "label" in exclude.types:
+                    return False
+                if panel_key.startswith("tool_") and "tool" in exclude.types:
+                    return False
+                if panel_key.startswith("workflow_") and "workflow" in exclude.types:
+                    return False
+
+        return True
+
+    return filter_function
+
+
+class StaticToolPanelView(ToolPanelView):
+    _definition: StaticToolBoxView
+
+    def __init__(self, definition: StaticToolBoxView):
+        self._definition = definition
+
+    def apply_view(self, base_tool_panel: ToolPanelElements, toolbox_registry: ToolBoxRegistry) -> ToolPanelElements:
+
+        def apply_filter(definition, elems):
+            if definition.excludes:
+                elems.apply_filter(build_filter(definition.excludes))
+
+        def definition_with_items_to_panel(definition, allow_sections: bool = True):
+            new_panel = ToolPanelElements()
+            for element in definition.items_expanded:
+                if element.content_type == "section":
+                    assert allow_sections
+                    section_def: Section = element
+                    section: ToolSection
+                    assert section_def.id is not None or section_def.name is not None
+                    if element.items:
+                        panel = definition_with_items_to_panel(section_def, allow_sections=False)
+                        section = ToolSection()
+                        if section_def.name is not None:
+                            name = section_def.name
+                        else:
+                            assert section_def.id is not None
+                            name = section_def.id
+                        section.name = name
+                        if section_def.id is not None:
+                            section.id = section_def.id
+                        else:
+                            # TODO: there has to be tool shed code to do this in a consistent way... where is it?
+                            section.id = name.replace(" ", "-").lower()
+                        section.elems = panel
+                    else:
+                        closest_section = base_tool_panel.closest_section(section_def.id, section_def.name)
+                        if closest_section is None:
+                            log.warning(f"Failed to find matching section for (id, name) = ({section_def.id}, {section_def.name})")
+                            continue
+                        section = closest_section.copy()
+                        if section_def.id is not None:
+                            section.id = section_def.id
+                        if section_def.name is not None:
+                            section.name = section_def.name
+                        apply_filter(section_def, section.elems)
+                    new_panel.append_section(section.id, section)
+                elif element.content_type == "section_alias":
+                    assert allow_sections
+                    closest_section = base_tool_panel.closest_section(element.section, element.section)
+                    if closest_section is None:
+                        log.warning(f"Failed to find matching section for (id, name) = ({element.section}, {element.section})")
+                        continue
+                    section = closest_section.copy()
+                    apply_filter(element, section.elems)
+                    new_panel.append_section(section.id, section)
+                elif element.content_type == "label":
+                    as_dict = {
+                        "id": element.id or element.text.lower().replace(" ", "-"),
+                        "text": element.text,
+                        "type": "label",
+                    }
+                    label = ToolSectionLabel(as_dict)
+                    key = f"label_{label.id}"
+                    new_panel[key] = label
+                elif element.content_type == "tool":
+                    tool_id = element.id
+                    if not toolbox_registry.has_tool(tool_id):
+                        log.warning(f"Failed to find tool_id {tool_id} from parent toolbox, cannot load into panel view")
+                        continue
+                    tool = toolbox_registry.get_tool(tool_id)
+                    toolbox_registry.add_tool_to_tool_panel_view(tool, new_panel)
+                elif element.content_type == "workflow":
+                    workflow_def: Workflow = element
+                    workflow = toolbox_registry.get_workflow(element.id)
+                    panel_id = f"workflow_{workflow_def.id}"
+                    new_panel[panel_id] = workflow
+                elif element.content_type == "items_from":
+                    closest_section = base_tool_panel.closest_section(element.items_from, element.items_from)
+                    if closest_section is None:
+                        log.warning(f"Failed to find matching section for (id, name) = ({element.items_from}, None)")
+                        continue
+                    elems = closest_section.elems.copy()
+                    apply_filter(element, elems)
+                    for key, item in elems.items():
+                        new_panel[key] = item
+                else:
+                    raise AssertionError("Unknown static toolbox configuration element encountered.")
+
+            if definition.excludes:
+                new_panel.apply_filter(build_filter(definition.excludes))
+
+            return new_panel
+
+        return definition_with_items_to_panel(self._definition)
+
+    def to_model(self) -> ToolPanelViewModel:
+        model_id = self._definition.id
+        name = self._definition.name
+        description = self._definition.description
+        view_type = ToolPanelViewModelType[self._definition.view_type.value]
+        model_class = self.__class__.__name__
+        return ToolPanelViewModel(
+            id=model_id,
+            name=name,
+            description=description,
+            model_class=model_class,
+            view_type=view_type,
+            searchable=True,
+        )
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index efbb7950443..fd3fa256697 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -48,6 +48,7 @@ from galaxy.tool_util.parser import (
 from galaxy.tool_util.parser.xml import XmlPageSource
 from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
 from galaxy.tool_util.toolbox import BaseGalaxyToolBox
+from galaxy.tool_util.toolbox.views.sources import StaticToolBoxViewSources
 from galaxy.tools import expressions
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
@@ -268,10 +269,17 @@ class ToolBox(BaseGalaxyToolBox):
         # FIXME: ./
         if tool_root_dir == './tools':
             tool_root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'bundled'))
+        view_sources = StaticToolBoxViewSources(
+            view_directories=app.config.panel_views_dir,
+            view_dicts=app.config.panel_views,
+        )
+        default_panel_view = app.config.default_panel_view
         super().__init__(
             config_filenames=config_filenames,
             tool_root_dir=tool_root_dir,
             app=app,
+            view_sources=view_sources,
+            default_panel_view=default_panel_view,
             save_integrated_tool_panel=save_integrated_tool_panel,
         )
 
diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py
index 849139b6d31..a324321e9e9 100644
--- a/lib/galaxy/tools/search/__init__.py
+++ b/lib/galaxy/tools/search/__init__.py
@@ -53,12 +53,44 @@ def get_or_create_index(index_dir: str, schema: Schema) -> index.Index:
 
 
 class ToolBoxSearch:
+    """Support searching across all fixed panel views in a toolbox.
+
+    Search is delegated off to ToolPanelViewSearch for each panel object.
+    """
+
+    def __init__(self, toolbox, index_dir: str, index_help: bool = True):
+        panel_searches = {}
+        for panel_view in toolbox.panel_views():
+            panel_view_id = panel_view.id
+            panel_index_dir = os.path.join(index_dir, panel_view_id)
+            panel_searches[panel_view_id] = ToolPanelViewSearch(toolbox, panel_view_id, panel_index_dir, index_help=index_help)
+        self.panel_searches = panel_searches
+        # We keep track of how many times the tool index has been rebuilt.
+        # We start at -1, so that after the first index the count is at 0,
+        # which is the same as the toolbox reload count. This way we can skip
+        # reindexing if the index count is equal to the toolbox reload count.
+        self.index_count = -1
+
+    def build_index(self, tool_cache, index_help: bool = True) -> None:
+        self.index_count += 1
+        for panel_search in self.panel_searches.values():
+            panel_search.build_index(tool_cache, index_help=index_help)
+
+    def search(self, *args, **kwd) -> List[str]:
+        panel_view = kwd.pop("panel_view")
+        if panel_view not in self.panel_searches:
+            raise KeyError(f"Unknown panel_view specified {panel_view}")
+        panel_search = self.panel_searches[panel_view]
+        return panel_search.search(*args, **kwd)
+
+
+class ToolPanelViewSearch:
     """
     Support searching tools in a toolbox. This implementation uses
     the Whoosh search library.
     """
 
-    def __init__(self, toolbox, index_dir: str, index_help: bool = True):
+    def __init__(self, toolbox, panel_view_id: str, index_dir: str, index_help: bool = True):
         self.schema = Schema(id=ID(stored=True, unique=True),
                              old_id=ID,
                              stub=KEYWORD,
@@ -70,12 +102,8 @@ class ToolBoxSearch:
         self.rex = analysis.RegexTokenizer()
         self.index_dir = index_dir
         self.toolbox = toolbox
+        self.panel_view_id = panel_view_id
         self.index = self._index_setup()
-        # We keep track of how many times the tool index has been rebuilt.
-        # We start at -1, so that after the first index the count is at 0,
-        # which is the same as the toolbox reload count. This way we can skip
-        # reindexing if the index count is equal to the toolbox reload count.
-        self.index_count = -1
 
     def _index_setup(self) -> index.Index:
         return get_or_create_index(index_dir=self.index_dir, schema=self.schema)
@@ -85,8 +113,7 @@ class ToolBoxSearch:
         Prepare search index for tools loaded in toolbox.
         Use `tool_cache` to determine which tools need indexing and which tools should be expired.
         """
-        log.debug('Starting to build toolbox index.')
-        self.index_count += 1
+        log.debug(f"Starting to build toolbox index of panel {self.panel_view_id}.")
         execution_timer = ExecutionTimer()
         with self.index.reader() as reader:
             # Index ocasionally contains empty stored fields
@@ -106,7 +133,7 @@ class ToolBoxSearch:
                 writer.delete_by_term('id', tool_id)
             for tool_id in tool_cache._new_tool_ids - indexed_tool_ids:
                 tool = self.toolbox.get_tool(tool_id)
-                if tool and tool.is_latest_version:
+                if tool and tool.is_latest_version and self.toolbox.panel_has_tool(tool, self.panel_view_id):
                     if tool.hidden:
                         # we check if there is an older tool we can return
                         if tool.lineage:
@@ -121,7 +148,7 @@ class ToolBoxSearch:
                             continue
                     add_doc_kwds = self._create_doc(tool_id=tool_id, tool=tool, index_help=index_help)
                     writer.update_document(**add_doc_kwds)
-        log.debug("Toolbox index finished %s", execution_timer)
+        log.debug(f"Toolbox index of panel {self.panel_view_id} finished {execution_timer}")
 
     def _create_doc(self, tool_id: str, tool, index_help: bool = True) -> Dict[str, str]:
         #  Do not add data managers to the public index
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index 5b65f247666..0d7eebb9e73 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -46,6 +46,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
 
         :param in_panel: if true, tools are returned in panel structure,
                          including sections and labels
+        :param view: ToolBox view to apply (default is 'default')
         :param trackster: if true, only tools that are compatible with
                           Trackster are returned
         :param q: if present search on the given query will be performed
@@ -59,6 +60,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
         q = kwds.get('q', '')
         tool_id = kwds.get('tool_id', '')
         tool_help = util.string_as_bool(kwds.get('tool_help', 'False'))
+        view = kwds.get("view", None)
 
         # Find whether to search.
         if q:
@@ -69,7 +71,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
                 else:
                     hits = None
             else:
-                hits = self._search(q)
+                hits = self._search(q, view)
             results = []
             if hits:
                 for hit in hits:
@@ -90,7 +92,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
 
         # Return everything.
         try:
-            return self.app.toolbox.to_dict(trans, in_panel=in_panel, trackster=trackster, tool_help=tool_help)
+            return self.app.toolbox.to_dict(trans, in_panel=in_panel, trackster=trackster, tool_help=tool_help, view=view)
         except Exception:
             raise exceptions.InternalServerError("Error: Could not convert toolbox to dictionary")
 
@@ -377,7 +379,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
                     detected_versions.append(tool.version)
         return detected_versions
 
-    def _search(self, q):
+    def _search(self, q, view):
         """
         Perform the search on the given query.
         Boosts and numer of results are configurable in galaxy.ini file.
@@ -388,6 +390,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
         :return:      Dictionary containing the tools' ids of the best hits.
         :return type: dict
         """
+        panel_view = view or self.app.config.default_panel_view
         tool_name_boost = self.app.config.get('tool_name_boost', 9)
         tool_id_boost = self.app.config.get('tool_id_boost', 9)
         tool_section_boost = self.app.config.get('tool_section_boost', 3)
@@ -401,6 +404,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin):
         tool_ngram_maxsize = self.app.config.get('tool_ngram_maxsize', 4)
 
         results = self.app.toolbox_search.search(q=q,
+                                                 panel_view=panel_view,
                                                  tool_name_boost=tool_name_boost,
                                                  tool_id_boost=tool_id_boost,
                                                  tool_section_boost=tool_section_boost,
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 4d2739e3351..f758026b3f2 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -2440,15 +2440,16 @@ mapping:
           stable API. (The module state definitions may change and workflows built using
           these modules may not function in the future.)
 
-      enable_beta_edam_toolbox:
-        type: bool
-        default: false
+      edam_panel_views:
+        type: str
+        default: 'operations,topics'
         required: false
         desc: |
-          Enable beta EDAM organised toolbox which ignores admin managed
-          sections in lieu of EDAM topics only.
+          Comma-separated list of the EDAM panel views to load - choose from merged, operations, topics.
+          Set to empty string to disable EDAM all together. Set default_panel_view to 'ontology:edam_topics'
+          to override default tool panel to use an EDAM view.
 
-      beta_edam_toolbox_ontology_path:
+      edam_toolbox_ontology_path:
         type: str
         default: EDAM.tsv
         path_resolves_to: data_dir
@@ -2456,6 +2457,34 @@ mapping:
         desc: |
           Sets the path to EDAM ontology file - if the path doesn't exist PyPI package data will be loaded.
 
+      panel_views_dir:
+        type: str
+        default: config/plugins/activities
+        required: false
+        desc: |
+          Directory to check out for toolbox tool panel views. The path is relative to the
+          Galaxy root dir.  To use an absolute path begin the path with '/'.  This is a
+          comma-separated list.
+
+      panel_views:
+        type: seq
+        sequence:
+          - type: any
+        required: false
+        desc: |
+          Definitions of static toolbox panel views embedded directly in the config instead of reading
+          YAML from directory with panel_views_dir.
+
+      default_panel_view:
+        type: str
+        default: default
+        required: false
+        desc: |
+          Default tool panel view for the current Galaxy configuration. This should refer to an id of
+          a panel view defined using the panel_views or panel_views_dir configuration options or an
+          EDAM panel view. The default panel view is simply called `default` and refers to the tool
+          panel state defined by the integrated tool panel.
+
       default_workflow_export_format:
         type: str
         default: ga
diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py
index 5b4882b7040..e0868ba2115 100644
--- a/lib/galaxy_test/driver/driver_util.py
+++ b/lib/galaxy_test/driver/driver_util.py
@@ -15,6 +15,7 @@ import sys
 import tempfile
 import threading
 import time
+from typing import Optional
 from urllib.parse import urlparse
 
 import nose.config
@@ -155,6 +156,7 @@ def setup_galaxy_config(
     conda_auto_init=False,
     conda_auto_install=False,
     use_shared_connection_for_amqp=False,
+    allow_tool_conf_override: bool = True,
 ):
     """Setup environment and build config for test Galaxy instance."""
     # For certain docker operations this needs to be evaluated out - e.g. for cwltool.
@@ -165,6 +167,7 @@ def setup_galaxy_config(
     new_file_path = tempfile.mkdtemp(prefix='new_files_path_', dir=tmpdir)
     job_working_directory = tempfile.mkdtemp(prefix='job_working_directory_', dir=tmpdir)
 
+    user_library_import_dir: Optional[str]
     if use_test_file_dir:
         first_test_file_dir = ensure_test_file_dir_set()
         if not os.path.isabs(first_test_file_dir):
@@ -196,7 +199,10 @@ def setup_galaxy_config(
     # For storing Data Manager outputs and .loc files so that real ones don't get clobbered
     galaxy_data_manager_data_path = tempfile.mkdtemp(prefix='data_manager_tool-data', dir=tmpdir)
 
-    tool_conf = os.environ.get('GALAXY_TEST_TOOL_CONF', default_tool_conf)
+    if allow_tool_conf_override:
+        tool_conf = os.environ.get('GALAXY_TEST_TOOL_CONF', default_tool_conf)
+    else:
+        tool_conf = default_tool_conf
     conda_auto_install = os.environ.get('GALAXY_TEST_CONDA_AUTO_INSTALL', conda_auto_install)
     conda_auto_init = os.environ.get('GALAXY_TEST_CONDA_AUTO_INIT', conda_auto_init)
     conda_prefix = os.environ.get('GALAXY_TEST_CONDA_PREFIX')
@@ -1002,7 +1008,8 @@ class GalaxyTestDriver(TestDriver):
         else:
             default_tool_conf = getattr(config_object, "default_tool_conf", None)
             datatypes_conf_override = getattr(config_object, "datatypes_conf_override", None)
-
+        allow_tool_conf_override = getattr(config_object, "allow_tool_conf_override", True)
+        self.allow_tool_conf_override = allow_tool_conf_override
         self.default_tool_conf = default_tool_conf
         self.datatypes_conf_override = datatypes_conf_override
 
@@ -1047,7 +1054,8 @@ class GalaxyTestDriver(TestDriver):
                         log_format=self.log_format,
                         conda_auto_init=getattr(config_object, "conda_auto_init", False),
                         conda_auto_install=getattr(config_object, "conda_auto_install", False),
-                        use_shared_connection_for_amqp=getattr(config_object, "use_shared_connection_for_amqp", False)
+                        use_shared_connection_for_amqp=getattr(config_object, "use_shared_connection_for_amqp", False),
+                        allow_tool_conf_override=self.allow_tool_conf_override,
                     )
                     galaxy_config = setup_galaxy_config(
                         galaxy_db_path,
diff --git a/packages/tool_util/requirements.txt b/packages/tool_util/requirements.txt
index f0ac0b2b776..85b88243440 100644
--- a/packages/tool_util/requirements.txt
+++ b/packages/tool_util/requirements.txt
@@ -2,3 +2,5 @@ galaxy-util>=20.1.0.dev0
 galaxy-containers
 lxml
 sortedcontainers
+pydantic
+typing-extensions
diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py
index d7863021cd3..2f4d88de6f6 100644
--- a/packages/tool_util/setup.py
+++ b/packages/tool_util/setup.py
@@ -43,6 +43,7 @@ PACKAGES = [
     'galaxy.tool_util.toolbox',
     'galaxy.tool_util.toolbox.filters',
     'galaxy.tool_util.toolbox.lineages',
+    'galaxy.tool_util.toolbox.views',
     'galaxy.tool_util.verify',
     'galaxy.tool_util.verify.asserts',
 ]
diff --git a/test/integration/panel_views_1/custom_1.yml b/test/integration/panel_views_1/custom_1.yml
new file mode 100644
index 00000000000..b3bafb71850
--- /dev/null
+++ b/test/integration/panel_views_1/custom_1.yml
@@ -0,0 +1,14 @@
+id: my-custom
+name: Custom Panel Filter
+type: generic
+items:
+- type: label
+  text: The Start
+- type: tool
+  id: empty_list
+- type: label
+  text: The Middle
+- type: tool
+  id: count_list
+- type: label
+  text: The End
diff --git a/test/integration/panel_views_1/custom_10.yml b/test/integration/panel_views_1/custom_10.yml
new file mode 100644
index 00000000000..2141fdfecb4
--- /dev/null
+++ b/test/integration/panel_views_1/custom_10.yml
@@ -0,0 +1,4 @@
+name: Filtered Test Section
+type: generic
+items:
+- sections: [test,filter]
diff --git a/test/integration/panel_views_1/custom_2.yml b/test/integration/panel_views_1/custom_2.yml
new file mode 100644
index 00000000000..b9c6dc229ff
--- /dev/null
+++ b/test/integration/panel_views_1/custom_2.yml
@@ -0,0 +1,7 @@
+name: Filtered Test Section
+type: activity
+items:
+- type: section
+  id: test
+  excludes:
+  - tool_id: multi_data_optional
diff --git a/test/integration/panel_views_1/custom_3.yml b/test/integration/panel_views_1/custom_3.yml
new file mode 100644
index 00000000000..80a5e07c028
--- /dev/null
+++ b/test/integration/panel_views_1/custom_3.yml
@@ -0,0 +1,8 @@
+name: Filtered Test Section
+type: activity
+items:
+- type: section
+  id: test
+  excludes:
+  - tool_id_regex: 'multi_data_.*'
+  - tool_id_regex: '.*_text_option'
diff --git a/test/integration/panel_views_1/custom_4.yml b/test/integration/panel_views_1/custom_4.yml
new file mode 100644
index 00000000000..e6e21e5376a
--- /dev/null
+++ b/test/integration/panel_views_1/custom_4.yml
@@ -0,0 +1,15 @@
+name: Custom Panel 4
+type: generic
+items:
+- type: label
+  text: The Start
+- type: tool
+  id: empty_list
+- type: label
+  text: The Middle
+- type: tool
+  id: count_list
+- type: label
+  text: The End
+excludes:
+- types: [label]
diff --git a/test/integration/panel_views_1/custom_5.yml b/test/integration/panel_views_1/custom_5.yml
new file mode 100644
index 00000000000..e7477aa923f
--- /dev/null
+++ b/test/integration/panel_views_1/custom_5.yml
@@ -0,0 +1,9 @@
+name: Merged Section Example
+type: generic
+items:
+- type: section
+  id: my-new-section
+  name: My New Section
+  items:
+  - items_from: test
+  - items_from: filter
diff --git a/test/integration/panel_views_1/custom_6.yml b/test/integration/panel_views_1/custom_6.yml
new file mode 100644
index 00000000000..9567ba32413
--- /dev/null
+++ b/test/integration/panel_views_1/custom_6.yml
@@ -0,0 +1,13 @@
+name: Custom Panel in a New Section
+type: generic
+items:
+- type: section
+  name: My Completely New Sectin
+  items:
+  - type: label
+    text: The Start
+  - type: tool
+    id: empty_list
+  - label: The Middle
+  - type: tool
+    id: count_list
diff --git a/test/integration/panel_views_1/custom_7.yml b/test/integration/panel_views_1/custom_7.yml
new file mode 100644
index 00000000000..299246b9939
--- /dev/null
+++ b/test/integration/panel_views_1/custom_7.yml
@@ -0,0 +1,13 @@
+name: Merged Section Example
+type: generic
+items:
+- type: section
+  id: my-new-section
+  name: My New Section
+  items:
+  - items_from: test
+    excludes:
+    -  tool_id_regex: '.*column.*'
+  - type: label
+    text: A cool label is here
+  - items_from: filter
diff --git a/test/integration/panel_views_1/custom_8.yml b/test/integration/panel_views_1/custom_8.yml
new file mode 100644
index 00000000000..ab913f8164a
--- /dev/null
+++ b/test/integration/panel_views_1/custom_8.yml
@@ -0,0 +1,9 @@
+name: Merged Section Example
+type: generic
+items:
+- type: section
+  id: my-new-section
+  name: My New Section
+  items:
+  - items_from: Test Section
+  - items_from: filter
diff --git a/test/integration/panel_views_1/custom_9.yml b/test/integration/panel_views_1/custom_9.yml
new file mode 100644
index 00000000000..17ca47e6417
--- /dev/null
+++ b/test/integration/panel_views_1/custom_9.yml
@@ -0,0 +1,7 @@
+name: Filtered Test Section
+type: activity
+items:
+- section: test
+  excludes:
+  - tool_id_regex: 'multi_data_.*'
+  - tool_id_regex: '.*_text_option'
diff --git a/test/integration/panel_views_1/filter.yml b/test/integration/panel_views_1/filter.yml
new file mode 100644
index 00000000000..0afa0d6b211
--- /dev/null
+++ b/test/integration/panel_views_1/filter.yml
@@ -0,0 +1,5 @@
+name: Filtering Tools
+type: activity
+items:
+- type: section
+  id: filter
diff --git a/test/integration/test_edam_toolbox.py b/test/integration/test_edam_toolbox.py
index 66b0ab7503c..6509ab750ac 100644
--- a/test/integration/test_edam_toolbox.py
+++ b/test/integration/test_edam_toolbox.py
@@ -5,15 +5,13 @@ class EdamToolboxIntegrationTestCase(integration_util.IntegrationTestCase):
 
     framework_tool_and_types = True
 
-    def setUp(self):
-        super().setUp()
-
     @classmethod
     def handle_galaxy_config_kwds(cls, config):
-        config["enable_beta_edam_toolbox"] = True
+        config["edam_panel_views"] = "merged"
 
     def test_edam_toolbox(self):
-        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True))
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="ontology:edam_merged"))
+        index.raise_for_status()
         index_as_list = index.json()
         sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
         section_names = [s["name"] for s in sections]
@@ -22,3 +20,41 @@ class EdamToolboxIntegrationTestCase(integration_util.IntegrationTestCase):
         mapping_section_elems = mapping_section["elems"]
         # make sure our mapper tool was mapped using Edam correctly...
         assert [x for x in mapping_section_elems if x["id"] == "mapper"]
+
+        config_index = self.galaxy_interactor.get("configuration")
+        config_index.raise_for_status()
+        panel_views = config_index.json()["panel_views"]
+        assert len(panel_views) > 1
+        assert isinstance(panel_views, dict)
+        edam_panel_view = panel_views["ontology:edam_merged"]
+        assert edam_panel_view["view_type"] == "ontology"
+
+
+class EdamToolboxDefaultIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    framework_tool_and_types = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["edam_panel_views"] = "topics"
+        config["default_panel_view"] = "ontology:edam_topics"
+
+    def test_edam_toolbox(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True))
+        index.raise_for_status()
+        index_as_list = index.json()
+        sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
+        section_names = [s["name"] for s in sections]
+        assert "Mapping" in section_names
+        mapping_section = [s for s in sections if s["name"] == "Mapping"][0]
+        mapping_section_elems = mapping_section["elems"]
+        # make sure our mapper tool was mapped using Edam correctly...
+        assert [x for x in mapping_section_elems if x["id"] == "mapper"]
+
+        config_index = self.galaxy_interactor.get("configuration")
+        config_index.raise_for_status()
+        panel_views = config_index.json()["panel_views"]
+        assert len(panel_views) > 1
+        assert isinstance(panel_views, dict)
+        edam_panel_view = panel_views["ontology:edam_topics"]
+        assert edam_panel_view["view_type"] == "ontology"
diff --git a/test/integration/test_panel_views.py b/test/integration/test_panel_views.py
new file mode 100644
index 00000000000..38fcb413ef2
--- /dev/null
+++ b/test/integration/test_panel_views.py
@@ -0,0 +1,178 @@
+import os
+
+from galaxy_test.driver import integration_util
+
+THIS_DIR = os.path.dirname(__file__)
+PANEL_VIEWS_DIR_1 = os.path.join(THIS_DIR, "panel_views_1")
+
+
+class PanelViewsFromDirectoryIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    framework_tool_and_types = True
+    allow_tool_conf_override = False
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["panel_views_dir"] = PANEL_VIEWS_DIR_1
+
+    def test_section_copy(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="filter"))
+        index_as_list = index.json()
+        sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
+        section_names = [s["name"] for s in sections]
+        assert len(section_names) == 1
+        assert "For Tours" in section_names
+
+    def test_custom_label_order(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="my-custom"))
+        verify_my_custom(index)
+
+    def test_filtering_sections_by_tool_id(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_2"))
+        index.raise_for_status()
+        index_as_list = index.json()
+        sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
+        assert len(sections) == 1
+        section = sections[0]
+        tools = section["elems"]
+        assert len(tools) == 3, len(tools)
+
+    def test_filtering_sections_by_tool_id_regex(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_3"))
+        verify_custom_regex_filtered(index)
+
+    def test_filtering_root_by_type(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_4"))
+        index.raise_for_status()
+        index_as_list = index.json()
+        assert len(index_as_list) == 2
+        # Labels are filtered out...
+        assert model_classes(index_as_list) == ["Tool", "Tool"]
+        assert element_ids(index_as_list) == ["empty_list", "count_list"]
+
+    def test_custom_section_def(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_6"))
+        index.raise_for_status()
+        index_as_list = index.json()
+        assert len(index_as_list) == 1
+        assert model_classes(index_as_list) == ["ToolSection"]
+        section = index_as_list[0]
+        section_elems = section["elems"]
+        assert len(section_elems) == 4, model_classes(section_elems)
+        assert model_classes(section_elems) == ["ToolSectionLabel", "Tool", "ToolSectionLabel", "Tool"]
+        assert element_ids(section_elems) == ["the-start", "empty_list", "the-middle", "count_list"]
+
+    def test_section_embed(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_5"))
+        verify_custom_embed(index)
+
+    def test_section_embed_filtering(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_7"))
+        index.raise_for_status()
+        index_as_list = index.json()
+        assert len(index_as_list) == 1
+        assert model_classes(index_as_list) == ["ToolSection"]
+        section = index_as_list[0]
+        section_elems = section["elems"]
+        assert len(section_elems) == 5, model_classes(section_elems)
+        assert model_classes(section_elems) == ["Tool", "Tool", "Tool", "ToolSectionLabel", "Tool"]
+        elem_ids = element_ids(section_elems)
+        assert elem_ids[0:3] == ["multi_data_optional", "paths_as_file", "param_text_option"]
+        assert elem_ids[4] == "Filter1"
+
+    def test_section_reference_by_name(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_8"))
+        verify_custom_embed(index)
+
+    def test_section_alias(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_9"))
+        verify_custom_regex_filtered(index)
+
+    def test_expand_section_aliases(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True, view="custom_10"))
+        index.raise_for_status()
+        index_as_list = index.json()
+        assert len(index_as_list) == 2
+        assert model_classes(index_as_list) == ["ToolSection", "ToolSection"]
+
+
+class PanelViewsFromConfigIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    framework_tool_and_types = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["panel_views"] = [
+            {
+                "id": "my-custom",
+                "name": "My Custom",
+                "type": "generic",
+                "items": [
+                    {
+                        "type": "label",
+                        "text": "The Start",
+                    }, {
+                        "type": "tool",
+                        "id": "empty_list",
+                    }, {
+                        "type": "label",
+                        "text": "The Middle",
+                    }, {
+                        "type": "tool",
+                        "id": "count_list",
+                    }, {
+                        "type": "label",
+                        "text": "The End",
+                    }
+                ],
+            }
+        ]
+        config["default_panel_view"] = "my-custom"
+
+    def test_custom_label_order(self):
+        index = self.galaxy_interactor.get("tools", data=dict(in_panel=True))
+        verify_my_custom(index)
+
+
+def verify_my_custom(index):
+    index.raise_for_status()
+    index_as_list = index.json()
+    sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
+    assert len(sections) == 0
+
+    assert len(index_as_list) == 5
+    assert model_classes(index_as_list) == ["ToolSectionLabel", "Tool", "ToolSectionLabel", "Tool", "ToolSectionLabel"]
+
+
+def verify_custom_embed(index):
+    # custom_5 / custom_8
+    index.raise_for_status()
+    index_as_list = index.json()
+    assert len(index_as_list) == 1
+    assert model_classes(index_as_list) == ["ToolSection"]
+    section = index_as_list[0]
+    assert section["name"] == "My New Section"
+    assert section["id"] == "my-new-section"
+    section_elems = section["elems"]
+    assert len(section_elems) == 5, model_classes(section_elems)
+    assert model_classes(section_elems) == ["Tool", "Tool", "Tool", "Tool", "Tool"]
+    assert element_ids(section_elems) == ["multi_data_optional", "paths_as_file", "param_text_option", "column_param", "Filter1"]
+
+
+def verify_custom_regex_filtered(index):
+    # custom_3 / custom_9
+    index.raise_for_status()
+    index_as_list = index.json()
+    sections = [x for x in index_as_list if x["model_class"] == "ToolSection"]
+    assert len(sections) == 1
+    section = sections[0]
+    tools = section["elems"]
+    assert len(tools) == 2, len(tools)
+
+
+def element_ids(elements):
+    return [x["id"] for x in elements]
+
+
+def model_classes(elements):
+    return [x["model_class"] for x in elements]
diff --git a/test/unit/config/test_config_values.py b/test/unit/config/test_config_values.py
index b0cfa1856d9..a7e7a13317f 100644
--- a/test/unit/config/test_config_values.py
+++ b/test/unit/config/test_config_values.py
@@ -99,7 +99,6 @@ class ExpectedValues:
         self._expected_paths = {
             'admin_tool_recommendations_path': self._in_config_dir('tool_recommendations_overwrite.yml'),
             'auth_config_file': self._in_config_dir('auth_conf.xml'),
-            'beta_edam_toolbox_ontology_path': self._in_data_dir('EDAM.tsv'),
             'build_sites_config_file': self._in_sample_dir('build_sites.yml.sample'),
             'builds_file_path': self._in_root_dir('tool-data/shared/ucsc/builds.txt'),
             'cache_dir': self._in_data_dir('cache'),
@@ -113,6 +112,7 @@ class ExpectedValues:
             'datatypes_config_file': self._in_sample_dir('datatypes_conf.xml.sample'),
             'dependency_resolvers_config_file': self._in_config_dir('dependency_resolvers_conf.xml'),
             'dynamic_proxy_session_map': self._in_data_dir('session_map.sqlite'),
+            'edam_toolbox_ontology_path': self._in_data_dir('EDAM.tsv'),
             'error_report_file': self._in_config_dir('error_report.yml'),
             'file_path': self._in_data_dir('objects'),
             'file_sources_config_file': self._in_config_dir('file_sources_conf.yml'),
diff --git a/test/unit/tool_util/test_toolbox_view_parsing.py b/test/unit/tool_util/test_toolbox_view_parsing.py
new file mode 100644
index 00000000000..7c8ffd9468b
--- /dev/null
+++ b/test/unit/tool_util/test_toolbox_view_parsing.py
@@ -0,0 +1,101 @@
+import pydantic
+
+from galaxy.tool_util.toolbox.views.definitions import StaticToolBoxView, StaticToolBoxViewTypeEnum
+
+EXAMPLE_1 = {
+    "id": "rna",
+    "name": "RNA Seq",
+    "type": "activity",
+    "items": [
+        {
+            "type": "workflow",
+            "id": "12345abcd",
+        },
+        {
+            "type": "section",
+            "name": "My Cool Section",
+        },
+        {
+            "type": "label",
+            "text": "My Cool Label",
+            "id": "my-cool-label-id",
+        },
+        {
+            "type": "tool",
+            "id": "cat1",
+        },
+    ],
+}
+
+
+def test_root_parsing():
+    view = StaticToolBoxView.from_dict(EXAMPLE_1)
+    assert view.id == "rna"
+    assert view.name == "RNA Seq"
+    assert view.view_type == StaticToolBoxViewTypeEnum.activity
+    assert len(view.items) == 4
+    workflow_item = view.items[0]
+    assert workflow_item.content_type == "workflow"
+    assert workflow_item.id == "12345abcd"
+
+    section_item = view.items[1]
+    assert section_item.content_type == "section"
+    assert section_item.id is None
+    assert section_item.name == "My Cool Section"
+
+    outer_label_item = view.items[2]
+    assert outer_label_item.content_type == "label"
+    assert outer_label_item.id == "my-cool-label-id"
+    assert outer_label_item.text == "My Cool Label"
+
+    output_tool_item = view.items[3]
+    assert output_tool_item.content_type == "tool"
+    assert output_tool_item.id == "cat1"
+
+
+def test_section_parsing():
+    view = StaticToolBoxView.from_dict({
+        "id": "rna",
+        "name": "RNA Seq",
+        "type": "activity",
+        "items": [
+            {
+                "type": "section",
+                "name": "My Cool Section",
+                "items": [
+                    {
+                        "type": "workflow",
+                        "id": "123456",
+                    }
+                ],
+            },
+        ],
+    })
+    section_item = view.items[0]
+    assert len(section_item.items) == 1
+    assert section_item.items[0].content_type == "workflow"
+
+
+def test_no_nested_sections():
+    exception_raised = False
+    try:
+        StaticToolBoxView.from_dict({
+            "id": "rna",
+            "name": "RNA Seq",
+            "type": "activity",
+            "items": [
+                {
+                    "type": "section",
+                    "name": "My Cool Section",
+                    "items": [
+                        {
+                            "type": "section",
+                            "name": "My Cool Inner Section",
+                        },
+                    ],
+                },
+            ],
+        })
+    except pydantic.ValidationError:
+        exception_raised = True
+    assert exception_raised
diff --git a/test/unit/tool_util/test_toolbox_view_sources.py b/test/unit/tool_util/test_toolbox_view_sources.py
new file mode 100644
index 00000000000..9b608f7a7aa
--- /dev/null
+++ b/test/unit/tool_util/test_toolbox_view_sources.py
@@ -0,0 +1,45 @@
+import json
+import os
+import shutil
+import tempfile
+
+from galaxy.tool_util.toolbox.views.sources import StaticToolBoxViewSources
+from galaxy.util import safe_makedirs
+from .test_toolbox_view_parsing import EXAMPLE_1
+
+
+def test_from_dicts():
+    sources = StaticToolBoxViewSources(
+        view_dicts=[EXAMPLE_1]
+    )
+    defs = sources.get_definitions()
+    assert len(defs) == 1
+    assert defs[0].id == "rna"
+
+
+def test_from_directory():
+    target_dir = tempfile.mkdtemp()
+    dir_1 = os.path.join(target_dir, "dir1")
+    dir_2 = os.path.join(target_dir, "dir2")
+    safe_makedirs(dir_1)
+    safe_makedirs(dir_2)
+    with open(os.path.join(dir_1, "foo.json"), "w") as f:
+        json.dump(EXAMPLE_1, f)
+    with open(os.path.join(dir_2, "seqanalysis.yml"), "w") as f:
+        f.write("""
+name: Sequence Analysis
+type: generic
+items:
+- type: workflow
+  id: 12345abcd
+""")
+
+    config_param = f"{os.path.abspath(dir_1)},{os.path.abspath(dir_2)}"
+    try:
+        sources = StaticToolBoxViewSources(view_directories=config_param)
+        defs = sources.get_definitions()
+        assert len(defs) == 2
+        assert defs[0].id == "rna"
+        assert defs[1].id == "seqanalysis"
+    finally:
+        shutil.rmtree(target_dir)
diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py
index 62782beea85..f8b3b973a0a 100644
--- a/test/unit/unittest_utils/galaxy_mock.py
+++ b/test/unit/unittest_utils/galaxy_mock.py
@@ -197,6 +197,11 @@ class MockAppConfig(Bunch):
         self.delay_tool_initialization = True
         self.external_chown_script = None
 
+        self.default_panel_view = "default"
+        self.panel_views_dir = ''
+        self.panel_views = {}
+        self.edam_panel_views = ''
+
         self.config_file = None
 
     @property

From 5473bdd6b2c07e0e6d35f85c8e3411d300f9a63c Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 5 Aug 2021 19:52:02 -0400
Subject: [PATCH 340/452] Persist selected tool panel.

---
 client/package.json       |  3 ++-
 client/src/store/index.js |  8 +++++++-
 client/yarn.lock          | 13 +++++++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/client/package.json b/client/package.json
index 2c70f9711bf..c9e77080628 100644
--- a/client/package.json
+++ b/client/package.json
@@ -87,7 +87,8 @@
     "vueisotope": "^3.1.2",
     "vuex": "^3.4.0",
     "vuex-cache": "^3.2.0",
-    "vuex-persist": "^3.1.3"
+    "vuex-persist": "^3.1.3",
+    "vuex-persistedstate": "^4.0.0"
   },
   "scripts": {
     "watch": "gulp && yarn run save-build-hash && yarn run webpack-watch",
diff --git a/client/src/store/index.js b/client/src/store/index.js
index 4dcb0320444..30685be19cd 100644
--- a/client/src/store/index.js
+++ b/client/src/store/index.js
@@ -5,6 +5,8 @@
 import Vue from "vue";
 import Vuex from "vuex";
 import createCache from "vuex-cache";
+import createPersistedState from "vuex-persistedstate";
+
 import config from "config";
 
 import { gridSearchStore } from "./gridSearchStore";
@@ -30,9 +32,13 @@ import { syncVuextoGalaxy } from "./syncVuextoGalaxy";
 
 Vue.use(Vuex);
 
+const panelsState = createPersistedState({
+    paths: ["panels"],
+});
+
 export function createStore() {
     const storeConfig = {
-        plugins: [createCache()],
+        plugins: [createCache(), panelsState],
         modules: {
             user: userStore,
             config: configStore,
diff --git a/client/yarn.lock b/client/yarn.lock
index 829749ec72d..4e05539e974 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -13338,6 +13338,11 @@ shellwords@^0.1.1:
   resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"
   integrity sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==
 
+shvl@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/shvl/-/shvl-2.0.3.tgz#eb4bd37644f5684bba1fc52c3010c96fb5e6afd1"
+  integrity sha512-V7C6S9Hlol6SzOJPnQ7qzOVEWUQImt3BNmmzh40wObhla3XOYMe4gGiYzLrJd5TFa+cI2f9LKIRJTTKZSTbWgw==
+
 side-channel@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.3.tgz#cdc46b057550bbab63706210838df5d4c19519c3"
@@ -15304,6 +15309,14 @@ vuex-persist@^3.1.3:
     deepmerge "^4.2.2"
     flatted "^3.0.5"
 
+vuex-persistedstate@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/vuex-persistedstate/-/vuex-persistedstate-4.0.0.tgz#ed82f266ca98c869a2aad9cb9880c2f608c05f3a"
+  integrity sha512-jDs+awbV9YD2A2F6S5zgtYq1Bjd8v0YldOK6HPv1EJZzGMse0FtZTREfXvA7zlVfq9MpmSZJNmYQVylfpZ5znQ==
+  dependencies:
+    deepmerge "^4.2.2"
+    shvl "^2.0.3"
+
 vuex@^3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/vuex/-/vuex-3.4.0.tgz#20cc086062d750769fce1febb34e7fceeaebde45"

From 9bf22b4e693ab049b2bca8ebf2c92c9ba5431d06 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 6 Aug 2021 11:07:01 -0400
Subject: [PATCH 341/452] Don't serialize the toolbox in requests, pulling it
 from persistedstate and doesn't respect default panel view.

---
 lib/galaxy/webapps/base/controller.py             | 1 -
 lib/galaxy/webapps/galaxy/controllers/workflow.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py
index e7780e5d424..fb1f8c82f17 100644
--- a/lib/galaxy/webapps/base/controller.py
+++ b/lib/galaxy/webapps/base/controller.py
@@ -307,7 +307,6 @@ class JSAppLauncher(BaseUIController):
         config = {
             'active_view': 'analysis',
             'enable_webhooks': True if trans.app.webhooks_registry.webhooks else False,
-            'toolbox': trans.app.toolbox.to_dict(trans),
             'message_box_visible': trans.app.config.message_box_visible,
             'show_inactivity_warning': trans.app.config.user_activation_on and trans.user and not trans.user.active,
             'tool_shed_urls': list(trans.app.tool_shed_registry.tool_sheds.values()) if trans.app.tool_shed_registry else [],
diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py
index 80870f8d1c0..c9f456047f6 100644
--- a/lib/galaxy/webapps/galaxy/controllers/workflow.py
+++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py
@@ -724,7 +724,6 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
             'tags': item_tag_names,
             'initialVersion': version,
             'annotation': self.get_item_annotation_str(trans.sa_session, trans.user, stored),
-            'toolbox': trans.app.toolbox.to_dict(trans),
             'moduleSections': module_sections,
             'dataManagers': data_managers,
             'workflows': workflows

From 0f2235e9c306ca76895f242640ceee9b71847fac Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 10 Aug 2021 16:20:46 -0400
Subject: [PATCH 342/452] Fix linting.

---
 client/src/components/Panels/Buttons/PanelViewButton.vue | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/components/Panels/Buttons/PanelViewButton.vue b/client/src/components/Panels/Buttons/PanelViewButton.vue
index 84fc12ddc2e..4bb5e22a01a 100644
--- a/client/src/components/Panels/Buttons/PanelViewButton.vue
+++ b/client/src/components/Panels/Buttons/PanelViewButton.vue
@@ -31,7 +31,6 @@
         </b-dropdown-group>
         <b-dropdown-divider v-if="groupedPanelViews.length > 0"></b-dropdown-divider>
         <PanelViewMenuItem
-            v-if="ungroupedPanelViews.length > 0"
             v-for="(panelView, key) in ungroupedPanelViews"
             :key="key"
             :current-panel-view="currentPanelView"

From 7846d47d6d14707cb43cdbc4a1fcf20ef6009b7f Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Wed, 11 Aug 2021 18:19:24 +0200
Subject: [PATCH 343/452] Style fixes

---
 lib/galaxy/managers/group_roles.py           |  4 ++--
 lib/galaxy/schema/schema.py                  |  3 ++-
 lib/galaxy/webapps/galaxy/api/group_roles.py | 14 +++++++-------
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index 1991863f072..0fd5ba8ea3b 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -107,9 +107,9 @@ class GroupRolesManager:
     def _serialize_group_role(self, encoded_group_id: EncodedDatabaseIdField, role: model.Role):
         encoded_role_id = self._app.security.encode_id(role.id)
         try:
-            url  = url_for('role', group_id=encoded_group_id, id=encoded_role_id)
+            url = url_for('role', group_id=encoded_group_id, id=encoded_role_id)
         except AttributeError:
-            url  = "*deprecated attribute not filled in by FastAPI server*"
+            url = "*deprecated attribute not filled in by FastAPI server*"
         return {
             "id": encoded_role_id,
             "name": role.name,
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index c1b2edfa187..cc6df51681d 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -1909,7 +1909,8 @@ class GroupRoleModel(BaseModel):
     id: EncodedDatabaseIdField = RoleIdField
     name: str = RoleNameField
     url: str = Field(title="URL", description="URL for the group role")
-    
+
+
 class GroupRoleListModel(BaseModel):
     __root__: List[GroupRoleModel]
 
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index d7e0ed51e2b..8494fc15a7a 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -15,9 +15,9 @@ from galaxy.web import (
     require_admin,
 )
 from . import (
-    BaseGalaxyAPIController, 
-    depends, 
-    Router, 
+    BaseGalaxyAPIController,
+    depends,
+    Router,
     DependsOnTrans
 )
 
@@ -46,13 +46,13 @@ class FastAPIGroupRoles:
     @router.get('/api/groups/{group_id}/roles',
                 require_admin=True,
                 summary='Displays a collection (list) of groups.')
-    def index(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
+    def indexAfe(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
         return self.manager.index(trans, group_id)
 
     @router.get('/api/groups/{group_id}/roles/{role_id}',
                 require_admin=True,
                 summary='Displays information about a group role.')
-    def show(self,  trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
+    def show(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
         return self.manager.show(trans, role_id, group_id)
 
     @router.put('/api/groups/{group_id}/roles/{role_id}',
@@ -62,8 +62,8 @@ class FastAPIGroupRoles:
         return self.manager.update(trans, role_id, group_id)
 
     @router.delete('/api/groups/{group_id}/roles/{role_id}',
-                    require_admin=True,
-                    summary='Removes a role from a group')
+                   require_admin=True,
+                   summary='Removes a role from a group')
     def delete(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam):
         return self.manager.delete(trans, role_id, group_id)
 

From 384697d5bae786eb3987dffcbd7d9dc55e96f25d Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Wed, 11 Aug 2021 10:16:23 -0700
Subject: [PATCH 344/452] added visibility flag test

---
 lib/galaxy_test/api/test_history_contents.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 831c2122585..3919a6906d4 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -225,10 +225,28 @@ class HistoryContentsApiTestCase(ApiTestCase):
     def test_update_batch(self):
         hda1 = self._wait_for_new_hda()
         assert str(hda1["deleted"]).lower() == "false"
+        assert str(hda1["visible"]).lower() == "true"
+
+        # update deleted flag => true
         payload = dict(items=[{"history_content_type": "dataset", "id": hda1["id"]}], deleted=True)
         update_response = self._raw_update_batch(payload)
         objects = update_response.json()
-        assert objects[0]["deleted"]
+        assert objects[0]["deleted"] is True
+        assert objects[0]["visible"] is True
+
+        # update visibility flag => false
+        payload = dict(items=[{"history_content_type": "dataset", "id": hda1["id"]}], visible=False)
+        update_response = self._raw_update_batch(payload)
+        objects = update_response.json()
+        assert objects[0]["deleted"] is True
+        assert objects[0]["visible"] is False
+
+        # update both flags
+        payload = dict(items=[{"history_content_type": "dataset", "id": hda1["id"]}], deleted=False, visible=True)
+        update_response = self._raw_update_batch(payload)
+        objects = update_response.json()
+        assert objects[0]["deleted"] is False
+        assert objects[0]["visible"] is True
 
     def test_update_type_failures(self):
         hda1 = self._wait_for_new_hda()

From 501efc47e0eb7f49c027231bc9a5d568b136ef51 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Wed, 11 Aug 2021 19:20:49 +0200
Subject: [PATCH 345/452] Fix imports

---
 lib/galaxy/webapps/galaxy/api/group_roles.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 8494fc15a7a..7a7f8a3e445 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -1,11 +1,10 @@
 """
 API operations on Group objects.
 """
-
-from lib.galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 import logging
 
 from fastapi import Path
+from lib.galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.managers.group_roles import GroupRolesManager
@@ -17,8 +16,8 @@ from galaxy.web import (
 from . import (
     BaseGalaxyAPIController,
     depends,
-    Router,
-    DependsOnTrans
+    DependsOnTrans,
+    Router
 )
 
 

From 8833d7763d03d49be034bf86481671118e741646 Mon Sep 17 00:00:00 2001
From: Peter van Heusden <pvh@sanbi.ac.za>
Date: Wed, 11 Aug 2021 19:27:58 +0200
Subject: [PATCH 346/452] Identify tool by name, owner, toolshed

---
 .../Toolshed/InstalledList/Index.test.js      |  2 +
 .../Toolshed/InstalledList/Index.vue          | 44 +++++++++++++------
 client/src/components/Toolshed/services.js    |  3 +-
 3 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/client/src/components/Toolshed/InstalledList/Index.test.js b/client/src/components/Toolshed/InstalledList/Index.test.js
index 6342ea7e7c0..eff7c3e8741 100644
--- a/client/src/components/Toolshed/InstalledList/Index.test.js
+++ b/client/src/components/Toolshed/InstalledList/Index.test.js
@@ -20,6 +20,7 @@ Services.mockImplementation(() => {
                 {
                     name: "name_0",
                     description: "description_0",
+                    tool_shed: "toolshed_1",
                     tool_shed_status: {
                         latest_installable_revision: false,
                     },
@@ -27,6 +28,7 @@ Services.mockImplementation(() => {
                 {
                     name: "name_1",
                     description: "description_1",
+                    tool_shed: "toolshed_1",
                     tool_shed_status: {
                         latest_installable_revision: true,
                     },
diff --git a/client/src/components/Toolshed/InstalledList/Index.vue b/client/src/components/Toolshed/InstalledList/Index.vue
index 5f61b916b61..583142ddbf9 100644
--- a/client/src/components/Toolshed/InstalledList/Index.vue
+++ b/client/src/components/Toolshed/InstalledList/Index.vue
@@ -60,6 +60,7 @@ import { Services } from "../services";
 import LoadingSpan from "components/LoadingSpan";
 import Monitor from "./Monitor";
 import RepositoryDetails from "./Details";
+import {getGalaxyInstance} from "../../../app";
 
 Vue.use(BootstrapVue);
 
@@ -71,21 +72,9 @@ export default {
     },
     props: ["filter"],
     data() {
+
         return {
             error: null,
-            fields: [
-                {
-                    key: "name",
-                    sortable: true,
-                    sortByFormatted: (value, key, item) => {
-                        return `${this.isLatest(item)}_${value}`;
-                    },
-                },
-                {
-                    key: "owner",
-                    sortable: true,
-                },
-            ],
             loading: true,
             message: null,
             messageVariant: null,
@@ -105,6 +94,35 @@ export default {
         showMessage() {
             return !!this.message;
         },
+        numToolsheds() {
+            const toolsheds = new Set();
+            this.repositories.forEach((x) => {
+              toolsheds.add(x.tool_shed)
+            });
+            return toolsheds.size
+        },
+        fields() {
+            const fields = [
+                  {
+                    key: "name",
+                    sortable: true,
+                    sortByFormatted: (value, key, item) => {
+                      return `${this.isLatest(item)}_${value}`;
+                    },
+                  },
+                  {
+                    key: "owner",
+                    sortable: true,
+                  },
+                ];
+            if (this.numToolsheds > 1) {
+                fields.push({
+                  key: "tool_shed",
+                  sortable: true,
+                });
+            }
+            return fields;
+        }
     },
     created() {
         this.root = getAppRoot();
diff --git a/client/src/components/Toolshed/services.js b/client/src/components/Toolshed/services.js
index 2fb162ced2a..f67d9805ce0 100644
--- a/client/src/components/Toolshed/services.js
+++ b/client/src/components/Toolshed/services.js
@@ -134,6 +134,7 @@ export class Services {
                 return (
                     getSortValue(a.name, b.name) ||
                     getSortValue(a.owner, b.owner) ||
+                    getSortValue(a.tool_shed, b.tool_shed) ||
                     getSortValue(parseInt(b.ctx_rev), parseInt(a.ctx_rev))
                 );
             });
@@ -141,7 +142,7 @@ export class Services {
         const hash = {};
         const repositories = [];
         incoming.forEach((x) => {
-            const hashCode = `${x.name}_${x.owner}`;
+            const hashCode = `${x.name}_${x.owner}_${x.tool_shed}`;
             if (!filter || filter(x)) {
                 if (!hash[hashCode]) {
                     hash[hashCode] = true;

From 91c5bbbdb983cf2615e900fc3b3dbc2edce63db0 Mon Sep 17 00:00:00 2001
From: Peter van Heusden <pvh@sanbi.ac.za>
Date: Wed, 11 Aug 2021 19:41:51 +0200
Subject: [PATCH 347/452] Add explicit test for Tool Shed header

---
 client/src/components/Toolshed/InstalledList/Index.test.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Toolshed/InstalledList/Index.test.js b/client/src/components/Toolshed/InstalledList/Index.test.js
index eff7c3e8741..38c5016b382 100644
--- a/client/src/components/Toolshed/InstalledList/Index.test.js
+++ b/client/src/components/Toolshed/InstalledList/Index.test.js
@@ -28,7 +28,7 @@ Services.mockImplementation(() => {
                 {
                     name: "name_1",
                     description: "description_1",
-                    tool_shed: "toolshed_1",
+                    tool_shed: "toolshed_2",
                     tool_shed_status: {
                         latest_installable_revision: true,
                     },
@@ -60,5 +60,6 @@ describe("InstalledList", () => {
         expect(links.length).toBe(3);
         const badge = links.at(1).find(".badge");
         expect(badge.text()).toBe("Newer version available!");
+        expect(wrapper.find('th[role="columnheader"][aria-colindex="3"] > div').text()).toBe("Tool Shed");
     });
 });

From 39787cea89bfb03be6e59165c254d155ef68033a Mon Sep 17 00:00:00 2001
From: Peter van Heusden <pvh@sanbi.ac.za>
Date: Wed, 11 Aug 2021 20:44:28 +0200
Subject: [PATCH 348/452] Fix eslint / vue linting errors

---
 .../Toolshed/InstalledList/Details.vue        |  7 +++++-
 .../Toolshed/InstalledList/Index.vue          |  8 +++++--
 .../Toolshed/RepositoryDetails/Index.vue      | 11 ++++++++-
 .../RepositoryDetails/InstallationButton.vue  | 11 ++++++++-
 .../InstallationSettings.vue                  | 23 ++++++++++++++++++-
 .../RepositoryDetails/RepositoryTools.vue     |  7 +++++-
 .../Toolshed/SearchList/Categories.vue        | 11 ++++++++-
 .../components/Toolshed/SearchList/Index.vue  | 11 ++++++++-
 .../Toolshed/SearchList/Repositories.test.js  |  2 +-
 .../Toolshed/SearchList/Repositories.vue      | 15 +++++++++++-
 .../SearchList/ServerSelection.test.js        |  4 ++--
 .../Toolshed/SearchList/ServerSelection.vue   | 19 ++++++++++++++-
 client/src/components/Toolshed/services.js    |  4 ++--
 .../src/components/Toolshed/services.test.js  |  2 +-
 14 files changed, 118 insertions(+), 17 deletions(-)

diff --git a/client/src/components/Toolshed/InstalledList/Details.vue b/client/src/components/Toolshed/InstalledList/Details.vue
index 10b167cacda..933f749d0b6 100644
--- a/client/src/components/Toolshed/InstalledList/Details.vue
+++ b/client/src/components/Toolshed/InstalledList/Details.vue
@@ -20,7 +20,12 @@ import RepositoryDetails from "../RepositoryDetails/Index.vue";
 import LoadingSpan from "components/LoadingSpan";
 
 export default {
-    props: ["repo"],
+    props: {
+      repo: {
+        type: Object,
+        required: true
+      }
+    },
     components: {
         LoadingSpan,
         RepositoryDetails,
diff --git a/client/src/components/Toolshed/InstalledList/Index.vue b/client/src/components/Toolshed/InstalledList/Index.vue
index 583142ddbf9..2fcc8916c17 100644
--- a/client/src/components/Toolshed/InstalledList/Index.vue
+++ b/client/src/components/Toolshed/InstalledList/Index.vue
@@ -60,7 +60,6 @@ import { Services } from "../services";
 import LoadingSpan from "components/LoadingSpan";
 import Monitor from "./Monitor";
 import RepositoryDetails from "./Details";
-import {getGalaxyInstance} from "../../../app";
 
 Vue.use(BootstrapVue);
 
@@ -70,7 +69,12 @@ export default {
         Monitor,
         RepositoryDetails,
     },
-    props: ["filter"],
+    props: {
+      filter: {
+        type: String,
+        required: true
+      }
+    },
     data() {
 
         return {
diff --git a/client/src/components/Toolshed/RepositoryDetails/Index.vue b/client/src/components/Toolshed/RepositoryDetails/Index.vue
index 17d6d998d76..4fa3b84d898 100644
--- a/client/src/components/Toolshed/RepositoryDetails/Index.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/Index.vue
@@ -67,7 +67,16 @@ export default {
         InstallationButton,
         RepositoryTools,
     },
-    props: ["repo", "toolshedUrl"],
+    props: {
+      repo: {
+        type: Object,
+        required: true
+      },
+      toolshedUrl: {
+        type: String,
+        required: true
+      },
+    },
     data() {
         return {
             repoChecked: "fa fa-check text-success",
diff --git a/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue b/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
index 6a0b84d1dd0..0f09f22fc87 100644
--- a/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
@@ -19,7 +19,16 @@ import BootstrapVue from "bootstrap-vue";
 Vue.use(BootstrapVue);
 
 export default {
-    props: ["installed", "status"],
+    props: {
+      installed: {
+        type: Boolean,
+        required: false
+      },
+      status: {
+        type: String,
+        required: true
+      }
+    },
     data() {
         return {
             buttonClass: "btn-sm text-nowrap",
diff --git a/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue b/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue
index a4c6cf163dd..28fe26845e1 100644
--- a/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue
@@ -55,7 +55,28 @@ import { getGalaxyInstance } from "app";
 Vue.use(BootstrapVue);
 
 export default {
-    props: ["repo", "changesetRevision", "requiresPanel", "toolshedUrl", "modalStatic"],
+    props: {
+      repo: {
+        type: Object,
+        required: true,
+      },
+      changesetRevision: {
+        type: String,
+        required: true,
+      },
+      requiresPanel: {
+        type: Boolean,
+        required: true,
+      },
+      toolshedUrl: {
+        type: String,
+        required: true,
+      },
+      modalStatic: {
+        type: Boolean,
+        required: true,
+      },
+    },
     data() {
         return {
             modalShow: true,
diff --git a/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue b/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue
index c7b2b6e5565..32b6e313de0 100644
--- a/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue
@@ -33,7 +33,12 @@
 </template>
 <script>
 export default {
-    props: ["tools"],
+    props: {
+      tools: {
+        type: Array,
+        required: true
+      },
+    },
     data() {
         return {
             clsExpanded: "bg-transparent border-bottom",
diff --git a/client/src/components/Toolshed/SearchList/Categories.vue b/client/src/components/Toolshed/SearchList/Categories.vue
index 88e999cbf19..8ee6ab15bf9 100644
--- a/client/src/components/Toolshed/SearchList/Categories.vue
+++ b/client/src/components/Toolshed/SearchList/Categories.vue
@@ -24,7 +24,16 @@ import LoadingSpan from "components/LoadingSpan";
 Vue.use(BootstrapVue);
 
 export default {
-    props: ["toolshedUrl", "loading"],
+    props: {
+      toolshedUrl: {
+        type: String,
+        required: true
+      },
+      loading: {
+        type: Boolean,
+        required: true
+      },
+    },
     components: { LoadingSpan },
     data() {
         return {
diff --git a/client/src/components/Toolshed/SearchList/Index.vue b/client/src/components/Toolshed/SearchList/Index.vue
index 63dba70f2c1..742d16acd4f 100644
--- a/client/src/components/Toolshed/SearchList/Index.vue
+++ b/client/src/components/Toolshed/SearchList/Index.vue
@@ -34,7 +34,16 @@ import Categories from "./Categories.vue";
 import Repositories from "./Repositories.vue";
 import ServerSelection from "./ServerSelection.vue";
 export default {
-    props: ["query", "scrolled"],
+    props: {
+      query: {
+        type: String,
+        required: true
+      },
+      scrolled: {
+        type: Boolean,
+        required: true
+      },
+    },
     components: {
         Categories,
         Repositories,
diff --git a/client/src/components/Toolshed/SearchList/Repositories.test.js b/client/src/components/Toolshed/SearchList/Repositories.test.js
index ea3c1fb7d08..90e660a3216 100644
--- a/client/src/components/Toolshed/SearchList/Repositories.test.js
+++ b/client/src/components/Toolshed/SearchList/Repositories.test.js
@@ -33,7 +33,7 @@ describe("Repositories", () => {
     it("test repository details loading", async () => {
         const wrapper = mount(Repositories, {
             propsData: {
-                query: true,
+                query: "toolname",
                 scrolled: false,
                 toolshedUrl: "toolshedUrl",
             },
diff --git a/client/src/components/Toolshed/SearchList/Repositories.vue b/client/src/components/Toolshed/SearchList/Repositories.vue
index 4f9bdfa5140..f7ad240e86e 100644
--- a/client/src/components/Toolshed/SearchList/Repositories.vue
+++ b/client/src/components/Toolshed/SearchList/Repositories.vue
@@ -32,7 +32,20 @@ export default {
         LoadingSpan,
         RepositoryDetails,
     },
-    props: ["query", "scrolled", "toolshedUrl"],
+    props: {
+      query: {
+        type: String,
+        required: true
+      },
+      scrolled: {
+        type: Boolean,
+        required: true,
+      },
+      toolshedUrl: {
+        type: String,
+        required: true
+      },
+    },
     data() {
         return {
             repositories: [],
diff --git a/client/src/components/Toolshed/SearchList/ServerSelection.test.js b/client/src/components/Toolshed/SearchList/ServerSelection.test.js
index d3e138113fc..80524750123 100644
--- a/client/src/components/Toolshed/SearchList/ServerSelection.test.js
+++ b/client/src/components/Toolshed/SearchList/ServerSelection.test.js
@@ -11,11 +11,11 @@ describe("ServerSelection", () => {
                 toolshedUrl: "url_0",
                 toolshedUrls: ["url_0", "url_1"],
                 loading: false,
-                total: "total",
+                total: 10,
             },
             localVue,
         });
-        expect(wrapper.find(".description").text()).toBe("total repositories available at");
+        expect(wrapper.find(".description").text()).toBe("10 repositories available at");
         const $options = wrapper.findAll("a");
         expect($options.at(0).text()).toBe("url_0");
         expect($options.at(1).text()).toBe("url_0");
diff --git a/client/src/components/Toolshed/SearchList/ServerSelection.vue b/client/src/components/Toolshed/SearchList/ServerSelection.vue
index 8a7701feb3f..1a394cfdcce 100644
--- a/client/src/components/Toolshed/SearchList/ServerSelection.vue
+++ b/client/src/components/Toolshed/SearchList/ServerSelection.vue
@@ -38,7 +38,24 @@ import BootstrapVue from "bootstrap-vue";
 Vue.use(BootstrapVue);
 
 export default {
-    props: ["toolshedUrl", "toolshedUrls", "loading", "total"],
+    props: {
+      toolshedUrl: {
+        type: String,
+        required: true,
+      },
+      toolshedUrls: {
+        type: Array,
+        required: true,
+      },
+      loading: {
+        type: Boolean,
+        required: true,
+      },
+      total: {
+        type: Number,
+        required: true
+      },
+    },
     computed: {
         showDropdown() {
             return this.toolshedUrls.length > 1;
diff --git a/client/src/components/Toolshed/services.js b/client/src/components/Toolshed/services.js
index f67d9805ce0..eb00273238b 100644
--- a/client/src/components/Toolshed/services.js
+++ b/client/src/components/Toolshed/services.js
@@ -78,7 +78,7 @@ export class Services {
         const url = `${getAppRoot()}api/tool_shed_repositories/?uninstalled=False`;
         try {
             const response = await axios.get(url);
-            const repositories = this._groupByNameOwner(response.data, options.filter, options.selectLatest);
+            const repositories = this._groupByNameOwnerToolshed(response.data, options.filter, options.selectLatest);
             this._fixToolshedUrls(repositories, Galaxy.config.tool_shed_urls);
             return repositories;
         } catch (e) {
@@ -125,7 +125,7 @@ export class Services {
             rethrowSimple(e);
         }
     }
-    _groupByNameOwner(incoming, filter, selectLatest) {
+    _groupByNameOwnerToolshed(incoming, filter, selectLatest) {
         if (selectLatest) {
             const getSortValue = (x, y) => {
                 return x == y ? 0 : x < y ? -1 : 1;
diff --git a/client/src/components/Toolshed/services.test.js b/client/src/components/Toolshed/services.test.js
index ed3d573bffd..e8602826335 100644
--- a/client/src/components/Toolshed/services.test.js
+++ b/client/src/components/Toolshed/services.test.js
@@ -40,7 +40,7 @@ describe("Toolshed service helpers", () => {
     it("test fix toolshed helper", () => {
         const services = new Services();
         const filter = (x) => x.status !== "Installed";
-        const grouped = services._groupByNameOwner(incoming, filter, true);
+        const grouped = services._groupByNameOwnerToolshed(incoming, filter, true);
         services._fixToolshedUrls(grouped, urls);
         expect(grouped.length).toBe(2);
         expect(grouped[0].name).toBe("name_0");

From f520c6a70c3f92e746c8fecd5c698bc5a94f7171 Mon Sep 17 00:00:00 2001
From: Peter van Heusden <pvh@sanbi.ac.za>
Date: Wed, 11 Aug 2021 21:11:07 +0200
Subject: [PATCH 349/452] Reformat code with prettier

---
 .../Toolshed/InstalledList/Details.vue        | 10 ++---
 .../Toolshed/InstalledList/Index.vue          | 33 ++++++++-------
 .../Toolshed/InstalledList/Monitor.vue        |  2 +-
 .../Toolshed/RepositoryDetails/Index.vue      | 16 ++++----
 .../RepositoryDetails/InstallationButton.vue  | 16 ++++----
 .../InstallationSettings.vue                  | 40 +++++++++----------
 .../RepositoryDetails/RepositoryTools.vue     |  8 ++--
 .../Toolshed/SearchList/Categories.vue        | 16 ++++----
 .../components/Toolshed/SearchList/Index.vue  | 16 ++++----
 .../Toolshed/SearchList/Repositories.vue      | 24 +++++------
 .../Toolshed/SearchList/ServerSelection.vue   | 32 +++++++--------
 client/src/components/Toolshed/services.js    |  2 +-
 12 files changed, 107 insertions(+), 108 deletions(-)

diff --git a/client/src/components/Toolshed/InstalledList/Details.vue b/client/src/components/Toolshed/InstalledList/Details.vue
index 933f749d0b6..18b75966f22 100644
--- a/client/src/components/Toolshed/InstalledList/Details.vue
+++ b/client/src/components/Toolshed/InstalledList/Details.vue
@@ -21,10 +21,10 @@ import LoadingSpan from "components/LoadingSpan";
 
 export default {
     props: {
-      repo: {
-        type: Object,
-        required: true
-      }
+        repo: {
+            type: Object,
+            required: true,
+        },
     },
     components: {
         LoadingSpan,
@@ -39,7 +39,7 @@ export default {
     },
     created() {
         this.root = getAppRoot();
-        this.services = new Services({ root: this.root });
+        this.services = new Services();
         this.load();
     },
     methods: {
diff --git a/client/src/components/Toolshed/InstalledList/Index.vue b/client/src/components/Toolshed/InstalledList/Index.vue
index 2fcc8916c17..39e7d767655 100644
--- a/client/src/components/Toolshed/InstalledList/Index.vue
+++ b/client/src/components/Toolshed/InstalledList/Index.vue
@@ -70,13 +70,12 @@ export default {
         RepositoryDetails,
     },
     props: {
-      filter: {
-        type: String,
-        required: true
-      }
+        filter: {
+            type: String,
+            required: true,
+        },
     },
     data() {
-
         return {
             error: null,
             loading: true,
@@ -101,36 +100,36 @@ export default {
         numToolsheds() {
             const toolsheds = new Set();
             this.repositories.forEach((x) => {
-              toolsheds.add(x.tool_shed)
+                toolsheds.add(x.tool_shed);
             });
-            return toolsheds.size
+            return toolsheds.size;
         },
         fields() {
             const fields = [
-                  {
+                {
                     key: "name",
                     sortable: true,
                     sortByFormatted: (value, key, item) => {
-                      return `${this.isLatest(item)}_${value}`;
+                        return `${this.isLatest(item)}_${value}`;
                     },
-                  },
-                  {
+                },
+                {
                     key: "owner",
                     sortable: true,
-                  },
-                ];
+                },
+            ];
             if (this.numToolsheds > 1) {
                 fields.push({
-                  key: "tool_shed",
-                  sortable: true,
+                    key: "tool_shed",
+                    sortable: true,
                 });
             }
             return fields;
-        }
+        },
     },
     created() {
         this.root = getAppRoot();
-        this.services = new Services({ root: this.root });
+        this.services = new Services();
         this.load();
     },
     methods: {
diff --git a/client/src/components/Toolshed/InstalledList/Monitor.vue b/client/src/components/Toolshed/InstalledList/Monitor.vue
index 7c7a97349ce..0f22d2775ef 100644
--- a/client/src/components/Toolshed/InstalledList/Monitor.vue
+++ b/client/src/components/Toolshed/InstalledList/Monitor.vue
@@ -59,7 +59,7 @@ export default {
     },
     created() {
         this.root = getAppRoot();
-        this.services = new Services({ root: this.root });
+        this.services = new Services();
         this.load();
     },
     destroyed() {
diff --git a/client/src/components/Toolshed/RepositoryDetails/Index.vue b/client/src/components/Toolshed/RepositoryDetails/Index.vue
index 4fa3b84d898..503dbb0d08f 100644
--- a/client/src/components/Toolshed/RepositoryDetails/Index.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/Index.vue
@@ -68,14 +68,14 @@ export default {
         RepositoryTools,
     },
     props: {
-      repo: {
-        type: Object,
-        required: true
-      },
-      toolshedUrl: {
-        type: String,
-        required: true
-      },
+        repo: {
+            type: Object,
+            required: true,
+        },
+        toolshedUrl: {
+            type: String,
+            required: true,
+        },
     },
     data() {
         return {
diff --git a/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue b/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
index 0f09f22fc87..7a4f7b134b7 100644
--- a/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
@@ -20,14 +20,14 @@ Vue.use(BootstrapVue);
 
 export default {
     props: {
-      installed: {
-        type: Boolean,
-        required: false
-      },
-      status: {
-        type: String,
-        required: true
-      }
+        installed: {
+            type: Boolean,
+            required: false,
+        },
+        status: {
+            type: String,
+            required: true,
+        },
     },
     data() {
         return {
diff --git a/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue b/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue
index 28fe26845e1..9fae81a8d6b 100644
--- a/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/InstallationSettings.vue
@@ -56,26 +56,26 @@ Vue.use(BootstrapVue);
 
 export default {
     props: {
-      repo: {
-        type: Object,
-        required: true,
-      },
-      changesetRevision: {
-        type: String,
-        required: true,
-      },
-      requiresPanel: {
-        type: Boolean,
-        required: true,
-      },
-      toolshedUrl: {
-        type: String,
-        required: true,
-      },
-      modalStatic: {
-        type: Boolean,
-        required: true,
-      },
+        repo: {
+            type: Object,
+            required: true,
+        },
+        changesetRevision: {
+            type: String,
+            required: true,
+        },
+        requiresPanel: {
+            type: Boolean,
+            required: true,
+        },
+        toolshedUrl: {
+            type: String,
+            required: true,
+        },
+        modalStatic: {
+            type: Boolean,
+            required: true,
+        },
     },
     data() {
         return {
diff --git a/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue b/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue
index 32b6e313de0..1963f13da11 100644
--- a/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/RepositoryTools.vue
@@ -34,10 +34,10 @@
 <script>
 export default {
     props: {
-      tools: {
-        type: Array,
-        required: true
-      },
+        tools: {
+            type: Array,
+            required: true,
+        },
     },
     data() {
         return {
diff --git a/client/src/components/Toolshed/SearchList/Categories.vue b/client/src/components/Toolshed/SearchList/Categories.vue
index 8ee6ab15bf9..dba32473dc3 100644
--- a/client/src/components/Toolshed/SearchList/Categories.vue
+++ b/client/src/components/Toolshed/SearchList/Categories.vue
@@ -25,14 +25,14 @@ Vue.use(BootstrapVue);
 
 export default {
     props: {
-      toolshedUrl: {
-        type: String,
-        required: true
-      },
-      loading: {
-        type: Boolean,
-        required: true
-      },
+        toolshedUrl: {
+            type: String,
+            required: true,
+        },
+        loading: {
+            type: Boolean,
+            required: true,
+        },
     },
     components: { LoadingSpan },
     data() {
diff --git a/client/src/components/Toolshed/SearchList/Index.vue b/client/src/components/Toolshed/SearchList/Index.vue
index 742d16acd4f..54c58f2a0c0 100644
--- a/client/src/components/Toolshed/SearchList/Index.vue
+++ b/client/src/components/Toolshed/SearchList/Index.vue
@@ -35,14 +35,14 @@ import Repositories from "./Repositories.vue";
 import ServerSelection from "./ServerSelection.vue";
 export default {
     props: {
-      query: {
-        type: String,
-        required: true
-      },
-      scrolled: {
-        type: Boolean,
-        required: true
-      },
+        query: {
+            type: String,
+            required: true,
+        },
+        scrolled: {
+            type: Boolean,
+            required: true,
+        },
     },
     components: {
         Categories,
diff --git a/client/src/components/Toolshed/SearchList/Repositories.vue b/client/src/components/Toolshed/SearchList/Repositories.vue
index f7ad240e86e..3e8c73f721b 100644
--- a/client/src/components/Toolshed/SearchList/Repositories.vue
+++ b/client/src/components/Toolshed/SearchList/Repositories.vue
@@ -33,18 +33,18 @@ export default {
         RepositoryDetails,
     },
     props: {
-      query: {
-        type: String,
-        required: true
-      },
-      scrolled: {
-        type: Boolean,
-        required: true,
-      },
-      toolshedUrl: {
-        type: String,
-        required: true
-      },
+        query: {
+            type: String,
+            required: true,
+        },
+        scrolled: {
+            type: Boolean,
+            required: true,
+        },
+        toolshedUrl: {
+            type: String,
+            required: true,
+        },
     },
     data() {
         return {
diff --git a/client/src/components/Toolshed/SearchList/ServerSelection.vue b/client/src/components/Toolshed/SearchList/ServerSelection.vue
index 1a394cfdcce..27bb3b51aa1 100644
--- a/client/src/components/Toolshed/SearchList/ServerSelection.vue
+++ b/client/src/components/Toolshed/SearchList/ServerSelection.vue
@@ -39,22 +39,22 @@ Vue.use(BootstrapVue);
 
 export default {
     props: {
-      toolshedUrl: {
-        type: String,
-        required: true,
-      },
-      toolshedUrls: {
-        type: Array,
-        required: true,
-      },
-      loading: {
-        type: Boolean,
-        required: true,
-      },
-      total: {
-        type: Number,
-        required: true
-      },
+        toolshedUrl: {
+            type: String,
+            required: true,
+        },
+        toolshedUrls: {
+            type: Array,
+            required: true,
+        },
+        loading: {
+            type: Boolean,
+            required: true,
+        },
+        total: {
+            type: Number,
+            required: true,
+        },
     },
     computed: {
         showDropdown() {
diff --git a/client/src/components/Toolshed/services.js b/client/src/components/Toolshed/services.js
index eb00273238b..9fe192a15bf 100644
--- a/client/src/components/Toolshed/services.js
+++ b/client/src/components/Toolshed/services.js
@@ -128,7 +128,7 @@ export class Services {
     _groupByNameOwnerToolshed(incoming, filter, selectLatest) {
         if (selectLatest) {
             const getSortValue = (x, y) => {
-                return x == y ? 0 : x < y ? -1 : 1;
+                return x === y ? 0 : x < y ? -1 : 1;
             };
             incoming = incoming.sort((a, b) => {
                 return (

From 574d520540f0e5e6a9079a6814684b04a2194b67 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 14:27:37 +0200
Subject: [PATCH 350/452] Fix typo
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: David López <46503462+davelopez@users.noreply.github.com>
---
 lib/galaxy/webapps/galaxy/api/group_roles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 7a7f8a3e445..9af8282aebf 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -45,7 +45,7 @@ class FastAPIGroupRoles:
     @router.get('/api/groups/{group_id}/roles',
                 require_admin=True,
                 summary='Displays a collection (list) of groups.')
-    def indexAfe(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
+    def index(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
         return self.manager.index(trans, group_id)
 
     @router.get('/api/groups/{group_id}/roles/{role_id}',

From 35d31d6081b47ce8dda0384ceb2e122fb7a31048 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 14:28:19 +0200
Subject: [PATCH 351/452] Replace str url type with RelativeUrl
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: David López <46503462+davelopez@users.noreply.github.com>
---
 lib/galaxy/schema/schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index cc6df51681d..8702fc86504 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -1908,7 +1908,7 @@ RoleNameIdTuple = Tuple[str, EncodedDatabaseIdField]
 class GroupRoleModel(BaseModel):
     id: EncodedDatabaseIdField = RoleIdField
     name: str = RoleNameField
-    url: str = Field(title="URL", description="URL for the group role")
+    url: RelativeUrl = RelativeUrlField
 
 
 class GroupRoleListModel(BaseModel):

From b0bc577b0f0e78abd67c2613f6d7c62958337e80 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 15:14:46 +0200
Subject: [PATCH 352/452] Replace dict types with Pydantic models

---
 lib/galaxy/managers/group_roles.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index 0fd5ba8ea3b..758c01a60c0 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -2,10 +2,11 @@ import logging
 from typing import (
     Any,
     Dict,
-    List,
     Optional,
 )
 
+from lib.galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
+
 from galaxy import model
 from galaxy.app import MinimalManagerApp
 from galaxy.exceptions import (
@@ -25,7 +26,7 @@ class GroupRolesManager:
     def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
 
-    def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> List[Dict[str, Any]]:
+    def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField):
         """
         Returns a collection roles associated with the given group.
         """
@@ -34,7 +35,7 @@ class GroupRolesManager:
         for gra in group.roles:
             group_role = self._serialize_group_role(group_id, gra.role)
             rval.append(group_role)
-        return rval
+        return GroupRoleListModel(__root__=rval)
 
     def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> Dict[str, Any]:
         """
@@ -110,8 +111,9 @@ class GroupRolesManager:
             url = url_for('role', group_id=encoded_group_id, id=encoded_role_id)
         except AttributeError:
             url = "*deprecated attribute not filled in by FastAPI server*"
-        return {
+
+        return GroupRoleModel(**{
             "id": encoded_role_id,
             "name": role.name,
             "url": url
-        }
+        })

From 3515a219733453ed42b207028075dbce3b5b1c1c Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 15:15:31 +0200
Subject: [PATCH 353/452] Consider url field deprecation in group_roles test

---
 lib/galaxy_test/api/test_group_roles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py
index 67b00527ec3..d445a900f79 100644
--- a/lib/galaxy_test/api/test_group_roles.py
+++ b/lib/galaxy_test/api/test_group_roles.py
@@ -67,7 +67,7 @@ class GroupRolesApiTestCase(ApiTestCase):
         self._assert_status_code_is_ok(update_response)
         group_role = update_response.json()
         self._assert_valid_group_role(group_role, assert_id=encoded_role_id)
-        assert group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}"
+        assert (group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" or group_role["url"] == "*deprecated attribute not filled in by FastAPI server*")
 
     def test_update_only_admin(self):
         encoded_group_id = "any-group-id"

From cf2ccb02a50f4ef0bf16b5a8981a1f973ec135c6 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 16:18:49 +0200
Subject: [PATCH 354/452] Fix imports

---
 lib/galaxy/managers/group_roles.py           | 6 ++----
 lib/galaxy/webapps/galaxy/api/group_roles.py | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index 758c01a60c0..d0456987c0c 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -1,12 +1,9 @@
 import logging
 from typing import (
     Any,
-    Dict,
     Optional,
 )
 
-from lib.galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
-
 from galaxy import model
 from galaxy.app import MinimalManagerApp
 from galaxy.exceptions import (
@@ -15,6 +12,7 @@ from galaxy.exceptions import (
 from galaxy.managers.base import decode_id
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 from galaxy.web import url_for
 
 log = logging.getLogger(__name__)
@@ -37,7 +35,7 @@ class GroupRolesManager:
             rval.append(group_role)
         return GroupRoleListModel(__root__=rval)
 
-    def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> Dict[str, Any]:
+    def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
         """
         Returns information about a group role.
         """
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 9af8282aebf..a0ce3592113 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -4,11 +4,11 @@ API operations on Group objects.
 import logging
 
 from fastapi import Path
-from lib.galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.managers.group_roles import GroupRolesManager
 from galaxy.schema.fields import EncodedDatabaseIdField
+from galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 from galaxy.web import (
     expose_api,
     require_admin,

From 8e384e554adfc2193fa1ac416fb31df7b396ff18 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 16:27:55 +0200
Subject: [PATCH 355/452] Fix group_role url typo

---
 lib/galaxy/managers/group_roles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index d0456987c0c..19bb2c8285a 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -106,7 +106,7 @@ class GroupRolesManager:
     def _serialize_group_role(self, encoded_group_id: EncodedDatabaseIdField, role: model.Role):
         encoded_role_id = self._app.security.encode_id(role.id)
         try:
-            url = url_for('role', group_id=encoded_group_id, id=encoded_role_id)
+            url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
         except AttributeError:
             url = "*deprecated attribute not filled in by FastAPI server*"
 

From e0ca1a932c238be6d31e4b60ab2dcdc778c8fed9 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 18:45:25 +0200
Subject: [PATCH 356/452] Refactor

---
 lib/galaxy/managers/group_roles.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index 19bb2c8285a..0ec67583641 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -24,7 +24,7 @@ class GroupRolesManager:
     def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
 
-    def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField):
+    def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> GroupRoleListModel:
         """
         Returns a collection roles associated with the given group.
         """
@@ -110,8 +110,8 @@ class GroupRolesManager:
         except AttributeError:
             url = "*deprecated attribute not filled in by FastAPI server*"
 
-        return GroupRoleModel(**{
-            "id": encoded_role_id,
-            "name": role.name,
-            "url": url
-        })
+        return GroupRoleModel(
+            id=encoded_role_id,
+            name=role.name,
+            url=url
+        )

From f204483b76e051dbf9eb0a82fc2424bef1be3ff4 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 18:45:42 +0200
Subject: [PATCH 357/452] Remove url assertion from group_roles test

---
 lib/galaxy_test/api/test_group_roles.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py
index d445a900f79..bec1c7e08a4 100644
--- a/lib/galaxy_test/api/test_group_roles.py
+++ b/lib/galaxy_test/api/test_group_roles.py
@@ -67,7 +67,6 @@ class GroupRolesApiTestCase(ApiTestCase):
         self._assert_status_code_is_ok(update_response)
         group_role = update_response.json()
         self._assert_valid_group_role(group_role, assert_id=encoded_role_id)
-        assert (group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" or group_role["url"] == "*deprecated attribute not filled in by FastAPI server*")
 
     def test_update_only_admin(self):
         encoded_group_id = "any-group-id"

From e05aaa99f32df3c92da11f5110ef803c614852f2 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 18:53:44 +0200
Subject: [PATCH 358/452] Add type hints to other functions for consistency

---
 lib/galaxy/managers/group_roles.py           | 6 +++---
 lib/galaxy/webapps/galaxy/api/group_roles.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index 0ec67583641..6ac4c2163b0 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -35,7 +35,7 @@ class GroupRolesManager:
             rval.append(group_role)
         return GroupRoleListModel(__root__=rval)
 
-    def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
+    def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> GroupRoleModel:
         """
         Returns information about a group role.
         """
@@ -48,7 +48,7 @@ class GroupRolesManager:
 
         return self._serialize_group_role(group_id, role)
 
-    def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
+    def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> GroupRoleModel:
         """
         Adds a role to a group if it is not already associated.
         """
@@ -61,7 +61,7 @@ class GroupRolesManager:
 
         return self._serialize_group_role(group_id, role)
 
-    def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
+    def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> GroupRoleModel:
         """
         Removes a role from a group.
         """
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index a0ce3592113..291a09ebbc0 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -57,13 +57,13 @@ class FastAPIGroupRoles:
     @router.put('/api/groups/{group_id}/roles/{role_id}',
                 require_admin=True,
                 summary='Adds a role to a group')
-    def update(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam):
+    def update(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
         return self.manager.update(trans, role_id, group_id)
 
     @router.delete('/api/groups/{group_id}/roles/{role_id}',
                    require_admin=True,
                    summary='Removes a role from a group')
-    def delete(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam):
+    def delete(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
         return self.manager.delete(trans, role_id, group_id)
 
 

From 749e280a4520eab223333be98d7d5af47b5e5196 Mon Sep 17 00:00:00 2001
From: Peter van Heusden <pvh@sanbi.ac.za>
Date: Thu, 12 Aug 2021 21:04:42 +0200
Subject: [PATCH 359/452] Remove unused variables

---
 client/src/components/Toolshed/InstalledList/Monitor.vue      | 2 --
 client/src/components/Toolshed/RepositoryDetails/Index.vue    | 1 -
 .../Toolshed/RepositoryDetails/InstallationButton.vue         | 4 ----
 3 files changed, 7 deletions(-)

diff --git a/client/src/components/Toolshed/InstalledList/Monitor.vue b/client/src/components/Toolshed/InstalledList/Monitor.vue
index 0f22d2775ef..8b851ae64bb 100644
--- a/client/src/components/Toolshed/InstalledList/Monitor.vue
+++ b/client/src/components/Toolshed/InstalledList/Monitor.vue
@@ -30,7 +30,6 @@
 <script>
 import Vue from "vue";
 import BootstrapVue from "bootstrap-vue";
-import { getAppRoot } from "onload/loadConfig";
 import { Services } from "../services";
 import InstallationButton from "../RepositoryDetails/InstallationButton";
 
@@ -58,7 +57,6 @@ export default {
         },
     },
     created() {
-        this.root = getAppRoot();
         this.services = new Services();
         this.load();
     },
diff --git a/client/src/components/Toolshed/RepositoryDetails/Index.vue b/client/src/components/Toolshed/RepositoryDetails/Index.vue
index 503dbb0d08f..df38971c330 100644
--- a/client/src/components/Toolshed/RepositoryDetails/Index.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/Index.vue
@@ -30,7 +30,6 @@
                         </template>
                         <template v-slot:cell(actions)="row">
                             <InstallationButton
-                                :installed="row.item.installed"
                                 :status="row.item.status"
                                 @onInstall="setupRepository(row.item)"
                                 @onUninstall="uninstallRepository(row.item)"
diff --git a/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue b/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
index 7a4f7b134b7..ad8de843837 100644
--- a/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
+++ b/client/src/components/Toolshed/RepositoryDetails/InstallationButton.vue
@@ -20,10 +20,6 @@ Vue.use(BootstrapVue);
 
 export default {
     props: {
-        installed: {
-            type: Boolean,
-            required: false,
-        },
         status: {
             type: String,
             required: true,

From ed53b05f61aee216c1bf1aea933c9281d6050f8a Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Thu, 12 Aug 2021 16:50:18 -0400
Subject: [PATCH 360/452] Fix for empty docker_volumes/singularity_volumes in
 job config

---
 lib/galaxy/tool_util/deps/container_classes.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py
index e9cc48803ec..29ec8bb7148 100644
--- a/lib/galaxy/tool_util/deps/container_classes.py
+++ b/lib/galaxy/tool_util/deps/container_classes.py
@@ -117,6 +117,10 @@ def preprocess_volumes(volumes_raw_str, container_type):
     ro for Singularity iff no subdirectories are rw (Singularity does not allow ro
     parent directories with rw subdirectories).
 
+    >>> preprocess_volumes(None, DOCKER_CONTAINER_TYPE)
+    []
+    >>> preprocess_volumes("", DOCKER_CONTAINER_TYPE)
+    []
     >>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE)
     ['/a/b:rw']
     >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE)
@@ -131,6 +135,9 @@ def preprocess_volumes(volumes_raw_str, container_type):
     ['/a/b', '/a/b/c']
     """
 
+    if not volumes_raw_str:
+        return []
+
     volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")]
     volumes = []
     rw_paths = []

From 7f54f9da0cd7b04348830339f4e30af279849634 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 12 Aug 2021 14:03:47 -0400
Subject: [PATCH 361/452] Move page export button to title

---
 client/src/components/Markdown/Markdown.vue | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Markdown/Markdown.vue b/client/src/components/Markdown/Markdown.vue
index a1e7727f9e8..24b081b19c0 100644
--- a/client/src/components/Markdown/Markdown.vue
+++ b/client/src/components/Markdown/Markdown.vue
@@ -2,10 +2,18 @@
     <div class="markdown-wrapper">
         <LoadingSpan v-if="loading" />
         <div v-else>
-            <a v-if="effectiveExportLink" :href="exportLink" class="markdown-export position-absolute p-3">
-                <i class="fa fa-3x fa-download" />
-            </a>
             <div>
+                <b-button
+                    v-if="effectiveExportLink"
+                    class="float-right"
+                    title="Download PDF"
+                    variant="link"
+                    role="button"
+                    v-b-tooltip.hover.bottom
+                    @click="onDownload"
+                >
+                    <font-awesome-icon icon="download" />
+                </b-button>
                 <b-button
                     v-if="!readOnly"
                     class="float-right"
@@ -97,7 +105,7 @@ import MarkdownIt from "markdown-it";
 import markdownItRegexp from "markdown-it-regexp";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { library } from "@fortawesome/fontawesome-svg-core";
-import { faEdit } from "@fortawesome/free-solid-svg-icons";
+import { faDownload, faEdit } from "@fortawesome/free-solid-svg-icons";
 
 import LoadingSpan from "components/LoadingSpan";
 import HistoryDatasetAsImage from "./Elements/HistoryDatasetAsImage";
@@ -128,7 +136,7 @@ md.use(mdNewline);
 
 Vue.use(BootstrapVue);
 
-library.add(faEdit);
+library.add(faDownload, faEdit);
 
 export default {
     store: store,
@@ -279,6 +287,9 @@ export default {
                 content: content,
             };
         },
+        onDownload() {
+            window.location.href = this.exportLink;
+        },
     },
 };
 </script>

From 9ac8ad07f1f937c60d1878eae8fd818f3d137e04 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 5 Aug 2021 12:00:25 -0400
Subject: [PATCH 362/452] Add collection import button to collection element in
 pages

Add collection import handling to dataset collection display compoenent in pages

Add service handler for markdown elements

Remove debug statement in collection display

Add link hash to import button in collection display

Add dataset type to payload
---
 .../CollectionDisplay.vue                     | 43 ++++++++++++++++++-
 client/src/components/Markdown/services.js    | 19 ++++++++
 2 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 client/src/components/Markdown/services.js

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
index 1d3b81dff4d..6b69dc66870 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
@@ -15,6 +15,23 @@
                     >
                         <span class="fa fa-download" />
                     </b-button>
+                    <CurrentUser v-slot="{ user }">
+                        <UserHistories v-if="user" :user="user" v-slot="{ currentHistoryId }">
+                            <b-button
+                                v-if="currentHistoryId"
+                                @click="onCopyCollection(currentHistoryId)"
+                                href="#"
+                                role="button"
+                                variant="link"
+                                title="Import Collection"
+                                type="button"
+                                class="py-0 px-1"
+                                v-b-tooltip.hover
+                            >
+                                <span class="fa fa-file-import" />
+                            </b-button>
+                        </UserHistories>
+                    </CurrentUser>
                 </span>
                 <span>
                     <span>Dataset Collection:</span>
@@ -24,6 +41,9 @@
             <b-card-body>
                 <LoadingSpan v-if="loading" message="Loading Collection" />
                 <div v-else class="content-height">
+                    <b-alert v-if="!!messageText" :variant="messageVariant" show>
+                        {{ messageText }}
+                    </b-alert>
                     <CollectionTree :node="itemContent" :skip-head="true" />
                 </div>
             </b-card-body>
@@ -32,14 +52,20 @@
 </template>
 
 <script>
-import { getAppRoot } from "onload/loadConfig";
 import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
 import CollectionTree from "./CollectionTree";
 import LoadingSpan from "components/LoadingSpan";
+import CurrentUser from "components/providers/CurrentUser";
+import UserHistories from "components/History/providers/UserHistories";
+import { copyCollection } from "components/Markdown/services";
+
 export default {
     components: {
         CollectionTree,
+        CurrentUser,
         LoadingSpan,
+        UserHistories,
     },
     props: {
         args: {
@@ -55,6 +81,8 @@ export default {
         return {
             itemContent: null,
             loading: true,
+            messageText: null,
+            messageVariant: null,
         };
     },
     created() {
@@ -78,6 +106,19 @@ export default {
         },
     },
     methods: {
+        onCopyCollection(currentHistoryId) {
+            const hdcaId = this.args.history_dataset_collection_id;
+            copyCollection(hdcaId, currentHistoryId).then(
+                (response) => {
+                    this.messageVariant = "success";
+                    this.messageText = "Successfully copied to current history.";
+                },
+                (error) => {
+                    this.messageVariant = "danger";
+                    this.messageText = error;
+                }
+            );
+        },
         async getContent() {
             try {
                 const response = await axios.get(this.itemUrl);
diff --git a/client/src/components/Markdown/services.js b/client/src/components/Markdown/services.js
new file mode 100644
index 00000000000..f7c6f36a315
--- /dev/null
+++ b/client/src/components/Markdown/services.js
@@ -0,0 +1,19 @@
+import axios from "axios";
+import { rethrowSimple } from "utils/simple-error";
+import { getAppRoot } from "onload/loadConfig";
+
+export async function copyCollection(hdcaId, historyId) {
+    const url = `${getAppRoot()}api/histories/${historyId}/contents/dataset_collections`;
+    const payload = {
+        source: "hdca",
+        type: "dataset_collection",
+        content: hdcaId,
+        copy_elements: true,
+    };
+    try {
+        const { data } = await axios.post(url, payload);
+        return data;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}

From 7742e76972e547a7f3cec4940dbb4b01c0bbf774 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 12 Aug 2021 12:49:25 -0400
Subject: [PATCH 363/452] Add upload dialog to global Galaxy object

---
 client/src/app/galaxy.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/client/src/app/galaxy.js b/client/src/app/galaxy.js
index 86dd4f59728..4eb46506d29 100644
--- a/client/src/app/galaxy.js
+++ b/client/src/app/galaxy.js
@@ -8,6 +8,7 @@ import addLogging from "utils/add-logging";
 import localize from "utils/localization";
 import { getGalaxyInstance } from "app";
 import { create, dialog } from "utils/data";
+import { openGlobalUploadModal } from "components/upload/mount";
 
 // ============================================================================
 /** Base galaxy client-side application.
@@ -293,6 +294,11 @@ GalaxyApp.prototype.debuggingNamespaces = function _debuggingNamespaces(namespac
     }
 };
 
+/** Open Upload Dialog */
+GalaxyApp.prototype.uploadDialog = function uploadDialog() {
+    openGlobalUploadModal();
+};
+
 /** string rep */
 GalaxyApp.prototype.toString = function toString() {
     const userEmail = this.user ? this.user.get("email") || "(anonymous)" : "uninitialized";

From c8b5c12884eb42cee19ff3dee21e8383ab1418c7 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Thu, 12 Aug 2021 23:30:23 +0200
Subject: [PATCH 364/452] Add fastapi deprecation message function

---
 lib/galaxy/managers/group_roles.py      | 3 ++-
 lib/galaxy/webapps/galaxy/api/common.py | 4 ++++
 lib/galaxy/webapps/galaxy/api/roles.py  | 3 ++-
 lib/galaxy_test/api/test_group_roles.py | 2 ++
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index 6ac4c2163b0..bef283c9f64 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -14,6 +14,7 @@ from galaxy.managers.context import ProvidesAppContext
 from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 from galaxy.web import url_for
+from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 
 log = logging.getLogger(__name__)
 
@@ -108,7 +109,7 @@ class GroupRolesManager:
         try:
             url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
         except AttributeError:
-            url = "*deprecated attribute not filled in by FastAPI server*"
+            url = fastapi_deprecation_message()
 
         return GroupRoleModel(
             id=encoded_role_id,
diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py
index f8bc267c2c6..dbd889c1f60 100644
--- a/lib/galaxy/webapps/galaxy/api/common.py
+++ b/lib/galaxy/webapps/galaxy/api/common.py
@@ -28,3 +28,7 @@ def parse_serialization_params(
     if keys:
         key_list = keys.split(',')
     return dict(view=view, keys=key_list, default_view=default_view)
+
+
+def fastapi_deprecation_message():
+    return "*deprecated attribute not filled in by FastAPI server*"
diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py
index 210d96a0ae7..d427cc5f5f8 100644
--- a/lib/galaxy/webapps/galaxy/api/roles.py
+++ b/lib/galaxy/webapps/galaxy/api/roles.py
@@ -19,6 +19,7 @@ from galaxy.schema.schema import (
     RoleModel,
 )
 from galaxy.webapps.base.controller import url_for
+from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 from . import (
     BaseGalaxyAPIController,
     depends,
@@ -40,7 +41,7 @@ def role_to_model(trans, role):
     try:
         item['url'] = url_for('role', id=role_id)
     except AttributeError:
-        item['url'] = "*deprecated attribute not filled in by FastAPI server*"
+        item['url'] = fastapi_deprecation_message()
     return RoleModel(**item)
 
 
diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py
index bec1c7e08a4..aa56df78ddd 100644
--- a/lib/galaxy_test/api/test_group_roles.py
+++ b/lib/galaxy_test/api/test_group_roles.py
@@ -1,5 +1,6 @@
 from typing import List
 
+from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 from galaxy_test.base.populators import DatasetPopulator
 from ._framework import ApiTestCase
 
@@ -67,6 +68,7 @@ class GroupRolesApiTestCase(ApiTestCase):
         self._assert_status_code_is_ok(update_response)
         group_role = update_response.json()
         self._assert_valid_group_role(group_role, assert_id=encoded_role_id)
+        assert (group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" or group_role["url"] == fastapi_deprecation_message())
 
     def test_update_only_admin(self):
         encoded_group_id = "any-group-id"

From fe8366e6cbf36083b86556e6342effdf76bf9d39 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 13 Aug 2021 11:05:33 -0400
Subject: [PATCH 365/452] Fix component case-sensitivity; import from primary
 interface instead of mount

---
 client/src/app/galaxy.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/app/galaxy.js b/client/src/app/galaxy.js
index 4eb46506d29..e8cc2c67ce7 100644
--- a/client/src/app/galaxy.js
+++ b/client/src/app/galaxy.js
@@ -8,7 +8,7 @@ import addLogging from "utils/add-logging";
 import localize from "utils/localization";
 import { getGalaxyInstance } from "app";
 import { create, dialog } from "utils/data";
-import { openGlobalUploadModal } from "components/upload/mount";
+import { openGlobalUploadModal } from "components/Upload";
 
 // ============================================================================
 /** Base galaxy client-side application.

From 7c9c7f830536a0d15e5c8036618379d461743fa0 Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Fri, 13 Aug 2021 14:24:16 -0400
Subject: [PATCH 366/452] Rename containers_resolvers_config_file to
 container_resolvers_config_file

---
 doc/source/admin/galaxy_options.rst                    | 10 +++++-----
 lib/galaxy/config/__init__.py                          | 10 ++++++----
 .../config/sample/container_resolvers_conf.xml.sample  |  4 ++--
 lib/galaxy/config/sample/galaxy.yml.sample             |  8 ++++----
 lib/galaxy/tool_util/deps/containers.py                |  8 ++++----
 lib/galaxy/tool_util/deps/dependencies.py              |  4 ++--
 lib/galaxy/webapps/galaxy/config_schema.yml            |  6 +++---
 7 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index ba2ba87389a..55c452e3e78 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -679,14 +679,14 @@
 :Type: bool
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``containers_resolvers_config_file``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``container_resolvers_config_file``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
     Container resolvers configuration (beta). Set up a file describing
     container resolvers to use when discovering containers for Galaxy.
-    If this is set to None, the default containers loaded is
+    If this is set to None, the default container resolvers loaded is
     determined by enable_mulled_containers.
 :Default: ``None``
 :Type: str
@@ -4449,7 +4449,7 @@
     filtering
     (https://galaxyproject.org/user-defined-toolbox-filters/)
     functions.
-:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters``
+:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters``
 :Type: str
 
 
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 0eb7403b1ae..a49c3e32f9a 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -491,13 +491,15 @@ class CommonConfigurationMixin:
 
 class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
     deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file',
-                          'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist')
+                          'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist',
+                          'containers_resolvers_config_file')
     renamed_options = {
         'blacklist_file': 'email_domain_blocklist_file',
         'whitelist_file': 'email_domain_allowlist_file',
         'sanitize_whitelist_file': 'sanitize_allowlist_file',
         'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist',
         'fetch_url_whitelist': 'fetch_url_allowlist',
+        'containers_resolvers_config_file': 'container_resolvers_config_file',
     }
     default_config_file_name = 'galaxy.yml'
     deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'}
@@ -728,8 +730,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
             # self, populate config_dict
             self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping]
 
-        if self.containers_resolvers_config_file:
-            self.containers_resolvers_config_file = self._in_config_dir(self.containers_resolvers_config_file)
+        if self.container_resolvers_config_file:
+            self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file)
 
         # tool_dependency_dir can be "none" (in old configs). If so, set it to None
         if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none':
@@ -1175,7 +1177,7 @@ class ConfiguresGalaxyMixin:
             container_image_cache_path=self.config.container_image_cache_path,
             library_import_dir=self.config.library_import_dir,
             enable_mulled_containers=self.config.enable_mulled_containers,
-            containers_resolvers_config_file=self.config.containers_resolvers_config_file,
+            container_resolvers_config_file=self.config.container_resolvers_config_file,
             involucro_path=self.config.involucro_path,
             involucro_auto_init=self.config.involucro_auto_init,
             mulled_channels=self.config.mulled_channels,
diff --git a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample
index 78051d5cb64..ce59312b097 100644
--- a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample
+++ b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample
@@ -1,4 +1,4 @@
-<containers_resolvers>
+<container_resolvers>
   <explicit />
   <!-- explicit: resolves container URI for a job through explict container
        tags in the tool XML wrapper. -->
@@ -63,4 +63,4 @@
   <!-- Specify a fallback container for tools/jobs that don't match the above
        resolvers. -->
 
-</containers_resolvers>
+</container_resolvers>
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index 1184e78bfa4..89924ec739b 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -438,9 +438,9 @@ galaxy:
 
   # Container resolvers configuration (beta). Set up a file describing
   # container resolvers to use when discovering containers for Galaxy.
-  # If this is set to None, the default containers loaded is determined
-  # by enable_mulled_containers.
-  #containers_resolvers_config_file: null
+  # If this is set to None, the default container resolvers loaded is
+  # determined by enable_mulled_containers.
+  #container_resolvers_config_file: null
 
   # involucro is a tool used to build Docker or Singularity containers
   # for tools from Conda dependencies referenced in tools as
@@ -2188,7 +2188,7 @@ galaxy:
   # The base module(s) that are searched for modules for toolbox
   # filtering (https://galaxyproject.org/user-defined-toolbox-filters/)
   # functions.
-  #toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters
+  #toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters
 
   # Galaxy uses AMQP internally for communicating between processes.
   # For example, when reloading the toolbox or locking job execution,
diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py
index af49c501556..5bd43e4b453 100644
--- a/lib/galaxy/tool_util/deps/containers.py
+++ b/lib/galaxy/tool_util/deps/containers.py
@@ -194,12 +194,12 @@ class ContainerRegistry:
         self.mulled_resolution_cache = mulled_resolution_cache
 
     def __build_container_resolvers(self, app_info):
-        conf_file = getattr(app_info, 'containers_resolvers_config_file', None)
+        conf_file = getattr(app_info, 'container_resolvers_config_file', None)
         if not conf_file:
-            return self.__default_containers_resolvers()
+            return self.__default_container_resolvers()
         if not os.path.exists(conf_file):
             log.debug("Unable to find config file '%s'", conf_file)
-            return self.__default_containers_resolvers()
+            return self.__default_container_resolvers()
         plugin_source = plugin_config.plugin_source_from_path(conf_file)
         return self._parse_resolver_conf(plugin_source)
 
@@ -209,7 +209,7 @@ class ContainerRegistry:
         }
         return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
 
-    def __default_containers_resolvers(self):
+    def __default_container_resolvers(self):
         default_resolvers = [
             ExplicitContainerResolver(self.app_info),
             ExplicitSingularityContainerResolver(self.app_info),
diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py
index 760eac66239..8d513d01cf1 100644
--- a/lib/galaxy/tool_util/deps/dependencies.py
+++ b/lib/galaxy/tool_util/deps/dependencies.py
@@ -15,7 +15,7 @@ class AppInfo:
         container_image_cache_path=None,
         library_import_dir=None,
         enable_mulled_containers=False,
-        containers_resolvers_config_file=None,
+        container_resolvers_config_file=None,
         involucro_path=None,
         involucro_auto_init=True,
         mulled_channels=DEFAULT_CHANNELS,
@@ -29,7 +29,7 @@ class AppInfo:
         self.container_image_cache_path = container_image_cache_path
         self.library_import_dir = library_import_dir
         self.enable_mulled_containers = enable_mulled_containers
-        self.containers_resolvers_config_file = containers_resolvers_config_file
+        self.container_resolvers_config_file = container_resolvers_config_file
         self.involucro_path = involucro_path
         self.involucro_auto_init = involucro_auto_init
         self.mulled_channels = mulled_channels
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index be85ddc24ef..ec6f109fa19 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -504,14 +504,14 @@ mapping:
           Container availability will vary by tool, this option will only be used
           for job destinations with Docker or Singularity enabled.
 
-      containers_resolvers_config_file:
+      container_resolvers_config_file:
         type: str
         required: false
         desc: |
           Container resolvers configuration (beta). Set up a file describing
           container resolvers to use when discovering containers for Galaxy. If
-          this is set to None, the default containers loaded is determined by
-          enable_mulled_containers.
+          this is set to None, the default container resolvers loaded is
+          determined by enable_mulled_containers.
 
       involucro_path:
         type: str

From 0d3ef9d060e86f38db85675c9502662fe8f45460 Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Fri, 13 Aug 2021 14:55:23 -0400
Subject: [PATCH 367/452] Update tests for config option rename

---
 test/integration/test_config_defaults.py    | 2 +-
 test/integration/test_containerized_jobs.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/integration/test_config_defaults.py b/test/integration/test_config_defaults.py
index 343029b941a..bf91badb288 100644
--- a/test/integration/test_config_defaults.py
+++ b/test/integration/test_config_defaults.py
@@ -54,7 +54,7 @@ PATH_CONFIG_PROPERTIES = [
     'citation_cache_data_dir',
     'citation_cache_lock_dir',
     'cluster_files_directory',
-    'containers_resolvers_config_file',
+    'container_resolvers_config_file',
     'data_manager_config_file',
     'datatypes_config_file',
     'dependency_resolvers_config_file',
diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py
index 3ee0a3dbcd6..d7abc533f56 100644
--- a/test/integration/test_containerized_jobs.py
+++ b/test/integration/test_containerized_jobs.py
@@ -155,8 +155,8 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase):
         config["jobs_directory"] = cls.jobs_directory
         config["job_config_file"] = cls.job_config_file
         disable_dependency_resolution(config)
-        containers_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml")
-        with open(containers_resolvers_config_path, "w") as f:
+        container_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml")
+        with open(container_resolvers_config_path, "w") as f:
             f.write("""
 - type: mapping
   mappings:
@@ -164,7 +164,7 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase):
       tool_id: mulled_example_broken_no_requirements
       identifier: 'quay.io/biocontainers/bwa:0.7.15--0'
 """)
-        config["containers_resolvers_config_file"] = containers_resolvers_config_path
+        config["container_resolvers_config_file"] = container_resolvers_config_path
 
     @classmethod
     def setUpClass(cls):

From db6944523f4cbdb6e28b2567c37959741d82020b Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Fri, 13 Aug 2021 15:23:30 -0400
Subject: [PATCH 368/452] Support setting container resolvers directly in the
 app config

---
 doc/source/admin/galaxy_options.rst         | 16 ++++++++++++++++
 lib/galaxy/config/__init__.py               |  1 +
 lib/galaxy/config/sample/galaxy.yml.sample  |  9 +++++++++
 lib/galaxy/tool_util/deps/containers.py     | 20 +++++++++++++-------
 lib/galaxy/tool_util/deps/dependencies.py   |  2 ++
 lib/galaxy/webapps/galaxy/config_schema.yml | 14 ++++++++++++++
 test/integration/test_containerized_jobs.py | 19 +++++++++++++++++++
 7 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 55c452e3e78..70c10644e8a 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -692,6 +692,22 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~
+``container_resolvers``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Rather than specifying a container_resolvers_config_file, the
+    definition of the resolvers to enable can be embedded into
+    Galaxy's config with this option. This has no effect if a
+    container_resolvers_config_file is used.
+    The syntax, available resolvers, and documentation of their
+    options is explained in detail in the documentation:
+    https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html
+:Default: ``None``
+:Type: seq
+
+
 ~~~~~~~~~~~~~~~~~~
 ``involucro_path``
 ~~~~~~~~~~~~~~~~~~
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index a49c3e32f9a..216379255e1 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -1178,6 +1178,7 @@ class ConfiguresGalaxyMixin:
             library_import_dir=self.config.library_import_dir,
             enable_mulled_containers=self.config.enable_mulled_containers,
             container_resolvers_config_file=self.config.container_resolvers_config_file,
+            container_resolvers_config_dict=self.config.container_resolvers,
             involucro_path=self.config.involucro_path,
             involucro_auto_init=self.config.involucro_auto_init,
             mulled_channels=self.config.mulled_channels,
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index 89924ec739b..59df1a5a4ae 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -442,6 +442,15 @@ galaxy:
   # determined by enable_mulled_containers.
   #container_resolvers_config_file: null
 
+  # Rather than specifying a container_resolvers_config_file, the
+  # definition of the resolvers to enable can be embedded into Galaxy's
+  # config with this option. This has no effect if a
+  # container_resolvers_config_file is used.
+  # The syntax, available resolvers, and documentation of their options
+  # is explained in detail in the documentation:
+  # https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html
+  #container_resolvers: null
+
   # involucro is a tool used to build Docker or Singularity containers
   # for tools from Conda dependencies referenced in tools as
   # `requirement` s. The following path is the location of involucro on
diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py
index 5bd43e4b453..30dc9456607 100644
--- a/lib/galaxy/tool_util/deps/containers.py
+++ b/lib/galaxy/tool_util/deps/containers.py
@@ -195,13 +195,19 @@ class ContainerRegistry:
 
     def __build_container_resolvers(self, app_info):
         conf_file = getattr(app_info, 'container_resolvers_config_file', None)
-        if not conf_file:
-            return self.__default_container_resolvers()
-        if not os.path.exists(conf_file):
-            log.debug("Unable to find config file '%s'", conf_file)
-            return self.__default_container_resolvers()
-        plugin_source = plugin_config.plugin_source_from_path(conf_file)
-        return self._parse_resolver_conf(plugin_source)
+        conf_dict = getattr(app_info, 'container_resolvers_config_dict', None)
+        plugin_source = None
+        if conf_file and not os.path.exists(conf_file):
+            log.warning(f"Unable to find config file '{conf_file}'")
+        elif conf_file:
+            log.debug("Loading container resolution config from file '{conf_file}'")
+            plugin_source = plugin_config.plugin_source_from_path(conf_file)
+        elif conf_dict:
+            log.debug("Loading container resolution config inline from Galaxy configuration file")
+            plugin_source = plugin_config.plugin_source_from_dict(conf_dict)
+        if plugin_source:
+            return self._parse_resolver_conf(plugin_source)
+        return self.__default_container_resolvers()
 
     def _parse_resolver_conf(self, plugin_source):
         extra_kwds = {
diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py
index 8d513d01cf1..ec22c43afe1 100644
--- a/lib/galaxy/tool_util/deps/dependencies.py
+++ b/lib/galaxy/tool_util/deps/dependencies.py
@@ -16,6 +16,7 @@ class AppInfo:
         library_import_dir=None,
         enable_mulled_containers=False,
         container_resolvers_config_file=None,
+        container_resolvers_config_dict=None,
         involucro_path=None,
         involucro_auto_init=True,
         mulled_channels=DEFAULT_CHANNELS,
@@ -30,6 +31,7 @@ class AppInfo:
         self.library_import_dir = library_import_dir
         self.enable_mulled_containers = enable_mulled_containers
         self.container_resolvers_config_file = container_resolvers_config_file
+        self.container_resolvers_config_dict = container_resolvers_config_dict
         self.involucro_path = involucro_path
         self.involucro_auto_init = involucro_auto_init
         self.mulled_channels = mulled_channels
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index ec6f109fa19..a26c08487eb 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -513,6 +513,20 @@ mapping:
           this is set to None, the default container resolvers loaded is
           determined by enable_mulled_containers.
 
+      container_resolvers:
+        type: seq
+        sequence:
+          - type: any
+        desc: |
+          Rather than specifying a container_resolvers_config_file, the definition of the
+          resolvers to enable can be embedded into Galaxy's config with this option.
+          This has no effect if a container_resolvers_config_file is used.
+
+          The syntax, available resolvers, and documentation of their options is explained in detail in the
+          documentation:
+
+          https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html
+
       involucro_path:
         type: str
         default: involucro
diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py
index d7abc533f56..3887650cf55 100644
--- a/test/integration/test_containerized_jobs.py
+++ b/test/integration/test_containerized_jobs.py
@@ -183,6 +183,25 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase):
         assert "0.7.15-r1140" in output
 
 
+class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = cls._test_driver.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = cls.job_config_file
+        disable_dependency_resolution(config)
+        container_resolvers_config = [{
+            'type': 'mapping',
+            'mappings': [{
+                'container_type': 'docker',
+                'tool_id': 'mulled_example_broken_no_requirements',
+                'identifier': 'quay.io/biocontainers/bwa:0.7.15--0',
+            }],
+        }]
+        config["container_resolvers"] = container_resolvers_config
+
+
 # Singularity 2.4 in the official Vagrant issue has some problems running this test
 # case by default because subdirectories of /tmp don't bind correctly. Overridding
 # TMPDIR can fix this.

From 509e61998ec184b0e7f45826ebbe260fd671b1c9 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Mon, 16 Aug 2021 15:17:33 -0400
Subject: [PATCH 369/452] Swap globalUpload function to bundleEntries, as is
 more appropriate for externally exposed functionality

---
 client/src/app/galaxy.js    | 6 ------
 client/src/bundleEntries.js | 1 +
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/client/src/app/galaxy.js b/client/src/app/galaxy.js
index e8cc2c67ce7..86dd4f59728 100644
--- a/client/src/app/galaxy.js
+++ b/client/src/app/galaxy.js
@@ -8,7 +8,6 @@ import addLogging from "utils/add-logging";
 import localize from "utils/localization";
 import { getGalaxyInstance } from "app";
 import { create, dialog } from "utils/data";
-import { openGlobalUploadModal } from "components/Upload";
 
 // ============================================================================
 /** Base galaxy client-side application.
@@ -294,11 +293,6 @@ GalaxyApp.prototype.debuggingNamespaces = function _debuggingNamespaces(namespac
     }
 };
 
-/** Open Upload Dialog */
-GalaxyApp.prototype.uploadDialog = function uploadDialog() {
-    openGlobalUploadModal();
-};
-
 /** string rep */
 GalaxyApp.prototype.toString = function toString() {
     const userEmail = this.user ? this.user.get("email") || "(anonymous)" : "uninitialized";
diff --git a/client/src/bundleEntries.js b/client/src/bundleEntries.js
index a4bfdcd9dcd..824504c7b43 100644
--- a/client/src/bundleEntries.js
+++ b/client/src/bundleEntries.js
@@ -32,6 +32,7 @@ export { default as LegacyGridView } from "legacy/grid/grid-view";
 export { create_chart, create_histogram } from "reports/run_stats";
 export { default as ToolshedGroups } from "toolshed/toolshed.groups";
 export { default as IES } from "galaxy.interactive_environments";
+export { openGlobalUploadModal } from "components/Upload";
 
 export { Toast } from "ui/toast"; // TODO: remove when external consumers are updated/gone (IES right now)
 

From e3a0fe543b91177593165ec44d3e1a3cbe8fa9a7 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Mon, 16 Aug 2021 22:46:57 +0200
Subject: [PATCH 370/452] Move API related code to api/group_roles.py

---
 lib/galaxy/managers/group_roles.py           | 39 +++++---------------
 lib/galaxy/webapps/galaxy/api/group_roles.py | 29 +++++++++++++--
 lib/galaxy_test/api/test_group_roles.py      |  3 +-
 3 files changed, 35 insertions(+), 36 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index bef283c9f64..e6a484c26e9 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -12,9 +12,6 @@ from galaxy.exceptions import (
 from galaxy.managers.base import decode_id
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.schema.fields import EncodedDatabaseIdField
-from galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
-from galaxy.web import url_for
-from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 
 log = logging.getLogger(__name__)
 
@@ -25,18 +22,14 @@ class GroupRolesManager:
     def __init__(self, app: MinimalManagerApp) -> None:
         self._app = app
 
-    def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField) -> GroupRoleListModel:
+    def index(self, trans: ProvidesAppContext, group_id: EncodedDatabaseIdField):
         """
         Returns a collection roles associated with the given group.
         """
         group = self._get_group(trans, group_id)
-        rval = []
-        for gra in group.roles:
-            group_role = self._serialize_group_role(group_id, gra.role)
-            rval.append(group_role)
-        return GroupRoleListModel(__root__=rval)
+        return group.roles
 
-    def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> GroupRoleModel:
+    def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
         """
         Returns information about a group role.
         """
@@ -46,10 +39,10 @@ class GroupRolesManager:
         group_role = self._get_group_role(trans, group, role)
         if group_role is None:
             raise ObjectNotFound(f"Role {role.name} not in group {group.name}")
+        return role
+        # return self._serialize_group_role(group_id, role)
 
-        return self._serialize_group_role(group_id, role)
-
-    def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> GroupRoleModel:
+    def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
         """
         Adds a role to a group if it is not already associated.
         """
@@ -59,10 +52,9 @@ class GroupRolesManager:
         group_role = self._get_group_role(trans, group, role)
         if group_role is None:
             self._add_role_to_group(trans, group, role)
+        return role
 
-        return self._serialize_group_role(group_id, role)
-
-    def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> GroupRoleModel:
+    def delete(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
         """
         Removes a role from a group.
         """
@@ -73,7 +65,7 @@ class GroupRolesManager:
         if group_role is None:
             raise ObjectNotFound(f"Role {role.name} not in group {group.name}")
         self._remove_role_from_group(trans, group_role)
-        return self._serialize_group_role(group_id, role)
+        return role
 
     def _get_group(self, trans: ProvidesAppContext, encoded_group_id: EncodedDatabaseIdField) -> Any:
         decoded_group_id = decode_id(self._app, encoded_group_id)
@@ -103,16 +95,3 @@ class GroupRolesManager:
     def _remove_role_from_group(self, trans: ProvidesAppContext, group_role: model.GroupRoleAssociation):
         trans.sa_session.delete(group_role)
         trans.sa_session.flush()
-
-    def _serialize_group_role(self, encoded_group_id: EncodedDatabaseIdField, role: model.Role):
-        encoded_role_id = self._app.security.encode_id(role.id)
-        try:
-            url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
-        except AttributeError:
-            url = fastapi_deprecation_message()
-
-        return GroupRoleModel(
-            id=encoded_role_id,
-            name=role.name,
-            url=url
-        )
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 291a09ebbc0..ad5bfe086f3 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -1,9 +1,11 @@
 """
 API operations on Group objects.
 """
+
 import logging
 
 from fastapi import Path
+from routes.util import url_for
 
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.managers.group_roles import GroupRolesManager
@@ -13,6 +15,7 @@ from galaxy.web import (
     expose_api,
     require_admin,
 )
+from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 from . import (
     BaseGalaxyAPIController,
     depends,
@@ -38,6 +41,20 @@ RoleIDParam: EncodedDatabaseIdField = Path(
 )
 
 
+def group_role_to_model(trans, encoded_group_id, role):
+    encoded_role_id = trans.security.encode_id(role.id)
+    try:
+        url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
+    except AttributeError:
+        url = fastapi_deprecation_message()
+
+    return GroupRoleModel(
+        id=encoded_role_id,
+        name=role.name,
+        url=url
+    )
+
+
 @router.cbv
 class FastAPIGroupRoles:
     manager: GroupRolesManager = depends(GroupRolesManager)
@@ -46,25 +63,29 @@ class FastAPIGroupRoles:
                 require_admin=True,
                 summary='Displays a collection (list) of groups.')
     def index(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
-        return self.manager.index(trans, group_id)
+        group_roles = self.manager.index(trans, group_id)
+        return GroupRoleListModel(__root__=[group_role_to_model(trans, group_id, gr.role) for gr in group_roles])
 
     @router.get('/api/groups/{group_id}/roles/{role_id}',
                 require_admin=True,
                 summary='Displays information about a group role.')
     def show(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
-        return self.manager.show(trans, role_id, group_id)
+        role = self.manager.show(trans, role_id, group_id)
+        return group_role_to_model(trans, group_id, role)
 
     @router.put('/api/groups/{group_id}/roles/{role_id}',
                 require_admin=True,
                 summary='Adds a role to a group')
     def update(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
-        return self.manager.update(trans, role_id, group_id)
+        role = self.manager.update(trans, role_id, group_id)
+        return group_role_to_model(trans, group_id, role)
 
     @router.delete('/api/groups/{group_id}/roles/{role_id}',
                    require_admin=True,
                    summary='Removes a role from a group')
     def delete(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
-        return self.manager.delete(trans, role_id, group_id)
+        role = self.manager.delete(trans, role_id, group_id)
+        return group_role_to_model(trans, group_id, role)
 
 
 class GroupRolesAPIController(BaseGalaxyAPIController):
diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py
index aa56df78ddd..d445a900f79 100644
--- a/lib/galaxy_test/api/test_group_roles.py
+++ b/lib/galaxy_test/api/test_group_roles.py
@@ -1,6 +1,5 @@
 from typing import List
 
-from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 from galaxy_test.base.populators import DatasetPopulator
 from ._framework import ApiTestCase
 
@@ -68,7 +67,7 @@ class GroupRolesApiTestCase(ApiTestCase):
         self._assert_status_code_is_ok(update_response)
         group_role = update_response.json()
         self._assert_valid_group_role(group_role, assert_id=encoded_role_id)
-        assert (group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" or group_role["url"] == fastapi_deprecation_message())
+        assert (group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" or group_role["url"] == "*deprecated attribute not filled in by FastAPI server*")
 
     def test_update_only_admin(self):
         encoded_group_id = "any-group-id"

From ae38fb79572d591ee0db06d7e764f0502393f50c Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Mon, 16 Aug 2021 17:55:43 -0700
Subject: [PATCH 371/452] persist expanded items in session between page loads

---
 .../History/providers/ExpandedItems.js        | 19 ++++++++++++++
 .../components/History/providers/setCache.js  | 25 +++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 client/src/components/History/providers/setCache.js

diff --git a/client/src/components/History/providers/ExpandedItems.js b/client/src/components/History/providers/ExpandedItems.js
index a281794ef35..05c4fdc3452 100644
--- a/client/src/components/History/providers/ExpandedItems.js
+++ b/client/src/components/History/providers/ExpandedItems.js
@@ -1,3 +1,5 @@
+import { saveSet, loadSet } from "./setCache";
+
 export default {
     props: {
         scopeKey: { type: String, required: true },
@@ -8,6 +10,11 @@ export default {
             items: new Set(),
         };
     },
+    computed: {
+        expandedCount() {
+            return this.items.size;
+        },
+    },
     methods: {
         isExpanded(item) {
             const key = this.getItemKey(item);
@@ -29,11 +36,23 @@ export default {
                 this.reset();
             }
         },
+        items(newSet) {
+            saveSet(this.key, newSet);
+        },
+    },
+    created() {
+        this.key = "expanded-history-items";
+        const cachedSet = loadSet(this.key);
+        if (cachedSet) {
+            this.items = cachedSet;
+        }
     },
     render() {
         return this.$scopedSlots.default({
             isExpanded: this.isExpanded,
             setExpanded: this.setExpanded,
+            collapseAll: this.reset,
+            expandedCount: this.expandedCount,
         });
     },
 };
diff --git a/client/src/components/History/providers/setCache.js b/client/src/components/History/providers/setCache.js
new file mode 100644
index 00000000000..90d9bbe5b40
--- /dev/null
+++ b/client/src/components/History/providers/setCache.js
@@ -0,0 +1,25 @@
+// Caches expanded/selected Sets in sessionStorage
+// Putting this in separate file for ease of mocking
+
+export function saveSet(key, setObject) {
+    try {
+        const setToArray = Array.from(setObject);
+        const json = JSON.stringify(setToArray);
+        sessionStorage.setItem(key, json);
+    } catch (err) {
+        console.log("error saving stuff to cache", err);
+        sessionStorage.removeItem(key);
+    }
+}
+
+export function loadSet(key) {
+    try {
+        const arrJson = sessionStorage.getItem(key) || [];
+        const cachedArray = JSON.parse(arrJson);
+        return Array.isArray(cachedArray) ? new Set(cachedArray) : new Set();
+    } catch (err) {
+        console.log("erorr loading cached list", err);
+        sessionStorage.removeItem(key);
+        return null;
+    }
+}

From e37e8842e45f431bd2cb9857923842bc6b8868d4 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Mon, 16 Aug 2021 18:03:24 -0700
Subject: [PATCH 372/452] lint, cleanup

---
 client/src/components/History/providers/setCache.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/components/History/providers/setCache.js b/client/src/components/History/providers/setCache.js
index 90d9bbe5b40..94c3225f40b 100644
--- a/client/src/components/History/providers/setCache.js
+++ b/client/src/components/History/providers/setCache.js
@@ -1,5 +1,4 @@
 // Caches expanded/selected Sets in sessionStorage
-// Putting this in separate file for ease of mocking
 
 export function saveSet(key, setObject) {
     try {
@@ -7,7 +6,7 @@ export function saveSet(key, setObject) {
         const json = JSON.stringify(setToArray);
         sessionStorage.setItem(key, json);
     } catch (err) {
-        console.log("error saving stuff to cache", err);
+        console.warn("Error saving set to cache", key, err);
         sessionStorage.removeItem(key);
     }
 }
@@ -18,7 +17,7 @@ export function loadSet(key) {
         const cachedArray = JSON.parse(arrJson);
         return Array.isArray(cachedArray) ? new Set(cachedArray) : new Set();
     } catch (err) {
-        console.log("erorr loading cached list", err);
+        console.warn("Error loading cached list", key, err);
         sessionStorage.removeItem(key);
         return null;
     }

From d3a4e9d22a66dbd8faa978188c6df54ee60c0bd9 Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Tue, 17 Aug 2021 14:15:55 +0200
Subject: [PATCH 373/452] Fix legacy endpoints

---
 lib/galaxy/webapps/galaxy/api/group_roles.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index ad5bfe086f3..4d73f0f511a 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -98,7 +98,8 @@ class GroupRolesAPIController(BaseGalaxyAPIController):
         GET /api/groups/{encoded_group_id}/roles
         Displays a collection (list) of groups.
         """
-        return self.manager.index(trans, group_id)
+        group_roles = self.manager.index(trans, group_id)
+        return GroupRoleListModel(__root__=[group_role_to_model(trans, group_id, gr.role) for gr in group_roles])
 
     @require_admin
     @expose_api
@@ -107,7 +108,8 @@ class GroupRolesAPIController(BaseGalaxyAPIController):
         GET /api/groups/{encoded_group_id}/roles/{encoded_role_id}
         Displays information about a group role.
         """
-        return self.manager.show(trans, id, group_id)
+        role = self.manager.show(trans, id, group_id)
+        return group_role_to_model(trans, group_id, role)
 
     @require_admin
     @expose_api
@@ -116,7 +118,8 @@ class GroupRolesAPIController(BaseGalaxyAPIController):
         PUT /api/groups/{encoded_group_id}/roles/{encoded_role_id}
         Adds a role to a group
         """
-        return self.manager.update(trans, id, group_id)
+        role = self.manager.update(trans, id, group_id)
+        return group_role_to_model(trans, group_id, role)
 
     @require_admin
     @expose_api
@@ -125,4 +128,5 @@ class GroupRolesAPIController(BaseGalaxyAPIController):
         DELETE /api/groups/{encoded_group_id}/roles/{encoded_role_id}
         Removes a role from a group
         """
-        return self.manager.delete(trans, id, group_id)
+        role = self.manager.delete(trans, id, group_id)
+        return group_role_to_model(trans, group_id, role)

From a6ac841f85f36a2d73fa5398a0fb022ec3d3486e Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Mon, 16 Aug 2021 14:38:53 -0400
Subject: [PATCH 374/452] Support setting per-job-environment container
 resolvers using the `container_resolvers` key in the environment config.

---
 lib/galaxy/config/__init__.py                 |  4 +-
 lib/galaxy/tool_util/deps/containers.py       | 54 ++++++++++++++-----
 ...xample_broken_no_requirements_fallback.xml | 25 +++++++++
 test/functional/tools/samples_tool_conf.xml   |  1 +
 test/integration/dockerized_job_conf.xml      | 22 --------
 test/integration/dockerized_job_conf.yml      | 26 +++++++++
 test/integration/singularity_job_conf.xml     | 21 --------
 test/integration/singularity_job_conf.yml     | 18 +++++++
 test/integration/test_containerized_jobs.py   | 34 +++++++++++-
 9 files changed, 144 insertions(+), 61 deletions(-)
 create mode 100644 test/functional/tools/mulled_example_broken_no_requirements_fallback.xml
 delete mode 100644 test/integration/dockerized_job_conf.xml
 create mode 100644 test/integration/dockerized_job_conf.yml
 delete mode 100644 test/integration/singularity_job_conf.xml
 create mode 100644 test/integration/singularity_job_conf.yml

diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 216379255e1..83231bbb541 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -1208,8 +1208,8 @@ class ConfiguresGalaxyMixin:
                 for enabled_container_type in self.container_finder._enabled_container_types(destination.params):
                     container_types_to_destinations[enabled_container_type].append(destination)
         self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations)
-        self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes)
-        self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers)
+        self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes)
+        self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers)
 
     def _configure_tool_data_tables(self, from_shed_config):
         from galaxy.tools.data import ToolDataTableManager
diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py
index 30dc9456607..2d0d74ff409 100644
--- a/lib/galaxy/tool_util/deps/containers.py
+++ b/lib/galaxy/tool_util/deps/containers.py
@@ -42,7 +42,9 @@ class ContainerFinder:
 
     def __init__(self, app_info, mulled_resolution_cache=None):
         self.app_info = app_info
-        self.container_registry = ContainerRegistry(app_info, mulled_resolution_cache=mulled_resolution_cache)
+        self.mulled_resolution_cache = mulled_resolution_cache
+        self.default_container_registry = ContainerRegistry(app_info, mulled_resolution_cache=mulled_resolution_cache)
+        self.destination_container_registeries = {}
 
     def _enabled_container_types(self, destination_info):
         return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)]
@@ -51,11 +53,26 @@ class ContainerFinder:
         """Regardless of destination properties - find best container for tool.
 
         Given container types and container.ToolInfo description of the tool."""
-        return self.container_registry.find_best_container_description(enabled_container_types, tool_info, **kwds)
+        return self.default_container_registry.find_best_container_description(enabled_container_types, tool_info, **kwds)
 
     def resolve(self, enabled_container_types, tool_info, **kwds):
         """Regardless of destination properties - find ResolvedContainerDescription for tool."""
-        return self.container_registry.resolve(enabled_container_types, tool_info, **kwds)
+        return self.default_container_registry.resolve(enabled_container_types, tool_info, **kwds)
+
+    def _container_registry_for_destination(self, destination_info):
+        destination_id = destination_info.get("id")  # Probably not the way to get the ID?
+        destination_container_registry = None
+        if destination_id and destination_id not in self.destination_container_registeries:
+            if 'container_resolvers' in destination_info:
+                destination_container_registry = ContainerRegistry(self.app_info, destination_info=destination_info, mulled_resolution_cache=self.mulled_resolution_cache)
+                self.destination_container_registeries[destination_id] = destination_container_registry
+        elif not destination_id and 'container_resolvers' in destination_info:
+            destination_container_registry = ContainerRegistry(self.app_info, destination_info=destination_info, mulled_resolution_cache=self.mulled_resolution_cache)
+
+        if destination_container_registry is None and destination_id and destination_id in self.destination_container_registeries:
+            destination_container_registry = self.destination_container_registeries[destination_id]
+
+        return destination_container_registry or self.default_container_registry
 
     def find_container(self, tool_info, destination_info, job_info):
         enabled_container_types = self._enabled_container_types(destination_info)
@@ -102,7 +119,8 @@ class ContainerFinder:
                     return container
 
         # Otherwise lets see if we can find container for the tool.
-        container_description = self.find_best_container_description(enabled_container_types, tool_info)
+        container_registry = self._container_registry_for_destination(destination_info)
+        container_description = container_registry.find_best_container_description(enabled_container_types, tool_info)
         container = __destination_container(container_description)
         if container:
             return container
@@ -124,7 +142,7 @@ class ContainerFinder:
         return NULL_CONTAINER
 
     def resolution_cache(self):
-        return self.container_registry.get_resolution_cache()
+        return self.default_container_registry.get_resolution_cache()
 
     def __overridden_container_id(self, container_type, destination_info):
         if not self.__container_type_enabled(container_type, destination_info):
@@ -186,25 +204,33 @@ class NullContainerFinder:
 class ContainerRegistry:
     """Loop through enabled ContainerResolver plugins and find first match."""
 
-    def __init__(self, app_info, mulled_resolution_cache=None):
+    def __init__(self, app_info, destination_info=None, mulled_resolution_cache=None):
         self.resolver_classes = self.__resolvers_dict()
         self.enable_mulled_containers = app_info.enable_mulled_containers
         self.app_info = app_info
-        self.container_resolvers = self.__build_container_resolvers(app_info)
+        self.container_resolvers = self.__build_container_resolvers(app_info, destination_info)
         self.mulled_resolution_cache = mulled_resolution_cache
 
-    def __build_container_resolvers(self, app_info):
-        conf_file = getattr(app_info, 'container_resolvers_config_file', None)
-        conf_dict = getattr(app_info, 'container_resolvers_config_dict', None)
+    def __build_container_resolvers(self, app_info, destination_info):
+        app_conf_file = getattr(app_info, 'container_resolvers_config_file', None)
+        app_conf_dict = getattr(app_info, 'container_resolvers_config_dict', None)
+
+        if destination_info is not None:
+            conf_file = destination_info.get('container_resolvers_config_file', app_conf_file)
+            conf_dict = destination_info.get('container_resolvers', app_conf_dict)
+        else:
+            conf_file = app_conf_file
+            conf_dict = app_conf_dict
+
         plugin_source = None
-        if conf_file and not os.path.exists(conf_file):
+        if conf_dict:
+            log.debug("Loading container resolution config inline from Galaxy or job configuration file")
+            plugin_source = plugin_config.plugin_source_from_dict(conf_dict)
+        elif conf_file and not os.path.exists(conf_file):
             log.warning(f"Unable to find config file '{conf_file}'")
         elif conf_file:
             log.debug("Loading container resolution config from file '{conf_file}'")
             plugin_source = plugin_config.plugin_source_from_path(conf_file)
-        elif conf_dict:
-            log.debug("Loading container resolution config inline from Galaxy configuration file")
-            plugin_source = plugin_config.plugin_source_from_dict(conf_dict)
         if plugin_source:
             return self._parse_resolver_conf(plugin_source)
         return self.__default_container_resolvers()
diff --git a/test/functional/tools/mulled_example_broken_no_requirements_fallback.xml b/test/functional/tools/mulled_example_broken_no_requirements_fallback.xml
new file mode 100644
index 00000000000..f22d820c2c0
--- /dev/null
+++ b/test/functional/tools/mulled_example_broken_no_requirements_fallback.xml
@@ -0,0 +1,25 @@
+<tool id="mulled_example_broken_no_requirements_fallback" name="mulled_example_broken_no_requirements_fallback" version="0.1.0">
+    <!-- mulled_example_explicit without requirements... to test mapping with job conf inline container resolvers config -->
+    <!--
+    <requirements>
+        <requirement type="package" version="0.7.15">bwa</requirement>
+        <container type="docker">quay.io/biocontainers/bwa:0.7.15-\-0</container>
+    </requirements>
+    -->
+    <stdio>
+        <exit_code range="2:" />
+    </stdio>
+    <command><![CDATA[
+bwa > $output_1 2>&1
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+        <data name="output_1" />
+    </outputs>
+    <tests>
+    </tests>
+    <help><![CDATA[
+TODO: Fill in help.
+    ]]></help>
+</tool>
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index 4c27f1a0c8c..7c82aca504e 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -243,6 +243,7 @@
   <tool file="mulled_example_explicit.xml" />
   <tool file="mulled_example_invalid_case.xml" />
   <tool file="mulled_example_broken_no_requirements.xml" />
+  <tool file="mulled_example_broken_no_requirements_fallback.xml" />
 
   <tool file="simple_constructs.yml" />
 
diff --git a/test/integration/dockerized_job_conf.xml b/test/integration/dockerized_job_conf.xml
deleted file mode 100644
index d4cdce9f651..00000000000
--- a/test/integration/dockerized_job_conf.xml
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0"?>
-<job_conf>
-    <plugins>
-        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="1"/>
-    </plugins>
-
-    <destinations default="local_docker">
-        <destination id="local_docker" runner="local">
-            <param id="docker_enabled">true</param>
-            <param id="docker_sudo">false</param>
-        </destination>
-
-        <destination id="local_upload" runner="local">
-        </destination>
-
-    </destinations>
-
-    <tools>
-        <tool id="upload1" destination="local_upload" />
-    </tools>
-
-</job_conf>
diff --git a/test/integration/dockerized_job_conf.yml b/test/integration/dockerized_job_conf.yml
new file mode 100644
index 00000000000..dda06d8ae65
--- /dev/null
+++ b/test/integration/dockerized_job_conf.yml
@@ -0,0 +1,26 @@
+---
+runners:
+  local:
+    load: galaxy.jobs.runners.local:LocalJobRunner
+    workers: 1
+
+execution:
+  default: local_docker
+  environments:
+    local_docker:
+      runner: local
+      docker_enabled: true
+    local_docker_inline_container_resolvers:
+      runner: local
+      docker_enabled: true
+      container_resolvers:
+        - type: fallback
+          identifier: 'quay.io/biocontainers/bwa:0.7.15--0'
+    local_upload:
+      runner: local
+
+tools:
+  - id: upload1
+    environment: local_upload
+  - id: mulled_example_broken_no_requirements_fallback
+    environment: local_docker_inline_container_resolvers
diff --git a/test/integration/singularity_job_conf.xml b/test/integration/singularity_job_conf.xml
deleted file mode 100644
index 1a500dcab3f..00000000000
--- a/test/integration/singularity_job_conf.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<?xml version="1.0"?>
-<job_conf>
-    <plugins>
-        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="1"/>
-    </plugins>
-
-    <destinations default="local_singularity">
-        <destination id="local_singularity" runner="local">
-            <param id="singularity_enabled">true</param>
-        </destination>
-
-        <destination id="local_upload" runner="local">
-        </destination>
-
-    </destinations>
-
-    <tools>
-        <tool class="local" destination="local_upload" />
-    </tools>
-
-</job_conf>
diff --git a/test/integration/singularity_job_conf.yml b/test/integration/singularity_job_conf.yml
new file mode 100644
index 00000000000..470a4a9685d
--- /dev/null
+++ b/test/integration/singularity_job_conf.yml
@@ -0,0 +1,18 @@
+---
+runners:
+  local:
+    load: galaxy.jobs.runners.local:LocalJobRunner
+    workers: 1
+
+execution:
+  default: local_singularity
+  environments:
+    local_singularity:
+      runner: local
+      singularity_enabled: true
+    local_upload:
+      runner: local
+
+tools:
+  - id: upload1
+    environment: local_upload
diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py
index 3887650cf55..2981e0d1e3f 100644
--- a/test/integration/test_containerized_jobs.py
+++ b/test/integration/test_containerized_jobs.py
@@ -12,8 +12,8 @@ from galaxy_test.driver import integration_util
 from .test_job_environments import RunsEnvironmentJobs
 
 SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
-DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml")
-SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.xml")
+DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.yml")
+SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.yml")
 EXTENDED_TIMEOUT = 120
 
 
@@ -202,6 +202,36 @@ class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase):
         config["container_resolvers"] = container_resolvers_config
 
 
+class InlineJobEnvironmentContainerResolverTestCase(integration_util.IntegrationTestCase):
+
+    framework_tool_and_types = True
+    container_type = 'docker'
+    job_config_file = DOCKERIZED_JOB_CONFIG_FILE
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = cls._test_driver.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = cls.job_config_file
+        disable_dependency_resolution(config)
+
+    @classmethod
+    def setUpClass(cls):
+        skip_if_container_type_unavailable(cls)
+        super().setUpClass()
+
+    def setUp(self):
+        super().setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+        self.history_id = self.dataset_populator.new_history()
+
+    def test_inline_environment_container_resolver_configuration(self):
+        self.dataset_populator.run_tool("mulled_example_broken_no_requirements_fallback", {}, self.history_id)
+        self.dataset_populator.wait_for_history(self.history_id, assert_ok=True)
+        output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT)
+        assert "0.7.15-r1140" in output
+
+
 # Singularity 2.4 in the official Vagrant issue has some problems running this test
 # case by default because subdirectories of /tmp don't bind correctly. Overridding
 # TMPDIR can fix this.

From 19c469c9cda847b6b924d90c645a90edc4735ff8 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 17 Aug 2021 15:30:08 -0400
Subject: [PATCH 375/452] Fix up plugin_config module modifying configuration
 dictionaries.

---
 lib/galaxy/util/plugin_config.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py
index 66deafe4bcb..aec2a2a1856 100644
--- a/lib/galaxy/util/plugin_config.py
+++ b/lib/galaxy/util/plugin_config.py
@@ -89,7 +89,9 @@ def __load_plugins_from_dicts(plugins_dict, configs, extra_kwds, plugin_type_key
                 break
         assert plugin_type is not None, f"Could not determine plugin type for [{config}]"
         plugin_kwds = config
-        plugin_kwds.update(extra_kwds)
+        if extra_kwds:
+            plugin_kwds = plugin_kwds.copy()
+            plugin_kwds.update(extra_kwds)
         plugin = plugins_dict[plugin_type](**plugin_kwds)
         plugins.append(plugin)
 

From c4951e86abbe689d8bb2fe7b516e24e666a1ae47 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Tue, 17 Aug 2021 13:37:54 -0700
Subject: [PATCH 376/452] beta history, editing ui fixes

---
 client/package.json                           |   3 +-
 .../src/components/History/ContentFilters.vue |   2 +-
 .../ContentItem/CollectionContentItem.js      |   2 +-
 .../History/ContentItem/ContentItem.js        |  28 +-
 .../History/ContentItem/Dataset/Dataset.vue   |   2 +
 .../ContentItem/Dataset/DatasetMenu.vue       | 299 +++++++++---------
 .../History/ContentItem/Dataset/DatasetUI.vue | 186 +++++++----
 .../ContentItem/DatasetCollection/DscMenu.vue |  12 +-
 .../ContentItem/DatasetCollection/DscUI.vue   |  62 ++--
 .../History/ContentItem/Subcollection.vue     |   1 -
 .../History/ContentItem/Subdataset.vue        |  62 ++++
 .../components/History/ContentItem/index.js   |   3 -
 .../History/ContentItem/styles.scss           |  51 +--
 .../components/History/ContentOperations.vue  | 291 ++++++++++-------
 client/src/components/History/CopyModal.vue   |  13 +-
 .../History/CurrentCollection/Details.vue     | 105 ++++--
 .../History/CurrentCollection/Panel.vue       |  32 +-
 .../History/CurrentCollection/TopNav.vue      |  54 +---
 .../History/CurrentHistoryPanel.vue           |  36 +--
 client/src/components/History/EditorMenu.vue  |  73 +++++
 .../src/components/History/HistoriesMenu.vue  | 117 ++++---
 client/src/components/History/History.vue     |  21 +-
 .../src/components/History/HistoryDetails.vue | 272 ++++++----------
 client/src/components/History/HistoryMenu.vue | 119 +++++++
 .../History/HistorySelectorModal.vue          | 106 +++++++
 client/src/components/History/HistoryTags.vue |   3 +-
 client/src/components/History/Layout.vue      |  59 ++--
 client/src/components/History/Scroller.vue    |  78 +++--
 client/src/components/History/StatusIcon.vue  |  91 ++----
 .../History/caching/loadDscContent.js         |   4 +-
 .../History/caching/operators/chunk.js        |  42 ---
 .../src/components/History/model/Dataset.js   |  25 ++
 .../History/model/DatasetCollection.js        |  22 +-
 .../src/components/History/model/History.js   |  25 ++
 .../components/History/model/History.test.js  | 101 ++++++
 client/src/components/History/model/crud.js   |   4 +-
 .../src/components/History/model/queries.js   |  21 +-
 .../CollectionContentProvider.js              |   6 +-
 .../collectionPayload.js                      |   7 +-
 .../ContentProvider/aggregateCacheUpdates.js  |  14 +-
 .../ContentProvider/processContentStreams.js  |  10 +-
 .../History/providers/DatasetProvider.js      |   0
 .../History/providers/ExpandedItems.js        |  19 ++
 .../HistoryContentProvider/contentPayload.js  |  20 +-
 .../providers/UserHistories/UserHistories.js  |  10 +-
 .../components/History/providers/setCache.js  |  25 ++
 client/src/components/IconButton.vue          |  51 +++
 .../src/components/Workflow/WorkflowList.vue  |  10 +-
 client/src/components/plugins/icons.js        |  93 ++++++
 client/src/components/plugins/index.js        |   1 +
 client/src/utils/mountVueComponent.js         |   5 +-
 client/src/utils/observable/chunk.js          |  18 +-
 client/src/utils/safeAssign.js                |  19 ++
 client/tests/jest/helpers.js                  |   4 +-
 client/yarn.lock                              |  56 ++--
 55 files changed, 1814 insertions(+), 981 deletions(-)
 create mode 100644 client/src/components/History/ContentItem/Subdataset.vue
 create mode 100644 client/src/components/History/EditorMenu.vue
 create mode 100644 client/src/components/History/HistoryMenu.vue
 create mode 100644 client/src/components/History/HistorySelectorModal.vue
 delete mode 100644 client/src/components/History/caching/operators/chunk.js
 create mode 100644 client/src/components/History/model/History.test.js
 create mode 100644 client/src/components/History/providers/DatasetProvider.js
 create mode 100644 client/src/components/History/providers/setCache.js
 create mode 100644 client/src/components/IconButton.vue
 create mode 100644 client/src/components/plugins/icons.js

diff --git a/client/package.json b/client/package.json
index 0011ab19a15..2bc381f9237 100644
--- a/client/package.json
+++ b/client/package.json
@@ -29,7 +29,7 @@
     "axios": "^0.21.1",
     "backbone": "1.4.0",
     "bootstrap": "4.5.0",
-    "bootstrap-vue": "^2.15.0",
+    "bootstrap-vue": "^2.21.2",
     "citation-js": "^0.5.0-alpha.5",
     "d3": "3",
     "date-fns": "^2.21.1",
@@ -63,6 +63,7 @@
     "pouchdb-erase": "^1.0.2",
     "pouchdb-find": "^7.2.2",
     "pouchdb-upsert": "^2.2.0",
+    "pretty-bytes": "^5.6.0",
     "proper-skip-list": "^4.0.2",
     "pyre-to-regexp": "^0.0.5",
     "regression": "^2.0.1",
diff --git a/client/src/components/History/ContentFilters.vue b/client/src/components/History/ContentFilters.vue
index da1663d41b8..03c9536728f 100644
--- a/client/src/components/History/ContentFilters.vue
+++ b/client/src/components/History/ContentFilters.vue
@@ -1,7 +1,7 @@
 <template>
     <b-input-group v-if="params">
         <DebouncedInput v-model.trim="filterText" v-slot="{ value, input }">
-            <b-form-input size="sm" :value="value" @input="input" :placeholder="'Search Filter' | localize" />
+            <b-form-input size="sm" :value="value" @input="input" :placeholder="'Content Filter' | localize" />
         </DebouncedInput>
 
         <b-input-group-append>
diff --git a/client/src/components/History/ContentItem/CollectionContentItem.js b/client/src/components/History/ContentItem/CollectionContentItem.js
index efa92a9083a..eb785ff8d8e 100644
--- a/client/src/components/History/ContentItem/CollectionContentItem.js
+++ b/client/src/components/History/ContentItem/CollectionContentItem.js
@@ -8,7 +8,7 @@ export default {
             const { history_content_type } = this.item;
             switch (history_content_type) {
                 case "dataset":
-                    return "Dataset";
+                    return "Subdataset";
                 case "dataset_collection":
                     return "Subcollection";
                 default:
diff --git a/client/src/components/History/ContentItem/ContentItem.js b/client/src/components/History/ContentItem/ContentItem.js
index f17e129a5f3..b39fd514d4e 100644
--- a/client/src/components/History/ContentItem/ContentItem.js
+++ b/client/src/components/History/ContentItem/ContentItem.js
@@ -11,18 +11,25 @@
 import Placeholder from "./Placeholder";
 import Dataset from "./Dataset";
 import DatasetCollection from "./DatasetCollection";
-// import SubDataset from "./SubDataset";
+import Subdataset from "./Subdataset";
 import Subcollection from "./Subcollection";
+import "./styles.scss";
 
 export default {
     template: `
         <component :is="contentItemComponent"
-            class="content-item p-1"
+            :data-ci-type="contentItemComponent"
+            class="content-item"
             :class="{ loading }"
             :tabindex="index"
+            :writable="writable"
             v-on="$listeners"
-            v-bind="bindProps"
-            @mouseover.native.self.stop="setFocus(index)"
+            v-bind="$attrs"
+            :item="item"
+            :index="index"
+            :row-key="rowKey"
+            :writable="writable"
+            @mouseover.native.stop="setFocus(index)"
             @keydown.native.arrow-up.self.stop="setFocus(index - 1)"
             @keydown.native.arrow-down.self.stop="setFocus(index + 1)"
         />
@@ -32,13 +39,15 @@ export default {
         Placeholder,
         Dataset,
         DatasetCollection,
-        // SubDataset,
+        Subdataset,
         Subcollection,
     },
 
     props: {
         item: { type: Object, required: true },
-        index: { type: Number, required: true },
+        index: { type: Number, required: false, default: null },
+        rowKey: { type: [Number, String], required: false, default: "" },
+        writable: { type: Boolean, required: false, default: true },
     },
 
     data: () => ({
@@ -50,8 +59,8 @@ export default {
             if (this.suppressFocus) {
                 return;
             }
-            const ul = this.$el.closest(".scroller");
-            const el = ul.querySelector(`[tabindex="${index}"]`);
+            const scrollContainer = this.$el.closest(".scroller");
+            const el = scrollContainer.querySelector(`[tabindex="${index}"]`);
             if (el) {
                 el.focus();
             }
@@ -66,9 +75,6 @@ export default {
             // override me
             return "Placeholder";
         },
-        bindProps() {
-            return { ...this.$props, ...this.$attrs };
-        },
     },
 
     created() {
diff --git a/client/src/components/History/ContentItem/Dataset/Dataset.vue b/client/src/components/History/ContentItem/Dataset/Dataset.vue
index cc00d07b0d4..14b3c358276 100644
--- a/client/src/components/History/ContentItem/Dataset/Dataset.vue
+++ b/client/src/components/History/ContentItem/Dataset/Dataset.vue
@@ -4,6 +4,7 @@
         v-bind="$attrs"
         v-on="$listeners"
         :dataset="dataset"
+        :row-key="rowKey"
         @update:dataset="onUpdate"
         @delete="onDelete"
         @undelete="onUndelete"
@@ -26,6 +27,7 @@ export default {
     },
     props: {
         item: { type: Object, required: true },
+        rowKey: { type: [Number, String], required: false, default: "" },
     },
     computed: {
         dataset() {
diff --git a/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue b/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue
index 42225f34e82..c037ddbc56a 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue
@@ -1,159 +1,170 @@
 <template>
-    <PriorityMenu :starting-height="27">
-        <PriorityMenuItem
-            v-if="notIn(STATES.NOT_VIEWABLE, STATES.DISCARDED)"
-            key="view-dataset"
-            :title="displayButtonTitle"
-            :disabled="dataset.purged || isIn(STATES.UPLOAD, STATES.NEW)"
-            @click.stop="viewData"
-            icon="fas fa-eye"
-        />
-
-        <PriorityMenuItem
-            v-if="notIn(STATES.DISCARDED)"
-            key="edit-dataset"
-            :title="editButtonTitle"
-            :disabled="dataset.deleted || isIn(STATES.UPLOAD, STATES.NEW)"
-            @click.stop="
-                backboneRoute('datasets/edit', {
-                    dataset_id: dataset.id,
-                })
-            "
-            icon="fa fa-pencil"
-        />
-
-        <PriorityMenuItem
-            v-if="dataset.accessible"
-            key="delete-dataset"
-            :title="deleteButtonTitle"
-            :disabled="dataset.purged"
-            @click.stop="onDeleteClick"
-            :icon="dataset.deleted ? 'fas fa-trash-restore' : 'fas fa-trash'"
-        />
-
-        <PriorityMenuItem
-            v-if="expanded"
-            key="edit-tags"
-            title="Edit Dataset Tags"
-            :pressed="showTags"
-            @click.stop="$emit('update:showTags', !showTags)"
-            icon="fas fa-tags"
-        />
-
-        <PriorityMenuItem
-            v-if="isIn(STATES.ERROR)"
-            key="show-error"
-            title="Error"
-            @click.stop="
-                backboneRoute('datasets/error', {
-                    dataset_id: dataset.id,
-                })
-            "
-            icon="fa fa-bug"
-        />
-
-        <PriorityMenuItem
-            v-if="!(showDownloads && dataset.hasMetaData)"
-            key="download"
-            title="Download"
-            :href="prependPath(dataset.getUrl('download'))"
-            target="_blank"
-            download
-            icon="fas fa-file-download"
-        />
-
-        <PriorityMenuItem
-            v-if="showDownloads && dataset.hasMetaData"
-            key="download-metadata"
-            title="Download"
-            :href="prependPath(dataset.getUrl('download'))"
-            target="_blank"
-            download
-            icon="fas fa-file-download"
-        />
-
-        <PriorityMenuItem
-            v-if="!dataset.purged && dataset.getUrl('download')"
-            key="copy-link"
-            title="Copy Link"
-            icon="fa fa-chain"
-            @click.stop="$emit('copy-link')"
-        />
-
-        <div v-if="showDownloads && dataset.hasMetaData">
-            <PriorityMenuItem
-                v-for="mf in dataset.meta_files"
-                :key="'download-' + mf.download_url"
-                :title="'Download ' + mf.file_type"
-                :href="prependPath(dataset.getUrl('meta_download') + mf.file_type)"
-                target="_blank"
-                download
-                icon="fas fa-file-download"
+    <div>
+        <b-button-group>
+            <IconButton
+                v-if="notIn(STATES.NOT_VIEWABLE, STATES.DISCARDED)"
+                icon="eye"
+                :title="displayButtonTitle"
+                :disabled="dataset.purged || isIn(STATES.UPLOAD, STATES.NEW)"
+                @click.stop="viewData"
+                variant="link"
+                class="px-1"
+            />
+            <IconButton
+                v-if="notIn(STATES.DISCARDED)"
+                icon="pen"
+                :title="editButtonTitle"
+                :disabled="dataset.deleted || isIn(STATES.UPLOAD, STATES.NEW)"
+                @click.stop="backboneRoute('datasets/edit', { dataset_id: dataset.id })"
+                variant="link"
+                class="px-1"
+            />
+            <IconButton
+                v-if="dataset.accessible"
+                :icon="dataset.deleted ? 'trash-restore' : 'trash'"
+                :title="deleteButtonTitle"
+                :disabled="dataset.purged"
+                v-b-modal="bsId('delete-modal')"
+                variant="link"
+                class="px-1"
+                @click.stop
             />
-        </div>
 
-        <PriorityMenuItem
-            v-if="dataset.rerunnable && dataset.creating_job && notIn(STATES.UPLOAD, STATES.NOT_VIEWABLE)"
-            key="run-job"
-            title="Run job again"
-            :href="prependPath(dataset.getUrl('rerun'))"
-            @click.stop="
-                backboneRoute('/', {
-                    job_id: dataset.creating_job,
-                })
-            "
-            icon="fa fa-redo"
-        />
+            <b-dropdown
+                size="sm"
+                variant="link"
+                :text="'Dataset Operations' | l"
+                toggle-class="p-1 pl-2"
+                class="flex-grow-0"
+                v-if="expanded"
+                boundary="window"
+            >
+                <template v-slot:button-content>
+                    <Icon icon="ellipsis-v" variant="link" />
+                    <span class="sr-only">Dataset Operations</span>
+                </template>
 
-        <PriorityMenuItem
-            v-if="showViz && hasViz && isIn(STATES.OK, STATES.FAILED_METADATA)"
-            key="visualize"
-            title="Visualize Data"
-            @click.stop.prevent="visualize"
-            icon="fa-bar-chart"
-        />
+                <b-dropdown-item
+                    v-if="isIn(STATES.ERROR)"
+                    key="show-error"
+                    title="Error"
+                    @click.stop="
+                        backboneRoute('datasets/error', {
+                            dataset_id: dataset.id,
+                        })
+                    "
+                >
+                    <Icon icon="exclamation-triangle" class="mr-1" />
+                    <span v-localize>Show Error</span>
+                </b-dropdown-item>
 
-        <PriorityMenuItem
-            v-if="currentUser && currentUser.id && dataset.creating_job"
-            key="tool-tip"
-            title="Tool Help"
-            @click.stop="showToolHelp(dataset.creating_job)"
-            icon="fa fa-question"
-        />
+                <b-dropdown-item
+                    v-if="!dataset.purged && dataset.getUrl('download')"
+                    title="Copy Link"
+                    @click.stop="$emit('copy-link')"
+                >
+                    <Icon icon="link" class="mr-1" />
+                    <span v-localize>Copy Link</span>
+                </b-dropdown-item>
 
-        <PriorityMenuItem
-            v-if="notIn(STATES.NOT_VIEWABLE)"
-            key="dataset-details"
-            title="View Details"
-            @click.stop.prevent="
-                iframeAdd({
-                    path: dataset.getUrl('show_params'),
-                    title: 'Dataset details',
-                })
-            "
-            icon="fa fa-info-circle"
-        />
-    </PriorityMenu>
+                <b-dropdown-item
+                    v-if="!(showDownloads && dataset.hasMetaData)"
+                    title="Download"
+                    :href="prependPath(dataset.getUrl('download'))"
+                    target="_blank"
+                    download
+                >
+                    <Icon icon="download" class="mr-1" />
+                    <span v-localize>Download</span>
+                </b-dropdown-item>
+
+                <b-dropdown-group v-if="showDownloads && dataset.hasMetaData">
+                    <b-dropdown-item
+                        v-for="mf in dataset.meta_files"
+                        :key="'download-' + mf.download_url"
+                        :title="'Download ' + mf.file_type"
+                        :href="prependPath(dataset.getUrl('meta_download') + mf.file_type)"
+                        target="_blank"
+                        download
+                    >
+                        <Icon icon="download" class="mr-1" />
+                        <span v-localize>{{ "Download " + mf.file_type }}</span>
+                    </b-dropdown-item>
+                </b-dropdown-group>
+
+                <b-dropdown-item
+                    v-if="dataset.rerunnable && dataset.creating_job && notIn(STATES.UPLOAD, STATES.NOT_VIEWABLE)"
+                    title="Run job again"
+                    :href="prependPath(dataset.getUrl('rerun'))"
+                    @click.stop="
+                        backboneRoute('/', {
+                            job_id: dataset.creating_job,
+                        })
+                    "
+                >
+                    <Icon icon="play" class="mr-1" />
+                    <span v-localize>Run job again</span>
+                </b-dropdown-item>
+
+                <b-dropdown-item
+                    v-if="showViz && hasViz && isIn(STATES.OK, STATES.FAILED_METADATA)"
+                    title="Visualize Data"
+                    @click.stop.prevent="visualize"
+                >
+                    <Icon icon="chart-area" class="mr-1" />
+                    <span v-localize>Visualize Data</span>
+                </b-dropdown-item>
+
+                <b-dropdown-item
+                    v-if="currentUser && currentUser.id && dataset.creating_job"
+                    title="Tool Help"
+                    @click.stop="showToolHelp(dataset.creating_job)"
+                >
+                    <Icon icon="question" class="mr-1" />
+                    <span v-localize>Tool Help</span>
+                </b-dropdown-item>
+
+                <b-dropdown-item
+                    v-if="notIn(STATES.NOT_VIEWABLE)"
+                    key="dataset-details"
+                    title="View Dataset Details"
+                    @click.stop.prevent="
+                        iframeAdd({
+                            path: dataset.getUrl('show_params'),
+                            title: 'View Dataset Details',
+                        })
+                    "
+                >
+                    <Icon icon="info-circle" class="mr-1" />
+                    <span v-localize>View Dataset Details</span>
+                </b-dropdown-item>
+            </b-dropdown>
+        </b-button-group>
+
+        <!-- modals -->
+        <b-modal :id="bsId('delete-modal')" title="Delete Dataset?" title-tag="h2" @ok="onDeleteClick">
+            <p v-localize>Really delete this dataset?</p>
+        </b-modal>
+    </div>
 </template>
 
 <script>
 import { mapGetters } from "vuex";
-import { Dataset, STATES } from "../../model";
-import { PriorityMenu, PriorityMenuItem } from "components/PriorityMenu";
+import { Dataset } from "../../model";
 import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
+import IconButton from "components/IconButton";
 
 export default {
     mixins: [legacyNavigationMixin],
+    inject: ["STATES"],
 
     components: {
-        PriorityMenu,
-        PriorityMenuItem,
+        IconButton,
     },
 
     props: {
         dataset: { type: Dataset, required: true },
         expanded: { type: Boolean, required: true },
-        showTags: { type: Boolean, required: false, default: false },
     },
 
     data() {
@@ -163,11 +174,6 @@ export default {
         };
     },
 
-    created() {
-        // make available to template
-        this.STATES = STATES;
-    },
-
     computed: {
         ...mapGetters("user", ["currentUser"]),
         ...mapGetters("config", ["config"]),
@@ -180,10 +186,10 @@ export default {
             if (this.dataset.purged) {
                 return "Cannot display datasets removed from disk";
             }
-            if (this.dataset.state == STATES.UPLOAD) {
+            if (this.dataset.state == this.STATES.UPLOAD) {
                 return "This dataset must finish uploading before it can be viewed";
             }
-            if (this.dataset.state == STATES.NEW) {
+            if (this.dataset.state == this.STATES.NEW) {
                 return "This dataset is not yet viewable";
             }
             return "View data";
@@ -196,7 +202,7 @@ export default {
             if (this.dataset.purged) {
                 return "Cannot edit attributes of datasets removed from disk";
             }
-            const unreadyStates = new Set([STATES.UPLOAD, STATES.NEW]);
+            const unreadyStates = new Set([this.STATES.UPLOAD, this.STATES.NEW]);
             if (unreadyStates.has(this.dataset.state)) {
                 return "This dataset is not yet editable";
             }
@@ -218,7 +224,7 @@ export default {
             if (!this.dataset.hasData) {
                 return false;
             }
-            const okStates = new Set([STATES.OK, STATES.FAILED_METADATA, STATES.ERROR]);
+            const okStates = new Set([this.STATES.OK, this.STATES.FAILED_METADATA, this.STATES.ERROR]);
             return okStates.has(this.dataset.state);
         },
 
@@ -278,6 +284,11 @@ export default {
             const eventName = this.dataset.deleted ? "undelete" : "delete";
             this.$emit(eventName);
         },
+
+        // bootstrap demands ids instead of simply using a ref so I guess we're doing this
+        bsId(suffix) {
+            return `dataset-${this.dataset.id}-${suffix}`;
+        },
     },
 };
 </script>
diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 59fe2466f95..2b15be928e9 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -8,7 +8,7 @@ either through the props, and make updates through the events -->
 <template>
     <div
         class="dataset history-content"
-        :id="typedId"
+        :data-content-type-id="typeId"
         :class="{ expanded, collapsed, selected }"
         :data-state="dataset.state"
         @keydown.arrow-left.self.stop="$emit('update:expanded', false)"
@@ -16,73 +16,101 @@ either through the props, and make updates through the events -->
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
         <!-- name, state buttons, menus -->
-        <nav class="content-top-menu" @click.stop="$emit('update:expanded', !expanded)">
-            <div class="content-status-indicators" @click.stop>
-                <b-check
-                    class="selector"
-                    v-if="showSelection"
-                    :checked="selected"
-                    @change="$emit('update:selected', $event)"
+        <nav class="p-1 d-flex align-items-center cursor-pointer" @click.stop="$emit('update:expanded', !expanded)">
+            <div class="d-flex align-items-center flex-grow-1 overflow-hidden">
+                <div class="pl-1" v-if="showSelectionBox">
+                    <b-check class="selector" :checked="selected" @change="$emit('update:selected', $event)"></b-check>
+                </div>
+
+                <StatusIcon
+                    v-if="!collapsed"
+                    class="status-icon px-1"
+                    :state="dataset.state"
+                    @click.stop="onStatusClick"
                 />
-                <StatusIcon v-if="!ok" class="status-icon px-1" :state="dataset.state" @click.stop="onStatusClick" />
-                <StateBtn
-                    v-if="!dataset.visible"
+
+                <IconButton
+                    v-if="!collapsed && !dataset.visible"
                     class="px-1"
                     state="hidden"
                     title="Unhide"
-                    icon="fa fa-eye-slash"
+                    icon="eye-slash"
                     @click.stop="$emit('unhide')"
                 />
-                <StateBtn
-                    v-if="dataset.isDeleted && !dataset.purged"
+
+                <IconButton
+                    v-if="!collapsed && dataset.isDeleted && !dataset.purged"
                     class="px-1"
                     state="deleted"
                     title="Undelete"
-                    icon="fas fa-trash-restore"
+                    icon="trash-restore"
                     @click.stop="$emit('undelete')"
                 />
+
+                <div class="content-title p-1 overflow-hidden">
+                    <h5 class="text-truncate" v-if="collapsed">
+                        <!-- <span class="hid">{{ dataset.hid }}</span> -->
+                        <span class="name">{{ dataset.title }}</span>
+                    </h5>
+                </div>
             </div>
 
-            <div class="content-title">
-                <h5 class="text-truncate">
-                    <span class="hid">{{ dataset.hid }}</span>
-                    <span v-if="collapsed || !dataset.canEditName" class="name">{{ dataset.title }}</span>
-                </h5>
-            </div>
-
-            <div class="content-item-menu">
+            <div class="d-flex">
                 <slot name="menu">
-                    <DatasetMenu :dataset="dataset" :expanded="expanded" v-on="$listeners" :show-tags.sync="showTags" />
+                    <DatasetMenu :dataset="dataset" :expanded="expanded" v-on="$listeners" />
                 </slot>
             </div>
         </nav>
 
-        <!--- read-only tags with name: prefix -->
-        <div v-if="collapsed && dataset.nameTags.length" class="nametags p-1">
+        <!--- read-only tags with name: prefix
+        <div v-if="collapsed && dataset.nameTags.length" class="nametags p-2">
             <Nametag v-for="tag in dataset.nameTags" :key="tag" :tag="tag" />
-        </div>
+        </div> -->
 
-        <!-- expanded view with editors -->
-        <header v-if="expanded" class="p-2 details">
-            <ClickToEdit
-                v-if="dataset.canEditName"
-                tag-name="h4"
-                :value="dataset.name"
-                @input="$emit('update:dataset', { name: $event })"
-                :display-label="dataset.title"
-                :tooltip-title="'Edit dataset name...' | localize"
-                tooltip-placement="left"
-            />
+        <div v-if="expanded" class="p-3 details">
+            <div class="d-flex">
+                <div class="flex-grow-1">
+                    <h4>{{ name || "(Dataset Name)" }}</h4>
 
-            <Annotation
-                class="mt-1"
-                :value="dataset.annotation"
-                @input="$emit('update:dataset', { annotation: $event })"
-            />
+                    <template v-if="!editing">
+                        <p v-if="annotation">{{ annotation }}</p>
+                        <div v-if="dataset.nameTags.length" class="nametags mt-2">
+                            <Nametag v-for="tag in dataset.nameTags" :key="tag" :tag="tag" />
+                        </div>
+                    </template>
 
-            <ContentTags v-if="showTags" class="mt-2" :content="dataset" />
-            <div v-else-if="dataset.nameTags.length" class="nametags mt-2">
-                <Nametag v-for="tag in dataset.nameTags" :key="tag" :tag="tag" />
+                    <div v-else class="my-3">
+                        <b-textarea
+                            class="mb-3"
+                            v-model="name"
+                            placeholder="Dataset Name"
+                            trim
+                            max-rows="4"
+                        ></b-textarea>
+
+                        <b-textarea
+                            class="mb-3"
+                            v-model="annotation"
+                            placeholder="Annotation (optional)"
+                            trim
+                            max-rows="4"
+                        ></b-textarea>
+
+                        <ContentTags class="mt-2" :content="dataset" />
+                    </div>
+                </div>
+
+                <EditorMenu
+                    class="ml-3 flex-grow-0 d-flex flex-column"
+                    v-if="writable"
+                    model-name="Collection"
+                    :editing.sync="editing"
+                    :writable="writable"
+                    :valid="valid"
+                    :dirty="dirty"
+                    @save="save"
+                    @revert="revert"
+                />
             </div>
 
             <div class="details">
@@ -102,41 +130,44 @@ either through the props, and make updates through the events -->
                 </div>
                 <pre v-if="dataset.peek" class="dataset-peek p-1" v-html="dataset.peek"></pre>
             </div>
-        </header>
+        </div>
     </div>
 </template>
 
 <script>
 import { Dataset, STATES } from "../../model";
-import ClickToEdit from "components/ClickToEdit";
-import Annotation from "components/Annotation";
 import { Nametag } from "components/Nametags";
-import { StatusIcon, StateBtn } from "../../StatusIcon";
+import StatusIcon from "../../StatusIcon";
 import DatasetMenu from "./DatasetMenu";
 import DatasetSummary from "./Summary";
 import ContentTags from "../../ContentTags";
 import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
+import IconButton from "components/IconButton";
+import EditorMenu from "../../EditorMenu";
+
 export default {
     mixins: [legacyNavigationMixin],
     components: {
-        ClickToEdit,
-        Annotation,
         StatusIcon,
-        StateBtn,
         DatasetMenu,
         DatasetSummary,
         ContentTags,
         Nametag,
+        IconButton,
+        EditorMenu,
     },
     props: {
         dataset: { type: Dataset, required: true },
         expanded: { type: Boolean, required: true },
         selected: { type: Boolean, required: false, default: false },
         showSelection: { type: Boolean, required: false, default: false },
+        writable: { type: Boolean, required: false, default: true },
+        selectable: { type: Boolean, required: false, default: true },
     },
     data() {
         return {
-            showTags: false,
+            editing: false,
+            tempContent: this.dataset.clone(),
         };
     },
     computed: {
@@ -146,11 +177,33 @@ export default {
         typeId() {
             return this.dataset.type_id;
         },
-        ok() {
-            return this.dataset.state == "ok";
+        isEditing() {
+            return this.writable && this.editing;
         },
-        typedId() {
-            return `dataset-${this.dataset.id}`;
+        valid() {
+            return true;
+        },
+        dirty() {
+            return !this.tempContent.equals(this.dataset);
+        },
+        name: {
+            get() {
+                return this.tempContent?.name || "";
+            },
+            set(newVal) {
+                this.patchTempContent({ name: newVal });
+            },
+        },
+        annotation: {
+            get() {
+                return this.tempContent?.annotation || "";
+            },
+            set(newVal) {
+                this.patchTempContent({ annotation: newVal });
+            },
+        },
+        showSelectionBox() {
+            return this.selectable && this.showSelection;
         },
     },
     methods: {
@@ -163,6 +216,23 @@ export default {
                     console.log("unhandled status icon click", this.dataset.state);
             }
         },
+        patchTempContent(props = {}) {
+            this.tempContent = this.tempContent.patch(props);
+        },
+        save() {
+            if (this.valid) {
+                this.$emit("update:dataset", this.tempContent);
+                this.editing = false;
+            }
+        },
+        reset() {
+            this.tempContent = this.dataset.clone();
+        },
+        revert() {
+            if (this.dirty) {
+                this.reset();
+            }
+        },
     },
 };
 </script>
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue b/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue
index fbe90484749..1f64d1e503c 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue
@@ -1,18 +1,20 @@
 <template>
-    <b-dropdown no-caret right variant="link" size="sm" boundary="window" toggle-class="p-1">
+    <b-dropdown no-caret right variant="link" size="sm" boundary="window" toggle-class="p-1 pl-3">
         <template v-slot:button-content>
-            <i class="fas fa-trash" />
+            <Icon icon="trash" variant="link" />
             <span class="sr-only">Delete Collection</span>
         </template>
 
-        <b-dropdown-item @click.stop="$emit('delete')"> Delete Collection Only </b-dropdown-item>
+        <b-dropdown-item @click.stop="$emit('delete')">
+            <span v-localize>Delete Collection Only </span>
+        </b-dropdown-item>
 
         <b-dropdown-item @click.stop="$emit('delete', { recursive: true })">
-            Delete Contained Datasets
+            <span v-localize>Delete Contained Datasets</span>
         </b-dropdown-item>
 
         <b-dropdown-item @click.stop="$emit('delete', { recursive: true, purge: true })">
-            Purge Contained Datasets
+            <span v-localize>Purge Contained Datasets</span>
         </b-dropdown-item>
     </b-dropdown>
 </template>
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index 1795656350f..8c712e6176a 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -9,16 +9,17 @@
         :data-state="dsc.state"
         @keydown.arrow-right.self.stop="$emit('viewCollection')"
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
-        @click.stop="$emit('viewCollection')"
     >
-        <nav class="content-top-menu">
-            <div class="content-status-indicators mr-1" @click.stop>
-                <b-check
-                    class="selector"
-                    v-if="showSelection"
-                    :checked="selected"
-                    @change="$emit('update:selected', $event)"
-                />
+        <nav class="p-1 d-flex align-items-center cursor-pointer">
+            <div class="d-flex align-items-center flex-grow-1 overflow-hidden">
+                <div class="pl-1" v-if="showSelection">
+                    <b-check
+                        class="selector"
+                        :checked="selected"
+                        @click.stop
+                        @change="$emit('update:selected', $event)"
+                    />
+                </div>
 
                 <StatusIcon
                     v-if="dsc.state != 'ok'"
@@ -27,44 +28,45 @@
                     @click.stop="onStatusClick"
                 />
 
-                <StateBtn
+                <IconButton
                     v-if="!dsc.visible"
                     class="px-1"
                     state="hidden"
                     title="Unhide"
-                    icon="fa fa-eye-slash"
+                    icon="eye-slash"
                     @click.stop="$emit('unhide')"
                 />
 
-                <StateBtn
+                <IconButton
                     v-if="dsc.deleted"
                     class="px-1"
                     state="deleted"
                     title="Undelete"
-                    icon="fas fa-trash-restore"
+                    icon="trash-restore"
                     @click.stop="$emit('undelete')"
                 />
 
-                <StateBtn
+                <IconButton
                     class="px-1"
                     state="ok"
                     title="Collection"
-                    icon="fas fa-folder"
+                    icon="folder"
                     @click.stop="$emit('viewCollection')"
+                    variant="link"
                 />
+
+                <div class="content-title flex-grow-1 overflow-hidden" @click.stop="$emit('viewCollection')">
+                    <h5 class="text-truncate">
+                        <!-- <span class="hid">{{ dsc.hid }}</span> -->
+                        <span class="name">{{ dsc.name }}</span>
+                        <span class="description">
+                            ({{ dsc.collectionType | localize }} {{ dsc.collectionCountDescription | localize }})
+                        </span>
+                    </h5>
+                </div>
             </div>
 
-            <div class="content-title mr-1">
-                <h5 class="text-truncate">
-                    <span class="hid">{{ dsc.hid }}</span>
-                    <span class="name">{{ dsc.name }}</span>
-                    <span class="description">
-                        ({{ dsc.collectionType | localize }} {{ dsc.collectionCountDescription | localize }})
-                    </span>
-                </h5>
-            </div>
-
-            <div class="content-item-menu">
+            <div class="d-flex" v-if="writable">
                 <slot name="menu">
                     <DscMenu v-if="!dsc.deleted" v-on="$listeners" />
                 </slot>
@@ -72,7 +74,7 @@
         </nav>
 
         <!--- read-only tags with name: prefix -->
-        <div v-if="dsc.nameTags.length" class="nametags p-1">
+        <div v-if="dsc.nameTags.length" class="nametags p-2">
             <Nametag v-for="tag in dsc.nameTags" :key="tag" :tag="tag" />
         </div>
 
@@ -82,23 +84,25 @@
 
 <script>
 import { DatasetCollection } from "../../model/DatasetCollection";
-import { StatusIcon, StateBtn } from "../../StatusIcon";
+import StatusIcon from "../../StatusIcon";
 import JobStateProgress from "./JobStateProgress";
 import DscMenu from "./DscMenu";
 import { Nametag } from "components/Nametags";
+import IconButton from "components/IconButton";
 
 export default {
     components: {
         StatusIcon,
-        StateBtn,
         JobStateProgress,
         DscMenu,
         Nametag,
+        IconButton,
     },
     props: {
         dsc: { type: DatasetCollection, required: true },
         selected: { type: Boolean, required: false, default: false },
         showSelection: { type: Boolean, required: false, default: false },
+        writable: { type: Boolean, required: false, default: true },
     },
     methods: {
         onStatusClick() {
diff --git a/client/src/components/History/ContentItem/Subcollection.vue b/client/src/components/History/ContentItem/Subcollection.vue
index d17a8f6b5a0..45b4a7e8f78 100644
--- a/client/src/components/History/ContentItem/Subcollection.vue
+++ b/client/src/components/History/ContentItem/Subcollection.vue
@@ -28,7 +28,6 @@ import { STATES } from "../model";
 export default {
     props: {
         item: { type: Object, required: true },
-        index: { type: Number, required: true },
     },
     computed: {
         dsc() {
diff --git a/client/src/components/History/ContentItem/Subdataset.vue b/client/src/components/History/ContentItem/Subdataset.vue
new file mode 100644
index 00000000000..010dfa38315
--- /dev/null
+++ b/client/src/components/History/ContentItem/Subdataset.vue
@@ -0,0 +1,62 @@
+<!-- a read-only dataset inside a collection. The collection query does not return enough
+information to properly render the dataset, so what we do is look it up on expand, checking the
+cache first to see if we already have the data -->
+
+<template>
+    <DatasetUI
+        v-if="dataset"
+        v-bind="$attrs"
+        v-on="$listeners"
+        :dataset="dataset"
+        :writable="false"
+        :selectable="false"
+        :row-key="rowKey"
+        @update:expanded="onExpand"
+    />
+</template>
+
+<script>
+import { Dataset } from "../model";
+import { getContentByTypeId, cacheContent } from "../caching";
+import { getContentDetails } from "../model/queries";
+import DatasetUI from "./Dataset/DatasetUI";
+
+export default {
+    components: {
+        DatasetUI,
+    },
+    props: {
+        item: { type: Object, required: true },
+    },
+    data() {
+        return {
+            localItem: { ...this.item },
+            loading: false,
+        };
+    },
+    computed: {
+        dataset() {
+            return new Dataset(this.localItem);
+        },
+    },
+    methods: {
+        onExpand(val) {
+            if (val) {
+                this.loadDetails();
+            }
+            this.$emit("update:expand", val);
+        },
+        async loadDetails() {
+            const { type_id, history_id } = this.item;
+            const existingDataset = await getContentByTypeId(history_id, type_id);
+            if (existingDataset) {
+                this.localItem = existingDataset;
+            } else {
+                const loadedDataset = await getContentDetails(this.item);
+                await cacheContent(loadedDataset);
+                this.localItem = loadedDataset;
+            }
+        },
+    },
+};
+</script>
diff --git a/client/src/components/History/ContentItem/index.js b/client/src/components/History/ContentItem/index.js
index 32d866ab9d0..5543ce9497a 100644
--- a/client/src/components/History/ContentItem/index.js
+++ b/client/src/components/History/ContentItem/index.js
@@ -1,9 +1,6 @@
 /**
  * Scrolling components for history contents and collection contents.
  */
-
-import "./styles.scss";
-
 export { default as ContentItem } from "./ContentItem";
 export { default as CollectionContentItem } from "./CollectionContentItem";
 export { default as HistoryContentItem } from "./HistoryContentItem";
diff --git a/client/src/components/History/ContentItem/styles.scss b/client/src/components/History/ContentItem/styles.scss
index f5384b3e8f8..03fcc5f41dd 100644
--- a/client/src/components/History/ContentItem/styles.scss
+++ b/client/src/components/History/ContentItem/styles.scss
@@ -3,7 +3,7 @@
 
 .content-item {
     outline: none;
-    border-bottom: 1px solid white;
+    border-bottom: 2px solid white;
     &:focus-within {
         box-shadow: inset 0 0 1em 0.25em rgba(255, 255, 255, 0.25);
     }
@@ -11,36 +11,9 @@
 
 // Nav across the top of the content item
 .content-item .content-top-menu {
-    /* a fixed-height is important for proper
-    functioning of the PriorityMenuItem dropdown since
-    it relies on flexbox and overflow */
     min-height: 2em;
 
-    cursor: pointer;
-
-    // flex parent
-    display: flex;
-    align-items: center;
-    justify-content: space-between;
-
-    // error, clock, etc.
-    .content-status-indicators {
-        // d-flex mr-1 align-items-center
-        display: flex;
-        align-items: center;
-    
-        &:blank {
-            display: none;
-        }
-    }
-
-    // name & labelling
     .content-title {
-
-        // flex child
-        flex-grow: 1;
-        overflow: hidden;
-
         .hid {
             color: adjust-color($text-color, $alpha: -0.6);
             font-size: 80%;
@@ -56,24 +29,4 @@
             font-weight: 700;
         }
     }
-
-    // priority menu on right side
-    .content-item-menu {
-        display: flex;
-        justify-content: flex-end;
-    }
-
-}
-
-// Setting a max-width here limits the number of icons that
-// will show. The priority menu is css awayre
-
-.content-item.collapsed .content-item-menu {
-    max-width: 90px;
-}
-
-// .content-menu button:hover,
-// .content-menu .show > button.dropdown-toggle {
-//     background-color: transparent;
-//     border-color: transparent;
-// }
+}
\ No newline at end of file
diff --git a/client/src/components/History/ContentOperations.vue b/client/src/components/History/ContentOperations.vue
index 3b3bd01cff4..33abb59142f 100644
--- a/client/src/components/History/ContentOperations.vue
+++ b/client/src/components/History/ContentOperations.vue
@@ -1,136 +1,168 @@
 <template>
     <section>
-        <header class="d-flex align-items-center justify-content-between">
-            <h6 class="text-nowrap mr-3">
-                <span v-localize>Matches</span>
-                <a href="#" @click="showFilter = !showFilter">
-                    <span>{{ totalMatches }}</span>
-                </a>
-            </h6>
-
-            <PriorityMenu style="max-width: 50%" :starting-height="27">
-                <PriorityMenuItem
-                    key="filter"
+        <nav class="content-operations-menu d-flex justify-content-between bg-secondary">
+            <b-button-group>
+                <IconButton
+                    :title="hasSelection ? 'Select All Contents' : 'Unselect All Contents'"
+                    icon="check-square"
+                    :disabled="!totalMatches"
+                    :pressed="hasSelection"
+                    @click="toggleSelectAll"
+                />
+                <IconButton
                     title="Filter History Content"
-                    icon="fa fa-filter"
-                    @click="showFilter = !showFilter"
+                    icon="filter"
+                    :disabled="!totalMatches"
                     :pressed="showFilter"
+                    @click="showFilter = !showFilter"
                 />
-
-                <PriorityMenuItem
-                    key="selection"
-                    title="Operations on multiple datasets"
-                    class="show-history-content-selectors-btn"
-                    icon="fas fa-check-square"
-                    @click="$emit('update:show-selection', !showSelection)"
-                    :pressed="showSelection"
+                <IconButton
+                    title="Collapse Expanded Items"
+                    icon="compress"
+                    :disabled="!expandedCount"
+                    @click="$emit('collapseAllContent')"
                 />
+            </b-button-group>
 
-                <PriorityMenuItem
-                    key="copy-datasets"
-                    class="copy-datasets-menu-item"
-                    title="Copy Datasets"
-                    icon="fas fa-copy"
-                    @click="iframeRedirect('/dataset/copy_datasets')"
-                />
-
-                <PriorityMenuItem
-                    key="collapse-expanded"
-                    title="Collapse Expanded Datasets"
-                    icon="fas fa-compress-alt"
-                    @click="emit('collapseAllContent')"
-                />
-
-                <PriorityMenuItem
-                    key="show-hidden-datasets"
-                    title="Unhide All Hidden Datasets"
-                    icon="fas fa-eye"
-                    @click="$bvModal.show('show-hidden-content')"
-                />
-
-                <PriorityMenuItem
-                    key="delete-hidden"
-                    title="Delete Hidden Datasets"
-                    icon="fas fa-trash"
-                    @click="$bvModal.show('delete-hidden-content')"
-                />
-
-                <PriorityMenuItem
-                    key="purge-hidden"
-                    title="Purge Hidden Datasets"
-                    icon="fas fa-burn"
-                    @click="$bvModal.show('purge-deleted-content')"
-                />
-            </PriorityMenu>
-        </header>
-
-        <transition name="shutterfade">
-            <content-filters v-if="showContentFilters" class="content-filters mt-1" :params.sync="localParams" />
-        </transition>
-
-        <transition name="shutterfade">
-            <b-button-toolbar
-                v-if="showSelection"
-                class="content-selection justify-content-between mt-1 list-action-menu"
-            >
-                <b-button-group>
-                    <b-button size="sm" @click="$emit('selectAllContent')">
-                        {{ "Select All" | localize }}
-                    </b-button>
-                    <b-button size="sm" @click="$emit('resetSelection')">
-                        {{ "Unselect All" | localize }}
-                    </b-button>
-                </b-button-group>
-
+            <b-button-group>
                 <b-dropdown
-                    class="ml-auto history-contents-list-action-menu-btn"
+                    class="history-contents-list-action-menu-btn"
                     size="sm"
-                    text="With Selected"
+                    text="Selection"
                     :disabled="!hasSelection"
                 >
-                    <b-dropdown-item @click="hideSelected">
-                        {{ "Hide Datasets" | localize }}
+                    <b-dropdown-text id="history-op-selected-content">
+                        <span v-localize v-if="hasSelection">With {{ numSelected }} selected items...</span>
+                        <span v-localize v-else>You don't have any selected content.</span>
+                    </b-dropdown-text>
+
+                    <b-dropdown-item aria-describedby="history-op-selected-content" v-b-modal:hide-selected-content>
+                        <span v-localize>Hide</span>
                     </b-dropdown-item>
-                    <b-dropdown-item @click="unhideSelected">
-                        {{ "Unhide Datasets" | localize }}
+
+                    <b-dropdown-item aria-describedby="history-op-selected-content" v-b-modal:show-selected-content>
+                        <span v-localize>Unhide</span>
                     </b-dropdown-item>
-                    <b-dropdown-item @click="deleteSelected">
-                        {{ "Delete Datasets" | localize }}
+
+                    <b-dropdown-item aria-describedby="history-op-selected-content" v-b-modal:delete-selected-content>
+                        <span v-localize>Delete</span>
                     </b-dropdown-item>
-                    <b-dropdown-item @click="undeleteSelected">
-                        {{ "Undelete Datasets" | localize }}
+
+                    <b-dropdown-item aria-describedby="history-op-selected-content" v-b-modal:restore-selected-content>
+                        <span v-localize>Undelete</span>
                     </b-dropdown-item>
-                    <b-dropdown-item @click="purgeSelected">
-                        {{ "Permanently Delete Datasets" | localize }}
-                    </b-dropdown-item>
-                    <b-dropdown-item @click="buildDatasetList">
-                        {{ "Build Dataset List" | localize }}
-                    </b-dropdown-item>
-                    <b-dropdown-item @click="buildDatasetPair">
-                        {{ "Build Dataset Pair" | localize }}
-                    </b-dropdown-item>
-                    <b-dropdown-item @click="buildListOfPairs">
-                        {{ "Build List of Dataset Pairs" | localize }}
-                    </b-dropdown-item>
-                    <b-dropdown-item @click="buildCollectionFromRules">
-                        {{ "Build Collection from Rules" | localize }}
+
+                    <b-dropdown-item
+                        aria-describedby="history-op-selected-content"
+                        v-b-modal:purge-selected-content
+                        :disabled="!hasSelection"
+                    >
+                        <span v-localize>Permanently Delete</span>
+                        <span v-if="numSelected">{{ numSelected }} items</span>
+                        <span v-else>(Disabled)</span>
                     </b-dropdown-item>
                 </b-dropdown>
-            </b-button-toolbar>
+
+                <b-dropdown
+                    class="history-contents-list-action-menu-btn"
+                    size="sm"
+                    text="History"
+                    :disabled="!totalMatches"
+                >
+                    <b-dropdown-text id="history-op-all-content">
+                        <span v-localize>With entire history...</span>
+                    </b-dropdown-text>
+
+                    <b-dropdown-item v-b-modal:show-all-hidden-content aria-describedby="history-op-all-content">
+                        <span v-localize>Unhide All Hidden Content</span>
+                    </b-dropdown-item>
+
+                    <b-dropdown-item v-b-modal:delete-all-hidden-content aria-describedby="history-op-all-content">
+                        <span v-localize>Delete All Hidden Content</span>
+                    </b-dropdown-item>
+
+                    <b-dropdown-item v-b-modal:purge-all-deleted-content aria-describedby="history-op-all-content">
+                        <span v-localize>Purge All Hidden Content</span>
+                    </b-dropdown-item>
+                </b-dropdown>
+
+                <b-dropdown
+                    class="history-contents-list-action-menu-btn"
+                    size="sm"
+                    text="Create New Content"
+                    :disabled="!totalMatches"
+                >
+                    <template v-slot:button-content>
+                        <Icon icon="plus" />
+                        <span class="sr-only" v-localize>Create New Content</span>
+                    </template>
+
+                    <b-dropdown-text id="history-op-new-content">
+                        <span>Create new content</span>
+                    </b-dropdown-text>
+
+                    <b-dropdown-item
+                        aria-describedby="history-op-new-content"
+                        @click="iframeRedirect('/dataset/copy_datasets')"
+                    >
+                        <span v-localize>Copy Datasets</span>
+                    </b-dropdown-item>
+
+                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildDatasetList">
+                        <span v-localize>Build Dataset List</span>
+                    </b-dropdown-item>
+
+                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildDatasetPair">
+                        <span v-localize>Build Dataset Pair</span>
+                    </b-dropdown-item>
+
+                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildListOfPairs">
+                        <span v-localize>Build List of Dataset Pairs</span>
+                    </b-dropdown-item>
+
+                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildCollectionFromRules">
+                        <span v-localize>Build Collection from Rules</span>
+                    </b-dropdown-item>
+                </b-dropdown>
+            </b-button-group>
+        </nav>
+
+        <transition name="shutterfade">
+            <content-filters v-if="showContentFilters" class="content-filters p-2" :params.sync="localParams" />
         </transition>
 
         <!-- #region Menus and popovers -->
 
-        <b-modal id="show-hidden-content" title="Show Hidden Datasets" title-tag="h2" @ok="unhideAll">
-            <p>{{ "Really unhide all hidden datasets?" | localize }}</p>
+        <b-modal id="hide-selected-content" title="Hide Selected Content?" title-tag="h2" @ok="hideSelected">
+            <p v-localize>Really hide {{ numSelected }} content items?</p>
         </b-modal>
 
-        <b-modal id="delete-hidden-content" title="Delete Hidden Datasets" title-tag="h2" @ok="deleteAllHidden">
-            <p>{{ "Really delete all hidden datasets?" | localize }}</p>
+        <b-modal id="show-selected-content" title="Show Selected Content?" title-tag="h2" @ok="unhideSelected">
+            <p v-localize>Really show {{ numSelected }} content items?</p>
         </b-modal>
 
-        <b-modal id="purge-deleted-content" title="Purge Deleted Datasets" title-tag="h2" @ok="purgeAllDeleted">
-            <p>{{ "Really delete all deleted datasets permanently? This cannot be undone." | localize }}</p>
+        <b-modal id="delete-selected-content" title="Delete Selected Content?" title-tag="h2" @ok="deleteSelected">
+            <p v-localize>Really delete {{ numSelected }} content items?</p>
+        </b-modal>
+
+        <b-modal id="restore-selected-content" title="Restore Selected Content?" title-tag="h2" @ok="undeleteSelected">
+            <p v-localize>Really restore {{ numSelected }} content items?</p>
+        </b-modal>
+
+        <b-modal id="purge-selected-content" title="Purge Selected Content?" title-tag="h2" @ok="purgeSelected">
+            <p v-localize>Permanently delete {{ numSelected }} content items? This cannot be undone.</p>
+        </b-modal>
+
+        <b-modal id="show-all-hidden-content" title="Show Hidden Datasets" title-tag="h2" @ok="unhideAll">
+            <p v-localize>Really unhide all hidden datasets?</p>
+        </b-modal>
+
+        <b-modal id="delete-all-hidden-content" title="Delete Hidden Datasets" title-tag="h2" @ok="deleteAllHidden">
+            <p v-localize>Really delete all hidden datasets?</p>
+        </b-modal>
+
+        <b-modal id="purge-all-deleted-content" title="Purge Deleted Datasets" title-tag="h2" @ok="purgeAllDeleted">
+            <p v-localize>"Really delete all deleted datasets permanently? This cannot be undone.</p>
         </b-modal>
 
         <!-- #endregion -->
@@ -152,18 +184,16 @@ import {
 } from "./model";
 import { createDatasetCollection } from "./model/queries";
 import { cacheContent } from "./caching";
-
 import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
-import ContentFilters from "./ContentFilters";
-import { PriorityMenu, PriorityMenuItem } from "components/PriorityMenu";
 import { buildCollectionModal } from "./adapters/buildCollectionModal";
+import ContentFilters from "./ContentFilters";
+import IconButton from "components/IconButton";
 
 export default {
     mixins: [legacyNavigationMixin],
     components: {
         ContentFilters,
-        PriorityMenu,
-        PriorityMenuItem,
+        IconButton,
     },
     props: {
         history: { type: History, required: true },
@@ -171,6 +201,8 @@ export default {
         contentSelection: { type: Map, required: true },
         showSelection: { type: Boolean, required: true },
         totalMatches: { type: Number, required: true },
+        expandedCount: { type: Number, required: false, default: 0 },
+        debug: { type: Boolean, default: false },
     },
     data() {
         return {
@@ -178,13 +210,11 @@ export default {
         };
     },
     computed: {
-        // pop it open if anything is selected
         showContentFilters() {
             const { showHidden, showDeleted, filterText } = this.params;
             return this.showFilter || showHidden || showDeleted || filterText;
         },
 
-        // pass-through
         localParams: {
             get() {
                 return this.params;
@@ -194,8 +224,12 @@ export default {
             },
         },
 
+        numSelected() {
+            return this.contentSelection.size || 0;
+        },
+
         hasSelection() {
-            return this.contentSelection.size > 0;
+            return this.numSelected > 0;
         },
 
         countHidden() {
@@ -203,22 +237,29 @@ export default {
         },
     },
     methods: {
+        toggleSelectAll() {
+            if (this.hasSelection) {
+                this.$emit("resetSelection");
+                this.$emit("update:show-selection", false);
+            } else {
+                this.$emit("selectAllContent");
+                this.$emit("update:show-selection", true);
+            }
+        },
+
         // #region history-wide bulk updates, does server query first to determine "selection"
 
         async unhideAll(evt) {
             await unhideAllHiddenContent(this.history);
             this.$emit("manualReload");
-            evt.vueTarget.hide();
         },
         async deleteAllHidden(evt) {
             await deleteAllHiddenContent(this.history);
             this.$emit("manualReload");
-            evt.vueTarget.hide();
         },
         async purgeAllDeleted(evt) {
             await purgeAllDeletedContent(this.history);
             this.$emit("manualReload");
-            evt.vueTarget.hide();
         },
 
         // #endregion
@@ -243,7 +284,10 @@ export default {
         async runOnSelection(fn) {
             const items = Array.from(this.contentSelection.values());
             const type_ids = items.map((o) => o.type_id);
-            await fn(this.history, type_ids);
+            const results = await fn(this.history, type_ids);
+            if (this.debug) {
+                console.log("operation results", results);
+            }
             this.$emit("resetSelection");
             this.$emit("manualReload");
         },
@@ -282,3 +326,10 @@ export default {
     },
 };
 </script>
+
+<style lang="scss">
+// remove borders around buttons in menu
+.content-operations-menu .btn-group .btn {
+    border-color: transparent !important;
+}
+</style>
diff --git a/client/src/components/History/CopyModal.vue b/client/src/components/History/CopyModal.vue
index 92314604b28..5d93707c302 100644
--- a/client/src/components/History/CopyModal.vue
+++ b/client/src/components/History/CopyModal.vue
@@ -1,5 +1,5 @@
 <template>
-    <b-modal ref="copyModal" v-model="showModal" :title="title" title-tag="h2">
+    <b-modal v-bind="$attrs" v-on="$listeners" :title="title" title-tag="h2">
         <transition name="fade">
             <b-alert :show="currentUser.isAnonymous" variant="warning" v-localize>
                 As an anonymous user, unless you login or register, you will lose your current history after copying
@@ -51,7 +51,6 @@ import { mapActions, mapGetters } from "vuex";
 export default {
     props: {
         history: { type: History, required: true },
-        value: { type: Boolean, required: false, default: false },
     },
     data() {
         return {
@@ -81,16 +80,6 @@ export default {
         formValid() {
             return this.newNameValid;
         },
-        showModal: {
-            get() {
-                return this.value;
-            },
-            set(newVal, oldVal) {
-                if (newVal !== oldVal) {
-                    this.$emit("input", newVal);
-                }
-            },
-        },
     },
     methods: {
         ...mapActions("betaHistory", ["copyHistory"]),
diff --git a/client/src/components/History/CurrentCollection/Details.vue b/client/src/components/History/CurrentCollection/Details.vue
index 32b4fbb2ec5..855d1567c20 100644
--- a/client/src/components/History/CurrentCollection/Details.vue
+++ b/client/src/components/History/CurrentCollection/Details.vue
@@ -1,45 +1,108 @@
+<!-- Edits a collection, if that collection is editable. -->
+
 <template>
-    <section class="history-details controls">
-        <ClickToEdit
-            :value="dsc.name"
-            v-if="writable"
-            @input="(name) => $emit('update:dsc', { name })"
-            tag-name="h3"
-            class="history-title mt-3"
-            :tooltip-title="'Rename collection...' | localize"
-        />
-        <h3 v-else class="history-title mt-3">{{ dsc.name }}</h3>
+    <section class="history-details d-flex">
+        <div class="flex-grow-1">
+            <h3 class="history-title">{{ dscName || "(Collection Name)" }}</h3>
+            <p class="mt-1">
+                <i class="fas fa-folder"></i>
+                a {{ dsc.collectionType | localize }}
+                {{ dsc.collectionCount | localize }}
+            </p>
 
-        <p class="mt-1">
-            <i class="fas fa-folder"></i>
-            a {{ dsc.collectionType | localize }}
-            {{ dsc.collectionCount | localize }}
-        </p>
+            <div v-if="isEditing" class="mt-3">
+                <b-textarea v-model="dscName" placeholder="Collection Name" trim max-rows="4"></b-textarea>
+                <ContentTags :content="dsc" class="mt-3" />
+            </div>
 
-        <transition name="shutterfade">
-            <ContentTags v-if="writable" :content="dsc" />
-            <div v-else-if="dsc.tags && dsc.tags.length" class="nametags p-1">
+            <div v-else-if="dsc.tags && dsc.tags.length">
                 <Nametag v-for="tag in dsc.tags" :key="tag" :tag="tag" />
             </div>
-        </transition>
+        </div>
+
+        <EditorMenu
+            class="ml-3 flex-grow-0 d-flex flex-column"
+            v-if="writable"
+            model-name="Collection"
+            :editing.sync="editing"
+            :writable="writable"
+            :valid="valid"
+            :dirty="dirty"
+            @save="save"
+            @revert="revert"
+        />
     </section>
 </template>
 
 <script>
 import { DatasetCollection } from "../model";
-import ClickToEdit from "components/ClickToEdit";
 import { Nametag } from "components/Nametags";
 import ContentTags from "../ContentTags";
+import EditorMenu from "../EditorMenu";
 
 export default {
     components: {
-        ClickToEdit,
         ContentTags,
         Nametag,
+        EditorMenu,
     },
     props: {
         dsc: { type: DatasetCollection, required: true },
         writable: { type: Boolean, required: true },
     },
+    data() {
+        return {
+            editing: false,
+            tempContent: this.dsc.clone(),
+        };
+    },
+    computed: {
+        dscId() {
+            return this.dsc.id;
+        },
+        isEditing() {
+            return this.writable && this.editing;
+        },
+        valid() {
+            return this.tempContent.name.length > 0;
+        },
+        dirty() {
+            return !this.tempContent.equals(this.dsc);
+        },
+        dscName: {
+            get() {
+                return this.tempContent.name;
+            },
+            set(newName) {
+                this.patchTempDsc({ name: newName });
+            },
+        },
+    },
+    watch: {
+        dscId(newId, oldId) {
+            if (newId != oldId) {
+                this.reset();
+            }
+        },
+    },
+    methods: {
+        reset() {
+            this.tempContent = this.dsc.clone();
+        },
+        patchTempDsc(props = {}) {
+            this.tempContent = this.tempContent.patch(props);
+        },
+        save() {
+            if (this.valid) {
+                this.$emit("update:dsc", this.tempContent);
+                this.editing = false;
+            }
+        },
+        revert() {
+            if (this.dirty) {
+                this.reset();
+            }
+        },
+    },
 };
 </script>
diff --git a/client/src/components/History/CurrentCollection/Panel.vue b/client/src/components/History/CurrentCollection/Panel.vue
index 298ba97e2be..9a259d92e05 100644
--- a/client/src/components/History/CurrentCollection/Panel.vue
+++ b/client/src/components/History/CurrentCollection/Panel.vue
@@ -9,10 +9,19 @@
                 v-slot="{ isExpanded, setExpanded }"
             >
                 <Layout class="dataset-collection-panel">
-                    <template v-slot:nav>
+                    <template v-slot:globalNav>
                         <TopNav :history="history" :selected-collections="selectedCollections" v-on="$listeners" />
                     </template>
 
+                    <template v-slot:localNav>
+                        <IconButton
+                            icon="download"
+                            title="Download Collection"
+                            :href="downloadCollectionUrl"
+                            download
+                        />
+                    </template>
+
                     <template v-slot:details>
                         <Details :dsc="dsc" :writable="writable" @update:dsc="updateDsc(dsc, $event)" />
                     </template>
@@ -23,13 +32,16 @@
                             :class="{ loadingBackground: loading }"
                             v-bind="payload"
                             @scroll="setScrollPos"
+                            :debug="false"
                         >
-                            <template v-slot="{ item, index }">
+                            <template v-slot="{ item, index, rowKey }">
                                 <CollectionContentItem
                                     :item="item"
                                     :index="index"
+                                    :row-key="rowKey"
                                     :expanded="isExpanded(item)"
-                                    :writable="writable"
+                                    :writable="false"
+                                    :selectable="false"
                                     @update:expanded="setExpanded(item, $event)"
                                     @viewCollection="$emit('viewCollection', item)"
                                 />
@@ -55,6 +67,7 @@ import Scroller from "../Scroller";
 import { CollectionContentItem } from "../ContentItem";
 
 import { reportPayload } from "../providers/ContentProvider/helpers";
+import IconButton from "components/IconButton";
 
 export default {
     filters: {
@@ -69,6 +82,7 @@ export default {
         Scroller,
         CollectionContentItem,
         ExpandedItems,
+        IconButton,
     },
     props: {
         history: { type: History, required: true },
@@ -86,14 +100,22 @@ export default {
         writable() {
             return this.isRoot;
         },
+        rootCollection() {
+            return this.selectedCollections[0];
+        },
+        downloadCollectionUrl() {
+            let url = "";
+            if (this.rootCollection) {
+                url = `${this.rootCollection.url}/download`;
+            }
+            return url;
+        },
     },
     methods: {
         // change the data of the root collection, anything past the root
         // collection is part of the dataset collection, which i believe is supposed to
         // be immutable, so only edit name, tags, blah of top-level selected collection,
-
         async updateDsc(collection, fields) {
-            // console.log("updateDsc", this.writable, collection, fields);
             if (this.writable) {
                 const ajaxResult = await updateContentFields(collection, fields);
                 await cacheContent({ ...collection, ...ajaxResult });
diff --git a/client/src/components/History/CurrentCollection/TopNav.vue b/client/src/components/History/CurrentCollection/TopNav.vue
index 85f324d930b..cea439b63df 100644
--- a/client/src/components/History/CurrentCollection/TopNav.vue
+++ b/client/src/components/History/CurrentCollection/TopNav.vue
@@ -1,46 +1,34 @@
 <template>
-    <div class="navigation">
-        <b-dropdown v-if="breadCrumbOptions.length" class="mr-auto" size="sm" text="Go to..." boundary="viewport">
-            <b-dropdown-item @click="close"> History: {{ history.name }} </b-dropdown-item>
-            <b-dropdown-item v-for="option in breadCrumbOptions" :key="option.key" @click="reselect(option.value)">
-                {{ option.text }}
-            </b-dropdown-item>
-        </b-dropdown>
+    <b-dropdown v-if="breadCrumbOptions.length" size="sm" text="Return to..." boundary="viewport" no-caret>
+        <template v-slot:button-content>
+            <Icon icon="arrow-up" class="mr-1" />
+            <b class="text-nowrap" v-localize>Return to...</b>
+        </template>
 
-        <b-button v-else size="sm" class="mr-auto back" @click="close"> Back to: {{ history.name }} </b-button>
+        <b-dropdown-item @click="close">
+            <span>History: {{ history.name }}</span>
+        </b-dropdown-item>
 
-        <PriorityMenu :starting-height="27">
-            <PriorityMenuItem
-                key="download-collection"
-                title="Download Collection"
-                icon="fas fa-file-download"
-                tag="a"
-                download
-                :href="downloadCollectionUrl"
-            />
-            <PriorityMenuItem key="back-up" title="Back One" icon="fas fa-level-up-alt" @click="back" />
-        </PriorityMenu>
-    </div>
+        <b-dropdown-item v-for="option in breadCrumbOptions" :key="option.key" @click="reselect(option.value)">
+            <span>{{ option.text }}</span>
+        </b-dropdown-item>
+    </b-dropdown>
+
+    <b-button v-else size="sm" title="`Return to: ${history.name}`" @click="close">
+        <Icon icon="arrow-up" class="mr-1" />
+        <b class="text-nowrap">Return to: {{ history.name }}</b>
+    </b-button>
 </template>
 
 <script>
 import { History } from "../model";
-import { PriorityMenuItem, PriorityMenu } from "components/PriorityMenu";
 
 export default {
-    components: {
-        PriorityMenuItem,
-        PriorityMenu,
-    },
     props: {
         history: { type: History, required: true },
         selectedCollections: { type: Array, required: true, validate: (val) => val.length > 0 },
     },
     computed: {
-        rootCollection() {
-            return this.selectedCollections[0];
-        },
-
         // List for the dropdown. Should be the history and every parent
         // collection excepting the current one.
         breadCrumbOptions() {
@@ -54,14 +42,6 @@ export default {
             });
             return options;
         },
-
-        downloadCollectionUrl() {
-            let url = "";
-            if (this.rootCollection) {
-                url = `${this.rootCollection.url}/download`;
-            }
-            return url;
-        },
     },
     methods: {
         reselect(newList) {
diff --git a/client/src/components/History/CurrentHistoryPanel.vue b/client/src/components/History/CurrentHistoryPanel.vue
index 6b58fe67d01..b54c8619a55 100644
--- a/client/src/components/History/CurrentHistoryPanel.vue
+++ b/client/src/components/History/CurrentHistoryPanel.vue
@@ -2,20 +2,17 @@
     <CurrentUser class="d-flex flex-column" v-slot="{ user }">
         <UserHistories v-if="user" :user="user" v-slot="{ currentHistory, histories, handlers }">
             <HistoryPanel v-if="currentHistory" :history="currentHistory" v-on="handlers" id="current-history-panel">
-                <template v-slot:nav>
-                    <div>
-                        <HistorySelector
-                            :histories="histories"
-                            :current-history="currentHistory"
-                            :show-modal="showModal"
-                            @update:currentHistory="handlers.setCurrentHistory"
-                            @hideModal="showModalHandler"
-                        />
-                        <HistoriesMenu v-on="handlers" @selectHistoryModal="showModalHandler" />
-                    </div>
+                <template v-slot:globalNav>
+                    <HistoriesMenu
+                        v-on="handlers"
+                        :histories="histories"
+                        :current-history="currentHistory"
+                        title="Current History"
+                    />
                 </template>
             </HistoryPanel>
 
+            <!-- still loading -->
             <div v-else class="flex-grow-1 loadingBackground h-100">
                 <span class="sr-only" v-localize>Loading History...</span>
             </div>
@@ -27,7 +24,6 @@
 import CurrentUser from "components/providers/CurrentUser";
 import UserHistories from "./providers/UserHistories";
 import HistoryPanel from "./HistoryPanel";
-import HistorySelector from "./HistorySelector";
 import HistoriesMenu from "./HistoriesMenu";
 
 export default {
@@ -35,18 +31,12 @@ export default {
         CurrentUser,
         UserHistories,
         HistoryPanel,
-        HistorySelector,
         HistoriesMenu,
     },
-    data() {
-        return {
-            showModal: false,
-        };
-    },
-    methods: {
-        showModalHandler(event) {
-            this.showModal = !this.showModal;
-        },
-    },
+    // data() {
+    //     return {
+    //         showModal: false,
+    //     };
+    // },
 };
 </script>
diff --git a/client/src/components/History/EditorMenu.vue b/client/src/components/History/EditorMenu.vue
new file mode 100644
index 00000000000..a3c84878860
--- /dev/null
+++ b/client/src/components/History/EditorMenu.vue
@@ -0,0 +1,73 @@
+<template>
+    <nav class="edit-controls">
+        <IconButton
+            :title="label('Edit') | l"
+            class="edit-button mb-1"
+            :pressed="editing"
+            icon="pen"
+            @click="$emit('update:editing', !editing)"
+        />
+
+        <template v-if="editing">
+            <!-- save -->
+            <IconButton
+                :title="label('Save') | l"
+                class="save-button mb-1"
+                :class="{ disabled: !valid }"
+                icon="save"
+                @click="$emit('save')"
+            />
+
+            <!-- revert -->
+            <IconButton
+                :title="label('Revert') | l"
+                class="revert-changes-button mb-1"
+                :class="{ disabled: !dirty }"
+                icon="undo"
+                @click="$emit('revert')"
+            />
+
+            <!-- form status -->
+            <IconButton
+                :title="statusLabel | l"
+                class="status-button mb-1"
+                :class="{ valid: valid, disabled: !valid }"
+                :icon="valid ? 'check' : 'exclamation-triangle'"
+                variant="link"
+            />
+        </template>
+    </nav>
+</template>
+
+<script>
+import IconButton from "components/IconButton";
+
+export default {
+    components: {
+        IconButton,
+    },
+    props: {
+        modelName: { type: String, required: true },
+        editing: { type: Boolean, required: true },
+        dirty: { type: Boolean, required: false, default: false },
+        valid: { type: Boolean, required: false, default: false },
+    },
+    computed: {
+        statusLabel() {
+            const status = this.valid ? "Valid" : "Invalid";
+            return `${status} ${this.modelName}`;
+        },
+    },
+    methods: {
+        label(verb) {
+            return `${verb} ${this.modelName}`;
+        },
+    },
+};
+</script>
+
+<style lang="scss">
+.edit-controls .disabled {
+    opacity: 0.25;
+}
+</style>
diff --git a/client/src/components/History/HistoriesMenu.vue b/client/src/components/History/HistoriesMenu.vue
index ceb38a7df17..8e88a20cf3a 100644
--- a/client/src/components/History/HistoriesMenu.vue
+++ b/client/src/components/History/HistoriesMenu.vue
@@ -1,66 +1,83 @@
+<!-- menu allowing user to change the current history, make a new one, basically anything that's
+"above" editing the current history -->
+
 <template>
-    <PriorityMenu :starting-height="27">
-        <PriorityMenuItem
-            key="select-new-history"
-            title="Select History"
-            icon="fa fa-folder-open"
-            @click="$emit('selectHistoryModal')"
+    <div>
+        <b-dropdown
+            size="sm"
+            variant="link"
+            :text="title | l"
+            toggle-class="text-decoration-none"
+            no-caret
+            class="histories-operation-menu"
+            data-description="histories operation menu"
+        >
+            <template v-slot:button-content>
+                <Icon class="mr-1" icon="folder" />
+                <span id="click-me-fool" class="text-nowrap">{{ title | l }}</span>
+            </template>
+
+            <b-dropdown-text>
+                <span>You have {{ histories.length }} histories.</span>
+            </b-dropdown-text>
+
+            <b-dropdown-divider></b-dropdown-divider>
+
+            <b-dropdown-item v-b-modal.history-selector-modal>
+                <Icon class="mr-1" icon="exchange-alt" />
+                <span v-localize>Change the Current History</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="$emit('createNewHistory')">
+                <Icon class="mr-1" icon="plus" />
+                <span v-localize>Create a New History</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="backboneRoute('/histories/list')">
+                <Icon class="mr-1" icon="list" />
+                <span v-localize>View Saved Histories</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="redirect('/history/view_multiple')">
+                <Icon class="mr-1" icon="columns" />
+                <span v-localize>Show Histories Side-by-Side</span>
+            </b-dropdown-item>
+
+            <b-dropdown-divider></b-dropdown-divider>
+
+            <b-dropdown-item @click="switchToLegacyHistoryPanel">
+                <Icon class="mr-1" icon="arrow-up" />
+                <span v-localize>Return to legacy panel</span>
+            </b-dropdown-item>
+        </b-dropdown>
+
+        <!-- modals -->
+        <HistorySelectorModal
+            id="history-selector-modal"
+            :histories="histories"
+            :current-history="currentHistory"
+            @selectHistory="$emit('setCurrentHistory', $event)"
         />
-        <PriorityMenuItem
-            key="create-new-history"
-            title="Create New History"
-            class="history-new-button"
-            icon="fa fa-plus"
-            @click="$emit('createNewHistory')"
-        />
-        <PriorityMenuItem
-            key="view-all-histories"
-            title="View All Histories"
-            icon="fa fa-columns"
-            class="history-view-multi-button"
-            @click="redirect('/history/view_multiple')"
-        />
-        <PriorityMenuItem
-            key="saved-histories"
-            title="Saved Histories"
-            icon="fas fa-save"
-            @click="backboneRoute('/histories/list')"
-        />
-        <PriorityMenuItem
-            key="clear-history-cache"
-            title="Refresh History"
-            icon="fa fa-refresh"
-            class="history-refresh-button"
-            @click.stop="clearCache"
-        />
-        <PriorityMenuItem
-            key="use-legacy-history"
-            title="Return to legacy history panel"
-            icon="fas fa-exchange-alt"
-            @click="switchToLegacyHistoryPanel"
-        />
-    </PriorityMenu>
+    </div>
 </template>
 
 <script>
-import { PriorityMenuItem, PriorityMenu } from "components/PriorityMenu";
-import { wipeDatabase, clearHistoryDateStore } from "./caching";
+import { History } from "./model";
 import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
 import { switchToLegacyHistoryPanel } from "./adapters/betaToggle";
-import { reloadPage } from "utils/redirect";
+import HistorySelectorModal from "./HistorySelectorModal";
 
 export default {
     mixins: [legacyNavigationMixin],
     components: {
-        PriorityMenu,
-        PriorityMenuItem,
+        HistorySelectorModal,
+    },
+    props: {
+        histories: { type: Array, required: true },
+        currentHistory: { type: History, required: true },
+        title: { type: String, required: false, default: "Histories" },
     },
     methods: {
-        async clearCache() {
-            await wipeDatabase();
-            await clearHistoryDateStore();
-            reloadPage();
-        },
         switchToLegacyHistoryPanel,
     },
 };
diff --git a/client/src/components/History/History.vue b/client/src/components/History/History.vue
index 5a6d3a2514b..48b1013848c 100644
--- a/client/src/components/History/History.vue
+++ b/client/src/components/History/History.vue
@@ -10,7 +10,7 @@
         <ExpandedItems
             :scope-key="history.id"
             :get-item-key="(item) => item.type_id"
-            v-slot="{ isExpanded, setExpanded }"
+            v-slot="{ expandedCount, isExpanded, setExpanded, collapseAll }"
         >
             <SelectedItems
                 :scope-key="history.id"
@@ -26,16 +26,20 @@
                 }"
             >
                 <Layout>
-                    <template v-slot:nav>
-                        <slot name="nav"></slot>
+                    <template v-slot:globalNav>
+                        <slot name="globalNav" :history="history"></slot>
+                    </template>
+
+                    <template v-slot:localNav>
+                        <HistoryMenu :history="history" v-on="$listeners" />
                     </template>
 
                     <template v-slot:details>
-                        <HistoryDetails class="history-details" :history="history" v-on="$listeners" />
+                        <HistoryDetails :history="history" v-on="$listeners" />
                     </template>
 
                     <template v-slot:messages>
-                        <HistoryMessages class="history-messages m-2" :history="history" />
+                        <HistoryMessages class="m-2" :history="history" />
                     </template>
 
                     <template v-slot:listcontrols>
@@ -51,6 +55,8 @@
                             @resetSelection="resetSelection"
                             @selectAllContent="selectItems(payload.contents)"
                             @manualReload="manualReload"
+                            :expanded-count="expandedCount"
+                            @collapseAllContent="collapseAll"
                         />
                     </template>
 
@@ -65,10 +71,11 @@
                             :debug="false"
                             @scroll="setScrollPos"
                         >
-                            <template v-slot="{ item, index }">
+                            <template v-slot="{ item, index, rowKey }">
                                 <HistoryContentItem
                                     :item="item"
                                     :index="index"
+                                    :row-key="rowKey"
                                     :show-selection="showSelection"
                                     :expanded="isExpanded(item)"
                                     @update:expanded="setExpanded(item, $event)"
@@ -101,6 +108,7 @@ import ToolHelpModal from "./ToolHelpModal";
 import Scroller from "./Scroller";
 import { HistoryContentItem } from "./ContentItem";
 import { reportPayload } from "./providers/ContentProvider/helpers";
+import HistoryMenu from "./HistoryMenu";
 
 export default {
     filters: {
@@ -118,6 +126,7 @@ export default {
         HistoryContentItem,
         ExpandedItems,
         SelectedItems,
+        HistoryMenu,
     },
     props: {
         history: { type: History, required: true },
diff --git a/client/src/components/History/HistoryDetails.vue b/client/src/components/History/HistoryDetails.vue
index a271f0e6006..a9bc27bf1e7 100644
--- a/client/src/components/History/HistoryDetails.vue
+++ b/client/src/components/History/HistoryDetails.vue
@@ -1,215 +1,145 @@
-<template>
-    <header>
-        <div class="d-flex justify-content-between align-items-center actions">
-            <h6 class="text-nowrap text-truncate mr-auto">
-                <span class="size history-size">{{ history.niceSize | localize }}</span>
-            </h6>
+<!--
+History object editor. Change name, edit tags, description.
+Emit new History model object when done via .sync syntax
+-->
 
-            <PriorityMenu style="max-width: 50%" :starting-height="28">
-                <PriorityMenuItem
-                    key="edit-history-tags"
-                    title="Edit History Tags"
-                    class="history-tag-btn"
-                    icon="fas fa-tags"
-                    :pressed="showTags"
-                    @click="showTags = !showTags"
-                />
-                <PriorityMenuItem
-                    key="edit-history-annotation"
-                    title="Edit Annotation"
-                    class="history-annotate-btn"
-                    icon="fas fa-comment"
-                    :pressed="showAnnotation"
-                    @click="showAnnotation = !showAnnotation"
-                />
-                <PriorityMenuItem
-                    key="copy-history"
-                    title="Copy History"
-                    icon="fas fa-copy"
-                    @click="showCopyModal = !showCopyModal"
-                />
-                <PriorityMenuItem
-                    key="share"
-                    title="Share or Publish"
-                    icon="fas fa-share-alt"
-                    @click="backboneRoute('/histories/sharing', { id: history.id })"
-                />
-                <PriorityMenuItem
-                    key="delete-this-history"
-                    title="Delete this History"
-                    icon="fas fa-trash"
-                    @click="showDeleteModal = !showDeleteModal"
-                />
-                <PriorityMenuItem
-                    key="purge-this-history"
-                    title="Purge this History"
-                    icon="fas fa-burn"
-                    @click="showPurgeModal = !showPurgeModal"
-                />
-                <PriorityMenuItem
-                    key="structure"
-                    title="Show Structure"
-                    icon="fas fa-code-branch"
-                    @click="backboneRoute('/histories/show_structure')"
-                />
-                <PriorityMenuItem
-                    key="extract-workflow"
-                    title="Extract Workflow"
-                    icon="fas fa-file-export"
-                    @click="iframeRedirect('/workflow/build_from_current_history')"
-                />
-                <PriorityMenuItem
-                    key="set-permissions"
-                    title="Set Permissions"
-                    icon="fas fa-user-lock"
-                    @click="backboneRoute('/histories/permissions', { id: history.id })"
-                />
-                <PriorityMenuItem
-                    key="make-private"
-                    title="Make Private"
-                    icon="fas fa-lock"
-                    @click="showPrivacyModal = !showPrivacyModal"
-                />
-                <PriorityMenuItem
-                    key="resume-paused-jobs"
-                    title="Resume Paused Jobs"
-                    icon="fas fa-play"
-                    @click="iframeRedirect('/history/resume_paused_jobs?current=True')"
-                />
-                <PriorityMenuItem
-                    key="export-citations"
-                    title="Export Tool Citations"
-                    icon="fas fa-stream"
-                    @click="backboneRoute('/histories/citations', { id: history.id })"
-                />
-                <PriorityMenuItem
-                    key="export-history-to-file"
-                    title="Export History to File"
-                    icon="fas fa-file-archive"
-                    @click="backboneRoute(exportLink)"
-                />
-            </PriorityMenu>
+<template>
+    <section class="history-details d-flex">
+        <div class="flex-grow-1">
+            <h3>{{ historyName || "(History Name)" }}</h3>
+            <h5 class="history-size">{{ history.size | niceFileSize }}</h5>
+
+            <!-- display title, annotation, tags -->
+            <div v-if="!editing">
+                <p v-if="annotation" class="mt-1">{{ annotation }}</p>
+                <HistoryTags class="mt-2" :history="history" :disabled="true" />
+            </div>
+
+            <!-- edit form, change title, annotation, or tags -->
+            <div v-else class="mt-3">
+                <b-textarea
+                    class="mb-2"
+                    v-model="historyName"
+                    placeholder="History Name"
+                    trim
+                    max-rows="4"
+                ></b-textarea>
+
+                <b-textarea
+                    class="mb-2"
+                    v-model="annotation"
+                    placeholder="Annotation (optional)"
+                    trim
+                    max-rows="4"
+                ></b-textarea>
+
+                <HistoryTags class="mt-3" :history="history" />
+            </div>
         </div>
 
-        <!-- click to edit the name of the history -->
-        <ClickToEdit
-            class="history-title mt-2"
-            tag-name="h3"
-            v-model="historyName"
-            :title="'Rename history...' | localize"
-            v-b-tooltip.hover="{ boundary: 'viewport', placement: 'left' }"
+        <EditorMenu
+            class="ml-3 flex-grow-0 d-flex flex-column"
+            v-if="writable"
+            model-name="History"
+            :editing.sync="editing"
+            :writable="writable"
+            :valid="valid"
+            :dirty="dirty"
+            @save="save"
+            @revert="revert"
         />
-
-        <transition name="shutterfade">
-            <Annotation v-if="showAnnotation" class="history-annotation mt-2" v-model="annotation" />
-        </transition>
-
-        <transition name="shutterfade">
-            <HistoryTags v-if="showTags" class="mt-2" :history="history" />
-        </transition>
-
-        <!-- #region modals -->
-
-        <CopyHistoryModal v-model="showCopyModal" :history="history" />
-
-        <b-modal v-model="showPrivacyModal" title="Make History Private" title-tag="h2" @ok="makePrivate">
-            <p v-localize>
-                This will make all the data in this history private (excluding library datasets), and will set
-                permissions such that all new data is created as private. Any datasets within that are currently shared
-                will need to be re-shared or published. Are you sure you want to do this?
-            </p>
-        </b-modal>
-
-        <b-modal v-model="showDeleteModal" title="Delete History?" title-tag="h2" @ok="deleteHistory">
-            <p v-localize>Really delete the current history?</p>
-        </b-modal>
-
-        <b-modal v-model="showPurgeModal" title="Permanently Delete History?" title-tag="h2" @ok="purgeHistory">
-            <p v-localize>Really delete the current history permanently? This cannot be undone.</p>
-        </b-modal>
-
-        <!-- #endregion -->
-    </header>
+    </section>
 </template>
 
 <script>
+import prettyBytes from "pretty-bytes";
 import { History } from "./model";
-import { PriorityMenuItem, PriorityMenu } from "components/PriorityMenu";
-import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
-import ClickToEdit from "components/ClickToEdit";
-import Annotation from "components/Annotation";
 import HistoryTags from "./HistoryTags";
-import CopyHistoryModal from "./CopyModal";
+import EditorMenu from "./EditorMenu";
 
 export default {
-    mixins: [legacyNavigationMixin],
     components: {
-        PriorityMenu,
-        PriorityMenuItem,
         HistoryTags,
-        Annotation,
-        ClickToEdit,
-        CopyHistoryModal,
+        EditorMenu,
+    },
+    filters: {
+        niceFileSize(rawSize = 0) {
+            return prettyBytes(rawSize);
+        },
     },
     props: {
         history: { type: History, required: true },
+        writable: { type: Boolean, required: false, default: true },
     },
     data() {
         return {
-            showTags: false,
-            showAnnotation: false,
-            showCopyModal: false,
-            showPrivacyModal: false,
-            showDeleteModal: false,
-            showPurgeModal: false,
+            editing: false,
+            tempHistory: this.history.clone(),
         };
     },
     computed: {
+        historyId() {
+            return this.history.id;
+        },
+        isEditing() {
+            return this.writable && this.editing;
+        },
+        valid() {
+            return this.tempHistory.name.length > 0;
+        },
+        dirty() {
+            return !this.tempHistory.equals(this.history);
+        },
         annotation: {
             get() {
-                return this.history.annotation || "";
+                return this.tempHistory.annotation;
             },
             set(annotation) {
-                if (annotation.length && annotation !== this.history.annotation) {
-                    this.updateFields({ annotation });
-                }
+                this.patchTempHistory({ annotation });
             },
         },
-
         historyName: {
             get() {
-                return this.history?.name || "";
+                return this.tempHistory.name;
             },
             set(name) {
-                if (name.length && name !== this.history.name) {
-                    this.updateFields({ name });
-                }
+                this.patchTempHistory({ name });
             },
         },
-        exportLink() {
-            return `histories/${this.history.id}/export`;
+        tags: {
+            get() {
+                return this.tempHistory.tags || [];
+            },
+            set(newTags) {
+                this.patchTempHistory({ tags: newTags });
+            },
+        },
+    },
+    watch: {
+        historyId(newId, oldId) {
+            if (newId != oldId) {
+                this.reset();
+            }
         },
     },
     methods: {
-        async updateFields(fields = {}) {
-            this.$emit("updateHistory", { id: this.history.id, ...fields });
+        reset() {
+            this.tempHistory = this.history.clone();
         },
-        async deleteHistory(evt) {
-            this.$emit("deleteHistory", this.history);
-            evt.vueTarget.hide();
+        patchTempHistory(props = {}) {
+            this.tempHistory = this.tempHistory.patch(props);
         },
-        async purgeHistory(evt) {
-            this.$emit("purgeHistory", this.history);
-            evt.vueTarget.hide();
+        save() {
+            if (this.valid) {
+                this.$emit("update:history", this.tempHistory);
+                this.editing = false;
+            }
         },
-        async makePrivate(evt) {
-            this.$emit("secureHistory", this.history);
-            evt.vueTarget.hide();
+        revert() {
+            if (this.dirty) {
+                this.reset();
+            }
         },
     },
 };
 </script>
-
-<style lang="scss">
-@import "scss/transitions.scss";
-</style>
diff --git a/client/src/components/History/HistoryMenu.vue b/client/src/components/History/HistoryMenu.vue
new file mode 100644
index 00000000000..a80bf9a9925
--- /dev/null
+++ b/client/src/components/History/HistoryMenu.vue
@@ -0,0 +1,119 @@
+<template>
+    <div>
+        <b-dropdown size="sm" variant="link" text="Histories" toggle-class="text-decoration-none">
+            <template v-slot:button-content>
+                <Icon icon="wrench" />
+                <span class="sr-only">Operations on the current history</span>
+            </template>
+
+            <b-dropdown-item v-b-modal:copy-history-modal>
+                <Icon icon="copy" class="mr-1" />
+                <span v-localize>Copy this History</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item v-b-modal:delete-history-modal>
+                <Icon icon="trash" class="mr-1" />
+                <span v-localize>Delete this History</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item v-b-modal:purge-history-modal>
+                <Icon icon="burn" class="mr-1" />
+                <span v-localize>Purge this History</span>
+            </b-dropdown-item>
+
+            <b-dropdown-divider></b-dropdown-divider>
+
+            <b-dropdown-item @click="iframeRedirect('/history/resume_paused_jobs?current=True')">
+                <Icon icon="play" class="mr-1" />
+                <span v-localize>Resume Paused Jobs</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="iframeRedirect('/workflow/build_from_current_history')">
+                <Icon icon="file-export" class="mr-1" />
+                <span v-localize>Extract Workflow</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="backboneRoute('/histories/show_structure')">
+                <Icon icon="code-branch" class="mr-1" />
+                <span v-localize>Show Structure</span>
+            </b-dropdown-item>
+
+            <b-dropdown-divider></b-dropdown-divider>
+
+            <b-dropdown-item @click="backboneRoute('/histories/sharing', { id: history.id })">
+                <Icon icon="share-alt" class="mr-1" />
+                <span v-localize>Share or Publish</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="backboneRoute('/histories/permissions', { id: history.id })">
+                <Icon icon="user-lock" class="mr-1" />
+                <span v-localize>Set Permissions</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item v-b-modal:history-privacy-modal>
+                <Icon icon="lock" class="mr-1" />
+                <span v-localize>Make Private</span>
+            </b-dropdown-item>
+
+            <b-dropdown-divider></b-dropdown-divider>
+
+            <b-dropdown-item @click="backboneRoute('/histories/citations', { id: history.id })">
+                <Icon icon="stream" class="mr-1" />
+                <span v-localize>Export Tool Citations</span>
+            </b-dropdown-item>
+
+            <b-dropdown-item @click="backboneRoute(history.exportLink)">
+                <Icon icon="file-archive" class="mr-1" />
+                <span v-localize>Export History to File</span>
+            </b-dropdown-item>
+        </b-dropdown>
+
+        <!-- #region modals -->
+
+        <CopyHistoryModal id="copy-history-modal" :history="history" />
+
+        <b-modal
+            id="history-privacy-modal"
+            title="Make History Private"
+            title-tag="h2"
+            @ok="$emit('secureHistory', history)"
+        >
+            <p v-localize>
+                This will make all the data in this history private (excluding library datasets), and will set
+                permissions such that all new data is created as private. Any datasets within that are currently shared
+                will need to be re-shared or published. Are you sure you want to do this?
+            </p>
+        </b-modal>
+
+        <b-modal id="delete-history-modal" title="Delete History?" title-tag="h2" @ok="$emit('deleteHistory', history)">
+            <p v-localize>Really delete the current history?</p>
+        </b-modal>
+
+        <b-modal
+            id="purge-history-modal"
+            title="Permanently Delete History?"
+            title-tag="h2"
+            @ok="$emit('purgeHistory', history)"
+        >
+            <p v-localize>Really delete the current history permanently? This cannot be undone.</p>
+        </b-modal>
+
+        <!-- #endregion -->
+    </div>
+</template>
+
+<script>
+import { History } from "./model";
+import CopyHistoryModal from "./CopyModal";
+import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
+
+export default {
+    mixins: [legacyNavigationMixin],
+    components: {
+        CopyHistoryModal,
+    },
+    props: {
+        history: { type: History, required: true },
+    },
+};
+</script>
diff --git a/client/src/components/History/HistorySelectorModal.vue b/client/src/components/History/HistorySelectorModal.vue
new file mode 100644
index 00000000000..58e2ffeea45
--- /dev/null
+++ b/client/src/components/History/HistorySelectorModal.vue
@@ -0,0 +1,106 @@
+<template>
+    <b-modal ref="modal" v-on="$listeners" v-bind="$attrs" :title="'Switch to History' | l">
+        <b-form-group :description="'Filter histories' | l">
+            <b-input v-model="filter" :placeholder="'Search Filter' | l" />
+        </b-form-group>
+
+        <b-table
+            ref="history-list"
+            striped
+            hover
+            sticky-header="50vh"
+            v-model="currentRows"
+            primary-key="id"
+            :fields="fields"
+            :filter="filter"
+            :items="histories"
+            :per-page="perPage"
+            :current-page="currentPage"
+            :selectable="true"
+            select-mode="single"
+            selected-variant="success"
+            @row-selected="switchToHistory"
+        >
+            <template v-slot:cell(tags)="row">
+                <stateless-tags :value="row.item.tags" :disabled="true" />
+            </template>
+            <template v-slot:cell(update_time)="data">
+                <UtcDate :date="data.value" mode="elapsed" />
+            </template>
+        </b-table>
+
+        <template v-slot:modal-footer>
+            <b-pagination v-model="currentPage" :total-rows="filteredRowCount" :per-page="perPage"></b-pagination>
+        </template>
+    </b-modal>
+</template>
+
+<script>
+import { StatelessTags } from "components/Tags";
+import UtcDate from "components/UtcDate";
+import { debounce } from "underscore";
+
+export default {
+    components: {
+        StatelessTags,
+        UtcDate,
+    },
+    props: {
+        currentHistory: { type: Object, required: true },
+        histories: { type: Array, default: () => [] },
+        perPage: { type: Number, required: false, default: 50 },
+    },
+    data() {
+        return {
+            filter: null,
+            currentPage: 1,
+            currentRows: [],
+        };
+    },
+    computed: {
+        filteredRowCount() {
+            return this.currentRows.length;
+        },
+        selectedIndex() {
+            return this.currentRows.findIndex((h) => h.id == this.currentHistory.id);
+        },
+    },
+    created() {
+        this.fields = [
+            { key: "name", sortable: true },
+            { key: "tags", sortable: true },
+            { key: "update_time", label: "Updated", sortable: true },
+        ];
+        this.debounceSelectCurrentRow = debounce(this.selectCurrentRow, 100);
+    },
+    watch: {
+        currentRows() {
+            this.selectCurrentRow();
+        },
+        filteredRowCount(newVal, oldVal) {
+            if (newVal != oldVal) {
+                this.currentPage = 1;
+            }
+        },
+        selectedIndex(idx, oldIdx) {
+            if (idx != oldIdx) {
+                this.debounceSelectCurrentRow();
+            }
+        },
+    },
+    methods: {
+        switchToHistory(selected) {
+            if (selected.length == 1) {
+                this.$emit("selectHistory", selected[0]);
+            }
+        },
+        selectCurrentRow() {
+            const idx = this.selectedIndex;
+            const list = this.$refs["history-list"];
+            if (list && idx > -1) {
+                list.selectRow(idx);
+            }
+        },
+    },
+};
+</script>
diff --git a/client/src/components/History/HistoryTags.vue b/client/src/components/History/HistoryTags.vue
index c8388e1380f..3623963e158 100644
--- a/client/src/components/History/HistoryTags.vue
+++ b/client/src/components/History/HistoryTags.vue
@@ -1,5 +1,5 @@
 <template>
-    <Tags :tags="history.tags" :store-key="storeKey" :tag-service="historyTagService" />
+    <Tags :tags="history.tags" :store-key="storeKey" :tag-service="historyTagService" :disabled="disabled" />
 </template>
 
 <script>
@@ -48,6 +48,7 @@ export default {
     components: { Tags },
     props: {
         history: { type: History, required: true },
+        disabled: { type: Boolean, required: false, default: false },
     },
     computed: {
         storeKey() {
diff --git a/client/src/components/History/Layout.vue b/client/src/components/History/Layout.vue
index b36f8bd1772..cbcf3e1ee3f 100644
--- a/client/src/components/History/Layout.vue
+++ b/client/src/components/History/Layout.vue
@@ -4,14 +4,17 @@ amateur-hour baby-tool like Bootstrap instead of taking the afternoon
 to learn how CSS is supposed to work. -->
 
 <template>
-    <section class="history d-flex flex-column">
-        <header class="p-2">
-            <!-- optional slot for injecting global navigation -->
-            <nav v-if="$slots.nav" class="top-nav pb-2">
-                <slot name="nav"></slot>
+    <section class="history beta d-flex flex-column">
+        <header>
+            <nav class="d-flex justify-content-between bg-secondary">
+                <slot name="globalNav">Global Nav Default</slot>
+                <slot name="localNav">Local nav Default</slot>
             </nav>
+
             <!-- details about the thing you're looking at -->
-            <slot name="details"></slot>
+            <div class="px-3 py-4 details">
+                <slot name="details"></slot>
+            </div>
         </header>
 
         <!-- warnings, messages, dirty limericks -->
@@ -20,12 +23,12 @@ to learn how CSS is supposed to work. -->
         </section>
 
         <!-- controls atop the list -->
-        <section v-if="$slots.listcontrols" class="p-2">
+        <section v-if="$slots.listcontrols" class="list-controls">
             <slot name="listcontrols"></slot>
         </section>
 
         <!-- scrolling list -->
-        <section class="scroller flex-grow-1">
+        <section class="position-relative flex-grow-1 scroller">
             <slot name="listing"></slot>
         </section>
 
@@ -37,13 +40,8 @@ to learn how CSS is supposed to work. -->
 </template>
 
 <style lang="scss">
-@import "scss/mixins.scss";
-@import "scss/transitions.scss";
-@import "scss/loadingBackground.scss";
-
-/* TODO: css reset? */
-
-.history {
+.history.beta {
+    /* TODO: css reset? */
     h1,
     h2,
     h3,
@@ -52,31 +50,14 @@ to learn how CSS is supposed to work. -->
     h6 {
         margin: 0;
     }
-}
 
-/*
-container css for first child of top-nav,
-flex horizontally
+    .list-controls {
+        margin-bottom: 2px;
+    }
 
-Vue can't put in multi-root slot contents so we have to do some kung-fu to get
-the layout right on the top-nav. There will have to be an extra div container
-around the contents of the nav slot, so we'll style that right here so the user
-doesn't need to do it manually every time. */
-
-.history > header > .top-nav > * {
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-    justify-content: space-between;
-}
-
-.history .messages:empty {
-    display: none;
-}
-
-/* make sure scroller actually scrolls */
-
-.history .scroller {
-    position: relative;
+    .list-controls:empty,
+    .messages:empty {
+        display: none;
+    }
 }
 </style>
diff --git a/client/src/components/History/Scroller.vue b/client/src/components/History/Scroller.vue
index 1ca1c7ed8b0..a9ba80d71ab 100644
--- a/client/src/components/History/Scroller.vue
+++ b/client/src/components/History/Scroller.vue
@@ -3,9 +3,14 @@
         <!-- mousewheel moves up and down linearly, allows for fine-tuning of view by moving just a
         few rows at a time -->
         <div ref="listing" class="listing">
-            <ul>
-                <li v-for="(item, index) in itemWindow" :key="getItemKey(item, index)">
-                    <slot :item="item" :index="index" :row-key="getItemKey(item, index)">
+            <ul class="list-unstyled m-0">
+                <li
+                    v-for="(item, index) in itemWindow"
+                    :key="getItemKey(item)"
+                    :data-row-index="index"
+                    :data-row-key="getItemKey(item)"
+                >
+                    <slot :item="item" :index="index" :row-key="getItemKey(item)">
                         <pre>{{ item }}</pre>
                     </slot>
                 </li>
@@ -15,6 +20,7 @@
         <!-- big blank scroll region uses scroll bar to drag view to entirely new region, calculated
         by visual height of the scroller knob --->
         <div
+            v-if="overlayActive"
             v-show="showScroller"
             ref="scrollSliderContainer"
             class="scrollSliderContainer"
@@ -23,6 +29,12 @@
         >
             <div ref="scrollSlider"></div>
         </div>
+
+        <!-- debugging overlay -->
+        <div v-if="debug" class="debugOverlay">
+            <pre>{{ $attrs }}</pre>
+            <pre>{{ debugProps }}</pre>
+        </div>
     </div>
 </template>
 
@@ -75,10 +87,26 @@ export default {
 
             // width of scroll bar container
             sbWidth: 30,
+
+            // whether to activate the 2nd scrollbar which does a fast-seek on large histories
+            overlayActive: true,
         };
     },
 
     computed: {
+        debugProps() {
+            return {
+                manualStartIndex: this.manualStartIndex,
+                keyField: this.keyField,
+                contentsLength: this.contents.length,
+                startKeyIndex: this.startKeyIndex,
+                totalMatches: this.totalMatches,
+                topRows: this.topRows,
+                bottomRows: this.bottomRows,
+                pageSize: this.pageSize,
+            };
+        },
+
         // The index to start rendering content at, can be manually adjusted with
         // the mouse wheel, and must be updated when new contents come in
         itemStartIndex() {
@@ -144,7 +172,7 @@ export default {
         // non-reactive data
         this.suppressEvents = false;
         this.suppressionTimeout = null;
-        this.debouncedUpdateCursor = debounce(this.updateCursor, 100);
+        // this.debouncedUpdateCursor = debounce(this.updateCursor, 100);
         this.debouncedAdjustScrollTop = debounce(this.adjustScrollTop, 100);
     },
 
@@ -193,7 +221,8 @@ export default {
         // value which we send out on scrollPos
 
         onScroll() {
-            this.debouncedUpdateCursor();
+            // this.debouncedUpdateCursor();
+            this.updateCursor();
         },
 
         // determines cursor from scrollTop, need to wait until after render to calculate
@@ -290,22 +319,15 @@ export default {
 </script>
 
 <style lang="scss">
+@import "scss/mixins.scss";
+
+.scrollContainer,
+.listing,
+.scrollSliderContainer {
+    @include absfill();
+}
+
 .scrollContainer {
-    position: absolute;
-    top: 0;
-    left: 0;
-    right: 0;
-    bottom: 0;
-
-    .listing,
-    .scrollSliderContainer {
-        position: absolute;
-        top: 0;
-        left: 0;
-        right: 0;
-        bottom: 0;
-    }
-
     .listing {
         overflow-y: scroll;
         z-index: 0;
@@ -326,14 +348,14 @@ export default {
         }
     }
 
-    ul {
-        list-style: none;
-    }
-
-    ul,
-    li {
-        padding: 0;
-        margin: 0;
+    .debugOverlay {
+        background: rgba(255, 255, 255, 0.75);
+        z-index: 2;
+        position: absolute;
+        bottom: 0;
+        left: 0;
+        right: 0;
+        height: auto;
     }
 }
 </style>
diff --git a/client/src/components/History/StatusIcon.vue b/client/src/components/History/StatusIcon.vue
index b626cf7fccf..033e6fc2739 100644
--- a/client/src/components/History/StatusIcon.vue
+++ b/client/src/components/History/StatusIcon.vue
@@ -1,82 +1,55 @@
 <template>
-    <StateBtn
-        v-if="inState(STATES.RUNNING)"
-        :state="state"
-        icon="upload-symbol upload-icon fa fa-spinner fa-spin"
+    <IconButton
+        v-if="stateIcon"
+        v-b-tooltip
+        :title="theTitle"
+        :icon="stateIcon"
+        :variant="variant"
         v-on="$listeners"
+        v-bind="$attrs"
     />
-
-    <StateBtn v-else-if="inState(STATES.QUEUED, STATES.NEW)" :state="state" icon="fas fa-clock" v-on="$listeners" />
-
-    <StateBtn v-else-if="inState(STATES.PAUSED)" :state="state" icon="fas fa-pause" v-on="$listeners" />
-
-    <StateBtn v-else-if="inState(STATES.ERROR)" :state="state" icon="fas fa-exclamation-triangle" v-on="$listeners" />
-
-    <StateBtn v-else-if="inState(STATES.MIXED)" :state="state" icon="fas fa-exclamation-circle" v-on="$listeners" />
-
-    <span v-else class="sr-only">
-        {{ state | localize }}
-    </span>
 </template>
 
 <script>
 import capitalize from "underscore.string/capitalize";
+import IconButton from "components/IconButton";
 
-export const StateBtn = {
-    template: `
-        <b-button class="state-button"
-            v-b-tooltip
-            :title="theTitle | localize"
-            size="sm"
-            variant="link"
-            v-on="$listeners"
-            v-bind="$attrs">
-            <i :class="icon" />
-            <span class="sr-only">{{ theTitle | localize }}</span>
-        </b-button>
-    `,
+export default {
+    inject: ["STATES"],
+    components: {
+        IconButton,
+    },
     props: {
         state: { type: String, required: true },
-        icon: { type: String, required: true },
-        title: { type: String, required: false, default: "" },
+        variant: { type: String, required: false, default: "link" },
     },
-    inject: ["STATES"],
     computed: {
+        stateIcon() {
+            if (this.inState(this.STATES.QUEUED, this.STATES.NEW)) {
+                return "clock";
+            }
+            if (this.inState(this.STATES.PAUSED)) {
+                return "pause";
+            }
+            if (this.inState(this.STATES.ERROR)) {
+                return "exclamation-triangle";
+            }
+            if (this.inState(this.STATES.MIXED)) {
+                return "exclamation-circle";
+            }
+            if (this.inState(this.STATES.OK)) {
+                return "check";
+            }
+            return null;
+        },
         theTitle() {
             return this.title || capitalize(this.state);
         },
     },
-};
-
-export const StatusIcon = {
-    components: {
-        StateBtn,
-    },
-    props: {
-        state: { type: String, required: true },
-    },
-    inject: ["STATES"],
     methods: {
         inState(...states) {
             return states.includes(this.state);
         },
     },
 };
-
-export default StatusIcon;
 </script>
-
-<style lang="css">
-/* Override ham-fisted base css rules */
-
-button.state-button,
-button.state-button.disabled,
-button.state-button:disabled,
-button.state-button:focus,
-button.state-button:hover,
-button.state-button:not(:disabled):active {
-    background-color: transparent !important;
-    border-style: none !important;
-    box-shadow: none !important;
-}
-</style>
diff --git a/client/src/components/History/caching/loadDscContent.js b/client/src/components/History/caching/loadDscContent.js
index 9ed786007f9..aa65fd96092 100644
--- a/client/src/components/History/caching/loadDscContent.js
+++ b/client/src/components/History/caching/loadDscContent.js
@@ -10,13 +10,14 @@ import { bulkCacheDscContent } from "./db";
 import { SearchParams } from "../model/SearchParams";
 import { prependPath } from "./workerConfig";
 import { summarizeCacheOperation, dateStore } from "./loadHistoryContents";
+import { show } from "utils/observable";
 
 /**
  * Load collection content (drill down)
  * Params: contents_url + search params + element_index
  */
 export const loadDscContent = (cfg = {}) => {
-    // const { onceEvery = 10 * 1000 } = cfg;
+    const { debug = false } = cfg;
 
     return publish((inputs$) => {
         const url$ = inputs$.pipe(nth(0));
@@ -24,6 +25,7 @@ export const loadDscContent = (cfg = {}) => {
         return inputs$.pipe(
             map(buildDscContentUrl),
             map(prependPath),
+            show(debug, (url) => console.log("Sending collection request:", url)),
             requestWithUpdateTime({ dateStore }),
             pluck("response"),
             withLatestFrom(url$),
diff --git a/client/src/components/History/caching/operators/chunk.js b/client/src/components/History/caching/operators/chunk.js
deleted file mode 100644
index 74a1a7f9cc2..00000000000
--- a/client/src/components/History/caching/operators/chunk.js
+++ /dev/null
@@ -1,42 +0,0 @@
-import { pipe } from "rxjs";
-import { map } from "rxjs/operators";
-
-/**
- * Bust incoming numeric source values into blocks of designated size.
- *
- * @param {*} chunkSize
- * @param {*} ceil
- */
-// prettier-ignore
-export const chunk = (chunkSize, ceil = false) => pipe(
-    map((chunkMe) => {
-        const rawVal = chunkMe / chunkSize;
-        return chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
-    })
-);
-
-/**
- * Change one parameter to be an multiple of indicated block size. Used to
- * regulate the URLs we send to the server so that some will be cached. Pos is
- * the position in the combined inputs array, and the chunkSize is the size of
- * the block to break the value into.
- *
- * Ex: Source Inputs [ x, y, z, 750 ],
- *     chunkParam(3, 200)
- *     Results in [ x, y, z, 600]
- *
- * @param {integer} pos Input array parameter number to chunk
- * @param {integer} chunkSize Size of chunks
- * @param {Boolean} ceil Math.ceil or Math.floor chunk value
- */
-// prettier-ignore
-export const chunkParam = (pos, chunkSize, ceil = false) => pipe(
-    map((inputs) => {
-        const chunkMe = inputs[pos];
-        const rawVal = 1.0 * chunkMe / chunkSize;
-        const chunkedVal = chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
-        const newInputs = inputs.slice();
-        newInputs[pos] = chunkedVal;
-        return newInputs;
-    })
-);
diff --git a/client/src/components/History/model/Dataset.js b/client/src/components/History/model/Dataset.js
index 68ca88bdcea..f5c3b6eae64 100644
--- a/client/src/components/History/model/Dataset.js
+++ b/client/src/components/History/model/Dataset.js
@@ -5,6 +5,7 @@
  */
 
 import { Content } from "./Content";
+import { scrubModelProps } from "utils/safeAssign";
 
 export class Dataset extends Content {
     getUrl(urlType) {
@@ -40,4 +41,28 @@ export class Dataset extends Content {
         const { display_apps = [], display_types = [] } = this;
         return [...display_apps, ...display_types];
     }
+
+    clone() {
+        const newProps = cleanDatasetProps(this);
+        return new Dataset(newProps);
+    }
+
+    patch(newProps) {
+        const cleanProps = cleanDatasetProps({ ...this, ...newProps });
+        return new Dataset(cleanProps);
+    }
+
+    equals(other) {
+        return Dataset.equals(this, other);
+    }
 }
+
+Dataset.equals = function (a, b) {
+    return JSON.stringify(a) == JSON.stringify(b);
+};
+
+const scrubber = scrubModelProps(Dataset);
+export const cleanDatasetProps = (props = {}) => {
+    const cleanProps = JSON.parse(JSON.stringify(props));
+    return scrubber(cleanProps);
+};
diff --git a/client/src/components/History/model/DatasetCollection.js b/client/src/components/History/model/DatasetCollection.js
index 2dc5f3f60fc..196bd186c31 100644
--- a/client/src/components/History/model/DatasetCollection.js
+++ b/client/src/components/History/model/DatasetCollection.js
@@ -7,11 +7,11 @@
 
 import { Content } from "./Content";
 import { JobStateSummary } from "./JobStateSummary";
+import { scrubModelProps } from "utils/safeAssign";
 
 export class DatasetCollection extends Content {
     loadProps(raw = {}) {
         if (!raw.contents_url) {
-            // console.log("ouch", raw);
             throw new Error("missing contents_url", raw);
         }
         super.loadProps(raw);
@@ -64,8 +64,28 @@ export class DatasetCollection extends Content {
     get jobSummary() {
         return new JobStateSummary(this);
     }
+
+    clone() {
+        const newProps = cleanDscProps(this);
+        return new DatasetCollection(newProps);
+    }
+
+    patch(newProps) {
+        const cleanProps = cleanDscProps({ ...this, ...newProps });
+        return new DatasetCollection(cleanProps);
+    }
+
+    equals(other) {
+        return DatasetCollection.equals(this, other);
+    }
 }
 
 DatasetCollection.equals = function (a, b) {
     return JSON.stringify(a) == JSON.stringify(b);
 };
+
+const scrubber = scrubModelProps(DatasetCollection);
+export const cleanDscProps = (props = {}) => {
+    const cleanProps = JSON.parse(JSON.stringify(props));
+    return scrubber(cleanProps);
+};
diff --git a/client/src/components/History/model/History.js b/client/src/components/History/model/History.js
index 5285eecbd19..388db974538 100644
--- a/client/src/components/History/model/History.js
+++ b/client/src/components/History/model/History.js
@@ -1,4 +1,5 @@
 import { dateMixin, ModelBase } from "./ModelBase";
+import { scrubModelProps } from "utils/safeAssign";
 
 export class History extends dateMixin(ModelBase) {
     // not deleted
@@ -36,8 +37,32 @@ export class History extends dateMixin(ModelBase) {
         }
         return status.join(", ");
     }
+
+    get exportLink() {
+        return `histories/${this.id}/export`;
+    }
+
+    clone() {
+        const newProps = cleanHistoryProps(this);
+        return new History(newProps);
+    }
+
+    patch(newProps) {
+        const cleanProps = cleanHistoryProps({ ...this, ...newProps });
+        return new History(cleanProps);
+    }
+
+    equals(other) {
+        return History.equals(this, other);
+    }
 }
 
 History.equals = function (a, b) {
     return JSON.stringify(a) == JSON.stringify(b);
 };
+
+const scrubber = scrubModelProps(History);
+export const cleanHistoryProps = (props = {}) => {
+    const cleanProps = JSON.parse(JSON.stringify(props));
+    return scrubber(cleanProps);
+};
diff --git a/client/src/components/History/model/History.test.js b/client/src/components/History/model/History.test.js
new file mode 100644
index 00000000000..2571b24120e
--- /dev/null
+++ b/client/src/components/History/model/History.test.js
@@ -0,0 +1,101 @@
+import { History, cleanHistoryProps } from "./History";
+
+describe("History model", () => {
+    const initialProps = {
+        id: "abc123",
+        name: "Foo",
+        tags: ["a", "b", "c"],
+        annotation: "I am a test history",
+    };
+
+    const h = new History(initialProps);
+
+    describe("clone", () => {
+        const c = h.clone();
+
+        it("must be able to clone itself for use in editors", () => {
+            // should be == but not ===
+            expect(h).not.toBe(c);
+            expect(c).toBeInstanceOf(History);
+            expect(History.equals(h, c)).toBeTruthy();
+        });
+
+        it("should have the same props before and after cloning", () => {
+            expect(h.id).toEqual(c.id);
+            expect(h.name).toEqual(c.name);
+            expect(h.tags).toEqual(c.tags);
+            expect(h.annotation).toEqual(c.annotation);
+        });
+    });
+
+    describe("cleanHistoryProps", () => {
+        it("should ignore computed prop names during cloning", () => {
+            const sloppyProps = {
+                ...initialProps,
+                // these are all computed values in the history class which are defined by getters,
+                // if we feed thsm into the constructor they should be ignored
+                statusDescription: "123",
+                totalItems: Number.POSITIVE_INFINITY,
+                hidItems: "four",
+                active: "The Fonz",
+            };
+
+            const cleanProps = cleanHistoryProps(sloppyProps);
+            expect(cleanProps).toEqual(initialProps);
+            expect(cleanProps).not.toBe(initialProps);
+
+            expect(cleanProps).not.toHaveProperty("statusDescription");
+            expect(cleanProps).not.toHaveProperty("totalItems");
+            expect(cleanProps).not.toHaveProperty("hidItems");
+            expect(cleanProps).not.toHaveProperty("active");
+
+            const h = new History(cleanProps);
+            expect(h.statusDescription).not.toEqual(sloppyProps.statusDescription);
+        });
+    });
+
+    describe("patch", () => {
+        const patchVals = {
+            name: "i was patched",
+            annotation: "patched annotation",
+            tags: ["foo", "bar", "blech"],
+        };
+
+        const clone = h.patch(patchVals);
+
+        it("should return a new instance of History", () => {
+            expect(h).not.toBe(clone);
+            expect(clone).toBeInstanceOf(History);
+            expect(History.equals(h, clone)).toBeFalsy();
+        });
+
+        it("should patch in passed properties", () => {
+            expect(clone.name).toEqual(patchVals.name);
+            expect(clone.annotation).toEqual(patchVals.annotation);
+            expect(clone.tags).toEqual(patchVals.tags);
+        });
+    });
+
+    describe("equivalence comparison", () => {
+        it("should recognize different props", () => {
+            const hDifferent = new History({ id: "def345" });
+            expect(History.equals(h, hDifferent)).toBeFalsy();
+        });
+
+        it("equivalence should work fine with cloned histories", () => {
+            const clone = h.clone();
+            // different object
+            expect(h).not.toBe(clone);
+            // but equivalent because props the same
+            expect(History.equals(h, clone)).toBeTruthy();
+        });
+
+        it("should work with patch", () => {
+            const clone = h.patch({ id: "floobar" });
+            // different object
+            expect(h).not.toBe(clone);
+            // not equivalent because different val
+            expect(History.equals(h, clone)).toBeFalsy();
+        });
+    });
+});
diff --git a/client/src/components/History/model/crud.js b/client/src/components/History/model/crud.js
index efab5d41316..b74a5ca9178 100644
--- a/client/src/components/History/model/crud.js
+++ b/client/src/components/History/model/crud.js
@@ -8,7 +8,9 @@ import { bulkContentUpdate, getAllContentByFilter } from "./queries";
 export const updateSelectedContent = (updates) => async (history, type_ids) => {
     // single ajax call with wierd syntax because.... reasons
     const changes = await bulkContentUpdate(history, type_ids, updates);
-    await bulkCacheContent(changes);
+    const cacheResult = await bulkCacheContent(changes, true);
+    console.log("cacheResult", changes, cacheResult);
+    return cacheResult;
 };
 
 export const hideSelectedContent = updateSelectedContent({
diff --git a/client/src/components/History/model/queries.js b/client/src/components/History/model/queries.js
index d497434cf3b..3b96eab7918 100644
--- a/client/src/components/History/model/queries.js
+++ b/client/src/components/History/model/queries.js
@@ -8,6 +8,7 @@
 import axios from "axios";
 import moment from "moment";
 import { prependPath } from "utils/redirect";
+import { History } from "./History";
 
 // #region setup & utils
 
@@ -87,9 +88,10 @@ const stdHistoryParams = {
  * Return list of available histories
  */
 export async function getHistoryList() {
-    const params = { view: "summary" };
+    const params = { view: "summary", keys: "size" };
     const response = await api.get("/histories", { params });
-    return doResponse(response);
+    const rawList = doResponse(response);
+    return rawList.map((props) => new History(props));
 }
 
 /**
@@ -101,7 +103,8 @@ export async function getHistoryById(id, since) {
     const sinceParam = since !== undefined ? moment.utc(since).toISOString() : null;
     const url = sinceParam ? `${path}?q=update_time-gt&qv=${sinceParam}` : path;
     const response = await api.get(url, { params: stdHistoryParams });
-    return doResponse(response);
+    const props = doResponse(response);
+    return new History(props);
 }
 
 /**
@@ -121,7 +124,8 @@ export async function createNewHistory(props = {}) {
     if (!id) {
         throw new Error("failed to create and select new history");
     }
-    return doResponse(createResponse);
+    const newHistoryProps = doResponse(createResponse);
+    return new History(newHistoryProps);
 }
 
 /**
@@ -139,7 +143,8 @@ export async function cloneHistory(history, name, copyAll) {
         current: true,
     };
     const response = await api.post(url, payload, { params: stdHistoryParams });
-    return doResponse(response);
+    const clonedProps = doResponse(response);
+    return new History(clonedProps);
 }
 
 /**
@@ -171,7 +176,8 @@ export async function undeleteHistoryById(id) {
 export async function updateHistoryFields(id, payload) {
     const url = `/histories/${id}`;
     const response = await api.put(url, payload, { params: stdHistoryParams });
-    return doResponse(response);
+    const props = doResponse(response);
+    return new History(props);
 }
 
 /**
@@ -202,7 +208,8 @@ export async function getCurrentHistoryFromServer() {
     const response = await api.get(url, {
         baseURL: prependPath("/"), // old api doesn't use api path
     });
-    return doResponse(response);
+    const props = doResponse(response);
+    return new History(props);
 }
 
 export async function setCurrentHistoryOnServer(history_id) {
diff --git a/client/src/components/History/providers/CollectionContentProvider/CollectionContentProvider.js b/client/src/components/History/providers/CollectionContentProvider/CollectionContentProvider.js
index d1c13228b19..5c4c07315fe 100644
--- a/client/src/components/History/providers/CollectionContentProvider/CollectionContentProvider.js
+++ b/client/src/components/History/providers/CollectionContentProvider/CollectionContentProvider.js
@@ -1,10 +1,14 @@
-import { DatasetCollection } from "../../model";
+import { SearchParams, DatasetCollection } from "../../model";
 import { ContentProvider, processContentStreams } from "../ContentProvider";
 import { collectionPayload } from "./collectionPayload";
 
 export default {
     mixins: [ContentProvider],
 
+    props: {
+        params: { type: SearchParams, default: () => new SearchParams({ showHidden: false }) },
+    },
+
     computed: {
         dsc() {
             if (this.parent instanceof DatasetCollection) {
diff --git a/client/src/components/History/providers/CollectionContentProvider/collectionPayload.js b/client/src/components/History/providers/CollectionContentProvider/collectionPayload.js
index 7ba1df9c3de..c0e92bc4501 100644
--- a/client/src/components/History/providers/CollectionContentProvider/collectionPayload.js
+++ b/client/src/components/History/providers/CollectionContentProvider/collectionPayload.js
@@ -1,5 +1,5 @@
 import { of, Observable, Subject, partition, merge } from "rxjs";
-import { tap, map, pluck, switchMap, publish, distinctUntilChanged, share } from "rxjs/operators";
+import { tap, map, pluck, switchMap, publish, distinctUntilChanged, share, throttleTime } from "rxjs/operators";
 import { chunk } from "utils/observable";
 import { SearchParams } from "../../model/SearchParams";
 import { loadDscContent } from "../../caching";
@@ -13,6 +13,8 @@ export const collectionPayload = (cfg = {}) => {
         pageSize = SearchParams.pageSize,
         debug = false,
         loadingEvents$ = new Subject(),
+        debouncePeriod = 100,
+        chunkSize = pageSize
     } = cfg;
 
     const { totalElements: totalMatches, contents_url } = dsc;
@@ -34,7 +36,7 @@ export const collectionPayload = (cfg = {}) => {
 
         // chunk the input to the server request so we don't request for every single mouse move
         const chunked$ = elementIndex$.pipe(
-            chunk(pageSize, false),
+            chunk(chunkSize),
             distinctUntilChanged(),
         );
 
@@ -48,6 +50,7 @@ export const collectionPayload = (cfg = {}) => {
         );
 
         const serverLoad$ = pagination$.pipe(
+            throttleTime(debouncePeriod, undefined, { trailing: true }),
             switchMap(pagination => of([contents_url, filters, pagination]).pipe(
                 tap(() => loadingEvents$.next(true)),
                 loadDscContent({ debug }),
diff --git a/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js b/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js
index 4875fd52970..41bcad3d798 100644
--- a/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js
+++ b/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js
@@ -14,7 +14,7 @@ import {
     timeoutWith,
     ignoreElements,
 } from "rxjs/operators";
-import { chunk, show, debounceBurst, shareButDie } from "utils/observable";
+import { show, debounceBurst, shareButDie } from "utils/observable";
 import { SearchParams } from "../../model/SearchParams";
 import { processContentUpdate, newUpdateMap, buildContentResult, getKeyForUpdateMap } from "./aggregation";
 import { SEEK } from "../../caching/enums";
@@ -41,7 +41,7 @@ export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
         // interval in which we create new monitors
         // if we chunk the inputs then we won't make as many
         // might be easier on memory
-        inputChunk = Math.floor(pageSize / 2),
+        // inputChunk = Math.floor(pageSize / 2),
 
         // reduces duplicate updates to the skiplist
         // which can be expensive
@@ -59,7 +59,7 @@ export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
     // avoids creating a new monitor for every single key emission
     // chunk to ceiling if descending, floor if ascending
     const monitorInputKey$ = targetKey$.pipe(
-        chunk(inputChunk, keyDirection == SEEK.DESC), 
+        // chunk(inputChunk, keyDirection == SEEK.DESC, "monitorInputKey"),
         distinctUntilChanged(),
         show(debug, (key) => console.log("monitorInputKey", key)),
     );
@@ -76,10 +76,10 @@ export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
     // when we're just going to change it again shortly
     const groupedUpdates$ = cacheUpdates$.pipe(
         groupBy(
-            update => update.key, 
-            update => update, 
+            update => update.key,
+            update => update,
             updateByKey$ => updateByKey$.pipe(
-                timeoutWith(updateGroupLifetime, EMPTY), 
+                timeoutWith(updateGroupLifetime, EMPTY),
                 ignoreElements()
             )
         ),
@@ -104,7 +104,7 @@ export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
         withLatestFrom(targetKey$),
     );
 
-    // query skiplist with scroll position (transformed into a key from the skiplist) 
+    // query skiplist with scroll position (transformed into a key from the skiplist)
     // to produce list of nearby content
     const contentList$ = contentListInputs$.pipe(
         map(buildContentResult({ pageSize, keyDirection, getKey })),
diff --git a/client/src/components/History/providers/ContentProvider/processContentStreams.js b/client/src/components/History/providers/ContentProvider/processContentStreams.js
index a589fdb2acb..6b28650ee2b 100644
--- a/client/src/components/History/providers/ContentProvider/processContentStreams.js
+++ b/client/src/components/History/providers/ContentProvider/processContentStreams.js
@@ -1,9 +1,9 @@
 /**
- * General plumbing for the 2 types of content providers. This funcction emits 3 observables:
+ * General plumbing for the 2 types of content providers. This function returns 3 observables:
  * loading, scrolling, and payload. Payload's the most important and the operator that genrates the
  * payload from the inputs is passed in as a parameter to this factory.
  */
-import { debounceTime, distinctUntilChanged, share, startWith, switchMap } from "rxjs/operators";
+import { distinctUntilChanged, share, startWith, switchMap } from "rxjs/operators";
 import { activity, whenAny, show } from "utils/observable";
 import { propMatch } from "./helpers";
 import { SearchParams, ScrollPos } from "../../model";
@@ -23,7 +23,7 @@ import { Subject } from "rxjs";
  */
 // prettier-ignore
 export function processContentStreams(payloadOperator, sources = {}, settings = {}) {
-    const { debouncePeriod, debug = false } = settings;
+    const { debouncePeriod, debug = true } = settings;
 
     // clean incoming source streams
     const parent$ = sources.parent$.pipe(
@@ -35,7 +35,7 @@ export function processContentStreams(payloadOperator, sources = {}, settings =
         show(debug, (params) => console.log('processContentStreams: params changed', params)),
     );
     const pos$ = sources.scrollPos$.pipe(
-        debounceTime(debouncePeriod),
+        // debounceTime(debouncePeriod),
         show(debug, (pos) => console.log('processContentStreams: pos changed', pos)),
         distinctUntilChanged(ScrollPos.equals),
     );
@@ -48,7 +48,7 @@ export function processContentStreams(payloadOperator, sources = {}, settings =
         activity(debouncePeriod),
         startWith(true)
     );
-    
+
     const resetPos$ = new Subject();
 
     // The actual loader, when history or params change we poll against the api and
diff --git a/client/src/components/History/providers/DatasetProvider.js b/client/src/components/History/providers/DatasetProvider.js
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/client/src/components/History/providers/ExpandedItems.js b/client/src/components/History/providers/ExpandedItems.js
index a281794ef35..05c4fdc3452 100644
--- a/client/src/components/History/providers/ExpandedItems.js
+++ b/client/src/components/History/providers/ExpandedItems.js
@@ -1,3 +1,5 @@
+import { saveSet, loadSet } from "./setCache";
+
 export default {
     props: {
         scopeKey: { type: String, required: true },
@@ -8,6 +10,11 @@ export default {
             items: new Set(),
         };
     },
+    computed: {
+        expandedCount() {
+            return this.items.size;
+        },
+    },
     methods: {
         isExpanded(item) {
             const key = this.getItemKey(item);
@@ -29,11 +36,23 @@ export default {
                 this.reset();
             }
         },
+        items(newSet) {
+            saveSet(this.key, newSet);
+        },
+    },
+    created() {
+        this.key = "expanded-history-items";
+        const cachedSet = loadSet(this.key);
+        if (cachedSet) {
+            this.items = cachedSet;
+        }
     },
     render() {
         return this.$scopedSlots.default({
             isExpanded: this.isExpanded,
             setExpanded: this.setExpanded,
+            collapseAll: this.reset,
+            expandedCount: this.expandedCount,
         });
     },
 };
diff --git a/client/src/components/History/providers/HistoryContentProvider/contentPayload.js b/client/src/components/History/providers/HistoryContentProvider/contentPayload.js
index 9da5846b91f..881d9b033eb 100644
--- a/client/src/components/History/providers/HistoryContentProvider/contentPayload.js
+++ b/client/src/components/History/providers/HistoryContentProvider/contentPayload.js
@@ -34,10 +34,10 @@ import { defaultPayload } from "../ContentProvider";
  */
 // prettier-ignore
 export const contentPayload = (cfg = {}) => {
-    const { 
-        parent: history, 
+    const {
+        parent: history,
         filters = new SearchParams(),
-        pageSize = SearchParams.pageSize, 
+        pageSize = SearchParams.pageSize,
         debouncePeriod = 250,
         loadingTimeout = 2000,
         disablePoll = false,
@@ -45,6 +45,7 @@ export const contentPayload = (cfg = {}) => {
         // optional feedback indicators
         loadingEvents$ = new Subject(),
         resetPos$ = new Subject(),
+        chunkSize = pageSize
     } = cfg;
 
     // These running totals are shared between instances of content payload because a lot of the
@@ -78,14 +79,15 @@ export const contentPayload = (cfg = {}) => {
             filter(hid => !isNaN(hid)),
             share(),
         );
-        
+
         // #endregion
 
         // #region server loading
 
         const serverHid$ = hid$.pipe(
-            chunk(pageSize, true),
+            chunk({ chunkSize, debug, label: "serverHid" }),
             distinctUntilChanged(),
+            debounceTime(debouncePeriod),
             show(debug, hid => console.log("serverHid", hid)),
             share(),
         );
@@ -109,7 +111,7 @@ export const contentPayload = (cfg = {}) => {
             mergeAll(),
             debounceTime(100),
         );
-        
+
         const cacheMonitor$ = cacheHid$.pipe(
             watchHistoryContents({ history, filters, pageSize, debouncePeriod, debug }),
             show(debug, (result) => console.log("cacheMonitor.contents (hid)", result.contents.map(o => o.hid))),
@@ -133,7 +135,7 @@ export const contentPayload = (cfg = {}) => {
         // #endregion
 
         // #region Reposition View to accomodate recent updates
-        
+
         // When new updates to the history come in, we should shift the scrollPos so that we can see
         // them. Without some kind of explicit move, the system will think they're just random
         // updates that are happening outside the region of interest, and there would be no
@@ -151,7 +153,7 @@ export const contentPayload = (cfg = {}) => {
                 const fudge = 2;
                 const scrollNearLastTop = Math.abs(hid - lastResponse.maxContentHid) < fudge;
                 const scrollerAtTop = scrollerInDefault || scrollNearLastTop;
-                
+
                 // console.group("should i reposition?");
                 // console.log("updatesAtTop", updatesAtTop);
                 // console.log("scrollerAtTop", scrollerAtTop);
@@ -190,7 +192,7 @@ export const contentPayload = (cfg = {}) => {
             const sub = payload$.subscribe(obs);
 
             // Tie stat feedback subscriptions to main payload sub
-            // Make sure not to combineLatest these subject accumulators or we'll 
+            // Make sure not to combineLatest these subject accumulators or we'll
             // end up with an infinite loop
             sub.add(newCursorToHid$.subscribe(cursorToHid$));
             sub.add(newHidToTopRows$.subscribe(hidToTopRows$));
diff --git a/client/src/components/History/providers/UserHistories/UserHistories.js b/client/src/components/History/providers/UserHistories/UserHistories.js
index e9547e7db02..22ac28a3b7e 100644
--- a/client/src/components/History/providers/UserHistories/UserHistories.js
+++ b/client/src/components/History/providers/UserHistories/UserHistories.js
@@ -43,6 +43,10 @@ export default {
             "loadUserHistories",
             "secureHistory",
         ]),
+
+        updateCurrentHistory(updates) {
+            this.updateHistory({ ...updates, id: this.currentHistory.id });
+        },
     },
     watch: {
         // when user changes reload histories
@@ -81,7 +85,11 @@ export default {
                 // save new history params should be an object with an id property and any additional
                 // properties that are to be updated on the server. A full history object is not required
                 updateHistory: this.updateHistory,
-                updateCurrentHistory: () => this.updateHistory(this.currentHistory),
+                updateCurrentHistory: this.updateCurrentHistory,
+
+                // also conform to .sync event name format
+                "update:history": this.updateHistory,
+                "update:currentHistory": this.updateCurrentHistory,
 
                 // delete history then clear currentHistoryId
                 deleteHistory: (history) => this.deleteHistory({ history }),
diff --git a/client/src/components/History/providers/setCache.js b/client/src/components/History/providers/setCache.js
new file mode 100644
index 00000000000..90d9bbe5b40
--- /dev/null
+++ b/client/src/components/History/providers/setCache.js
@@ -0,0 +1,25 @@
+// Caches expanded/selected Sets in sessionStorage
+// Putting this in separate file for ease of mocking
+
+export function saveSet(key, setObject) {
+    try {
+        const setToArray = Array.from(setObject);
+        const json = JSON.stringify(setToArray);
+        sessionStorage.setItem(key, json);
+    } catch (err) {
+        console.log("error saving stuff to cache", err);
+        sessionStorage.removeItem(key);
+    }
+}
+
+export function loadSet(key) {
+    try {
+        const arrJson = sessionStorage.getItem(key) || [];
+        const cachedArray = JSON.parse(arrJson);
+        return Array.isArray(cachedArray) ? new Set(cachedArray) : new Set();
+    } catch (err) {
+        console.log("erorr loading cached list", err);
+        sessionStorage.removeItem(key);
+        return null;
+    }
+}
diff --git a/client/src/components/IconButton.vue b/client/src/components/IconButton.vue
new file mode 100644
index 00000000000..713e34f9e2e
--- /dev/null
+++ b/client/src/components/IconButton.vue
@@ -0,0 +1,51 @@
+<template>
+    <b-button
+        class="galaxy-icon-button"
+        v-bind="$attrs"
+        v-on="$listeners"
+        :title="title"
+        :size="size"
+        :pressed="pressed"
+        :disabled="disabled"
+        :variant="localVariant"
+    >
+        <Icon :icon="icon" />
+        <span class="sr-only">{{ title | l }}</span>
+    </b-button>
+</template>
+
+<script>
+export default {
+    props: {
+        icon: { type: [String, Array, Object], required: true },
+        title: { type: String, required: true },
+        size: { type: String, required: false, default: "sm" },
+        pressed: { type: Boolean, required: false, default: false },
+        disabled: { type: Boolean, required: false, default: false },
+        // overrides variant set by pressed/disabled
+        variant: { type: String, required: false, default: null },
+    },
+    computed: {
+        localVariant() {
+            if (this.variant) {
+                // override with prop
+                return this.variant;
+            }
+            if (this.disabled) {
+                return "secondary";
+            }
+            return this.pressed ? "info" : "secondary";
+        },
+    },
+};
+</script>
+
+<style lang="scss">
+// removes irritating bootstrap outline
+.galaxy-icon-button.btn {
+    box-shadow: none !important;
+    &.disabled > svg {
+        opacity: 0.5;
+    }
+}
+</style>
diff --git a/client/src/components/Workflow/WorkflowList.vue b/client/src/components/Workflow/WorkflowList.vue
index c6985bbf017..e7a507d3c1c 100644
--- a/client/src/components/Workflow/WorkflowList.vue
+++ b/client/src/components/Workflow/WorkflowList.vue
@@ -59,7 +59,10 @@
                         <font-awesome-icon v-if="row.item.shared" v-b-tooltip.hover title="Shared" icon="share-alt" />
                     </template>
                     <template v-slot:cell(show_in_tool_panel)="row">
-                        <b-form-checkbox v-model="row.item.show_in_tool_panel" @change="bookmarkWorkflow(row.item)" />
+                        <b-form-checkbox
+                            :checked="row.item.show_in_tool_panel"
+                            @change="(checked) => bookmarkWorkflow(row.item, checked)"
+                        />
                     </template>
                     <template v-slot:cell(update_time)="data">
                         <UtcDate :date="data.value" mode="elapsed" />
@@ -192,14 +195,13 @@ export default {
         executeWorkflow: function (workflow) {
             window.location = `${this.root}workflows/run?id=${workflow.id}`;
         },
-        bookmarkWorkflow: function (workflow) {
+        bookmarkWorkflow: function (workflow, checked) {
             // This reloads the whole page, so that the workflow appears in the tool panel.
             // Ideally we would notify only the tool panel of a change
             const id = workflow.id;
-            const show_in_tool_panel = workflow.show_in_tool_panel;
             const tags = workflow.tags;
             const data = {
-                show_in_tool_panel: !show_in_tool_panel,
+                show_in_tool_panel: checked,
                 tags: tags,
             };
             this.services
diff --git a/client/src/components/plugins/icons.js b/client/src/components/plugins/icons.js
new file mode 100644
index 00000000000..348fb5965a9
--- /dev/null
+++ b/client/src/components/plugins/icons.js
@@ -0,0 +1,93 @@
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import {
+    faArrowUp,
+    faBolt,
+    faBurn,
+    faChartArea,
+    faCheckSquare,
+    faClock,
+    faCodeBranch,
+    faColumns,
+    faCompress,
+    faCopy,
+    faDownload,
+    faEllipsisH,
+    faEllipsisV,
+    faExchangeAlt,
+    faEye,
+    faFileArchive,
+    faFileExport,
+    faFilter,
+    faFolder,
+    faInfoCircle,
+    faList,
+    faLink,
+    faLock,
+    faPause,
+    faPen,
+    faPlay,
+    faPlus,
+    faQuestion,
+    faShareAlt,
+    faStream,
+    faTags,
+    faTrash,
+    faTrashRestore,
+    faExclamationTriangle,
+    faUndo,
+    faUserLock,
+    faWrench,
+} from "@fortawesome/free-solid-svg-icons";
+
+library.add(
+    faArrowUp,
+    faBolt,
+    faBurn,
+    faChartArea,
+    faCheckSquare,
+    faClock,
+    faCodeBranch,
+    faColumns,
+    faCompress,
+    faCopy,
+    faDownload,
+    faEllipsisH,
+    faEllipsisV,
+    faExchangeAlt,
+    faEye,
+    faFileArchive,
+    faFileExport,
+    faFilter,
+    faFolder,
+    faInfoCircle,
+    faLink,
+    faList,
+    faLock,
+    faPause,
+    faPen,
+    faPlay,
+    faPlus,
+    faQuestion,
+    faShareAlt,
+    faStream,
+    faTags,
+    faTrash,
+    faTrashRestore,
+    faExclamationTriangle,
+    faUndo,
+    faUserLock,
+    faWrench
+);
+
+export const iconMixin = {
+    components: {
+        Icon: FontAwesomeIcon,
+    },
+};
+
+export const iconPlugin = {
+    install(Vue) {
+        Vue.mixin(iconMixin);
+    },
+};
diff --git a/client/src/components/plugins/index.js b/client/src/components/plugins/index.js
index 5dd87fa1e5f..fa6ee155b98 100644
--- a/client/src/components/plugins/index.js
+++ b/client/src/components/plugins/index.js
@@ -1,6 +1,7 @@
 export { eventHubMixin, eventHubPlugin } from "./eventHub";
 export { vueRxShortcutPlugin, vueRxShortcuts } from "./vueRxShortcuts";
 export { localizationPlugin } from "./localization";
+export { iconPlugin, iconMixin } from "./icons";
 
 // I'm explicitly ommitting this because of its ugly dependencies, I want us to have to import the
 // legacy nav features manually (when required) rather than it coming in every time somebody accesses
diff --git a/client/src/utils/mountVueComponent.js b/client/src/utils/mountVueComponent.js
index 8f685614822..8ef37f8e9dc 100644
--- a/client/src/utils/mountVueComponent.js
+++ b/client/src/utils/mountVueComponent.js
@@ -6,7 +6,7 @@ import Vue from "vue";
 import Vuex from "vuex";
 import BootstrapVue from "bootstrap-vue";
 import store from "../store";
-import { eventHubPlugin, localizationPlugin, vueRxShortcutPlugin } from "components/plugins";
+import { eventHubPlugin, localizationPlugin, vueRxShortcutPlugin, iconPlugin } from "components/plugins";
 
 Vue.use(Vuex);
 
@@ -24,6 +24,9 @@ Vue.use(localizationPlugin);
 // rxjs utilities
 Vue.use(vueRxShortcutPlugin);
 
+// font-awesome svg icon registration/loading
+Vue.use(iconPlugin);
+
 export const mountVueComponent = (ComponentDefinition) => {
     const component = Vue.extend(ComponentDefinition);
     return (propsData, el) => new component({ store, propsData, el });
diff --git a/client/src/utils/observable/chunk.js b/client/src/utils/observable/chunk.js
index 2994b3f0570..850d6358705 100644
--- a/client/src/utils/observable/chunk.js
+++ b/client/src/utils/observable/chunk.js
@@ -3,14 +3,24 @@ import { map } from "rxjs/operators";
 /**
  * Bust incoming numeric source values into blocks of designated size.
  *
- * @param {*} chunkSize
+ * @param {Number|Object} cfg object or numeric chunksize
  * @param {*} ceil
  */
 // prettier-ignore
-export const chunk = (chunkSize, ceil = false) => {
+export const chunk = (cfg) => {
+    const settings = Number.isInteger(cfg) ? { chunkSize: Math.round(cfg) } : cfg;
+    const { chunkSize = 0, ceil = false, debug = false, label } = settings;
+    if (chunkSize == 0) {
+        throw new Error("Please provide a chunk size");
+    }
+
     return map((chunkMe) => {
-        const rawVal = chunkMe / chunkSize;
-        return chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
+        const rawVal = 1.0 * chunkMe / chunkSize;
+        const result = chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
+        if (debug) {
+            console.log(`chunk: ${label}`, chunkMe, result, settings);
+        }
+        return result;
     })
 };
 
diff --git a/client/src/utils/safeAssign.js b/client/src/utils/safeAssign.js
index 52d4cff9d0b..bef98ecefdd 100644
--- a/client/src/utils/safeAssign.js
+++ b/client/src/utils/safeAssign.js
@@ -9,3 +9,22 @@ export function safeAssign(target, source = {}) {
         .filter((prop) => Object.prototype.hasOwnProperty.call(target, prop))
         .forEach((prop) => (target[prop] = source[prop]));
 }
+
+/**
+ * Creates a reusable utility scrubber function that removes reserved initialization props from raw
+ * which are already defined by Klass. This lets us just stick one model instance into.
+ *
+ * Ex: const scrubber = scrubModelProps(History); *
+ *     const newHistory = new History(scrubber(oldHistory));
+ */
+
+export function scrubModelProps(Klass) {
+    const restrictedKeys = new Set(Object.getOwnPropertyNames(Klass.prototype));
+
+    return (raw = {}) => {
+        const pairs = Object.entries(raw || {});
+        return pairs.reduce((clean, [propName, val]) => {
+            return restrictedKeys.has(propName) ? clean : { ...clean, [propName]: val };
+        }, {});
+    };
+}
diff --git a/client/tests/jest/helpers.js b/client/tests/jest/helpers.js
index 369dcaf51a8..6da42993a13 100644
--- a/client/tests/jest/helpers.js
+++ b/client/tests/jest/helpers.js
@@ -7,6 +7,7 @@ import { createLocalVue, shallowMount } from "@vue/test-utils";
 import { localizationPlugin } from "components/plugins/localization";
 import { vueRxShortcutPlugin } from "components/plugins/vueRxShortcuts";
 import { eventHubPlugin } from "components/plugins/eventHub";
+import { iconPlugin } from "components/plugins/icons";
 import BootstrapVue from "bootstrap-vue";
 import Vuex from "vuex";
 
@@ -95,6 +96,7 @@ export function getLocalVue() {
     localVue.use(localizationPlugin);
     localVue.use(vueRxShortcutPlugin);
     localVue.use(eventHubPlugin);
+    localVue.use(iconPlugin);
     localVue.directive("b-tooltip", mockedDirective);
     localVue.directive("b-popover", mockedDirective);
     return localVue;
@@ -137,7 +139,7 @@ export const untilNthEmission = (src$, n = 1, safetyTimeout = 2000) => {
         let result;
         // prettier-ignore
         src$.pipe(
-            take(n), 
+            take(n),
             takeUntil(timer(safetyTimeout))
         ).subscribe({
             next: (val) => (result = val),
diff --git a/client/yarn.lock b/client/yarn.lock
index 52d83e861ff..6b9ca8eda87 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -1719,14 +1719,14 @@
     "@nodelib/fs.scandir" "2.1.3"
     fastq "^1.6.0"
 
-"@nuxt/opencollective@^0.3.0":
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/@nuxt/opencollective/-/opencollective-0.3.0.tgz#11d8944dcf2d526e31660bb69570be03f8fb72b7"
-  integrity sha512-Vf09BxCdj1iT2IRqVwX5snaY2WCTkvM0O4cWWSO1ThCFuc4if0Q/nNwAgCxRU0FeYHJ7DdyMUNSdswCLKlVqeg==
+"@nuxt/opencollective@^0.3.2":
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/@nuxt/opencollective/-/opencollective-0.3.2.tgz#83cb70cdb2bac5fad6f8c93529e7b11187d49c02"
+  integrity sha512-XG7rUdXG9fcafu9KTDIYjJSkRO38EwjlKYIb5TQ/0WDbiTUTtUtgncMscKOYzfsY86kGs05pAuMOR+3Fi0aN3A==
   dependencies:
-    chalk "^2.4.2"
-    consola "^2.10.1"
-    node-fetch "^2.6.0"
+    chalk "^4.1.0"
+    consola "^2.15.0"
+    node-fetch "^2.6.1"
 
 "@sentry/browser@^5.20.1":
   version "5.20.1"
@@ -3323,18 +3323,18 @@ boolbase@^1.0.0, boolbase@~1.0.0:
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
 
-bootstrap-vue@^2.15.0:
-  version "2.15.0"
-  resolved "https://registry.yarnpkg.com/bootstrap-vue/-/bootstrap-vue-2.15.0.tgz#0dfc12c054496c0f10efed510da1def41697cf3c"
-  integrity sha512-ncxWkDG0mKFVot314wWKJELi+ESO7k6ngV//qvJFs9iVzlFI8Hx3rBVbpcPW2vrJ+0vitH8N2SOwn4fdQ3frMQ==
+bootstrap-vue@^2.21.2:
+  version "2.21.2"
+  resolved "https://registry.yarnpkg.com/bootstrap-vue/-/bootstrap-vue-2.21.2.tgz#ec38f66c3a2205becccddb6158a991d96509ed0b"
+  integrity sha512-0Exe+4MZysqhZNXIKf4TzkvXaupxh9EHsoCRez0o5Dc0J7rlafayOEwql63qXv74CgZO8E4U8ugRNJko1vMvNw==
   dependencies:
-    "@nuxt/opencollective" "^0.3.0"
-    bootstrap ">=4.5.0 <5.0.0"
+    "@nuxt/opencollective" "^0.3.2"
+    bootstrap ">=4.5.3 <5.0.0"
     popper.js "^1.16.1"
     portal-vue "^2.1.7"
     vue-functional-data-merge "^3.1.0"
 
-bootstrap@4.5.0, "bootstrap@>=4.5.0 <5.0.0":
+bootstrap@4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-4.5.0.tgz#97d9dbcb5a8972f8722c9962483543b907d9b9ec"
   integrity sha512-Z93QoXvodoVslA+PWNdk23Hze4RBYIkpb5h8I2HY2Tu2h7A0LpAgLcyrhrSUyo2/Oxm2l1fRZPs1e5hnxnliXA==
@@ -3344,6 +3344,11 @@ bootstrap@>=4.0.0-beta:
   resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-5.0.2.tgz#aff23d5e0e03c31255ad437530ee6556e78e728e"
   integrity sha512-1Ge963tyEQWJJ+8qtXFU6wgmAVj9gweEjibUdbmcCEYsn38tVwRk8107rk2vzt6cfQcRr3SlZ8aQBqaD8aqf+Q==
 
+"bootstrap@>=4.5.3 <5.0.0":
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-4.6.0.tgz#97b9f29ac98f98dfa43bf7468262d84392552fd7"
+  integrity sha512-Io55IuQY3kydzHtbGvQya3H+KorS/M9rSNyfCGCg9WZ4pyT/lCxIlpJgG1GXW/PswzC84Tr2fBYi+7+jFVQQBw==
+
 brace-expansion@^1.1.7:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -3826,6 +3831,14 @@ chalk@^4.0.0, chalk@~4.1.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
+chalk@^4.1.0:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
+  integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
+  dependencies:
+    ansi-styles "^4.1.0"
+    supports-color "^7.1.0"
+
 char-regex@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
@@ -4390,10 +4403,10 @@ connect@^3.7.0:
     parseurl "~1.3.3"
     utils-merge "1.0.1"
 
-consola@^2.10.1:
-  version "2.10.1"
-  resolved "https://registry.yarnpkg.com/consola/-/consola-2.10.1.tgz#4693edba714677c878d520e4c7e4f69306b4b927"
-  integrity sha512-4sxpH6SGFYLADfUip4vuY65f/gEogrzJoniVhNUYkJHtng0l8ZjnDCqxxrSVRHOHwKxsy8Vm5ONZh1wOR3/l/w==
+consola@^2.15.0:
+  version "2.15.3"
+  resolved "https://registry.yarnpkg.com/consola/-/consola-2.15.3.tgz#2e11f98d6a4be71ff72e0bdf07bd23e12cb61550"
+  integrity sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==
 
 console-browserify@^1.1.0:
   version "1.1.0"
@@ -10086,7 +10099,7 @@ node-fetch@^1.0.1:
     encoding "^0.1.11"
     is-stream "^1.0.1"
 
-node-fetch@^2.2.0:
+node-fetch@^2.2.0, node-fetch@^2.6.1:
   version "2.6.1"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
@@ -11691,6 +11704,11 @@ prettier@^2.3.0:
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.0.tgz#b6a5bf1284026ae640f17f7ff5658a7567fc0d18"
   integrity sha512-kXtO4s0Lz/DW/IJ9QdWhAf7/NmPWQXkFr/r/WkR3vyI+0v8amTDxiaQSLzs8NBlytfLWX/7uQUMIW677yLKl4w==
 
+pretty-bytes@^5.6.0:
+  version "5.6.0"
+  resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb"
+  integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==
+
 pretty-format@^25.2.1, pretty-format@^25.5.0:
   version "25.5.0"
   resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-25.5.0.tgz#7873c1d774f682c34b8d48b6743a2bf2ac55791a"

From 059f3776e6441cf073cd9ee871e5f162b09f77e4 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Tue, 17 Aug 2021 13:56:31 -0700
Subject: [PATCH 377/452] cleanup, removed some non-essential edits

---
 client/src/components/History/ContentFilters.vue            | 2 +-
 client/src/components/History/ContentItem/ContentItem.js    | 5 ++---
 .../src/components/History/ContentItem/Dataset/Dataset.vue  | 2 --
 .../components/History/ContentItem/Dataset/DatasetUI.vue    | 2 +-
 client/src/components/History/ContentItem/index.js          | 3 +++
 client/src/components/History/CurrentHistoryPanel.vue       | 1 -
 .../providers/ContentProvider/aggregateCacheUpdates.js      | 6 ------
 .../providers/ContentProvider/processContentStreams.js      | 3 +--
 client/src/components/History/providers/DatasetProvider.js  | 0
 9 files changed, 8 insertions(+), 16 deletions(-)
 delete mode 100644 client/src/components/History/providers/DatasetProvider.js

diff --git a/client/src/components/History/ContentFilters.vue b/client/src/components/History/ContentFilters.vue
index 03c9536728f..da1663d41b8 100644
--- a/client/src/components/History/ContentFilters.vue
+++ b/client/src/components/History/ContentFilters.vue
@@ -1,7 +1,7 @@
 <template>
     <b-input-group v-if="params">
         <DebouncedInput v-model.trim="filterText" v-slot="{ value, input }">
-            <b-form-input size="sm" :value="value" @input="input" :placeholder="'Content Filter' | localize" />
+            <b-form-input size="sm" :value="value" @input="input" :placeholder="'Search Filter' | localize" />
         </DebouncedInput>
 
         <b-input-group-append>
diff --git a/client/src/components/History/ContentItem/ContentItem.js b/client/src/components/History/ContentItem/ContentItem.js
index b39fd514d4e..1dd82051f2a 100644
--- a/client/src/components/History/ContentItem/ContentItem.js
+++ b/client/src/components/History/ContentItem/ContentItem.js
@@ -13,7 +13,6 @@ import Dataset from "./Dataset";
 import DatasetCollection from "./DatasetCollection";
 import Subdataset from "./Subdataset";
 import Subcollection from "./Subcollection";
-import "./styles.scss";
 
 export default {
     template: `
@@ -59,8 +58,8 @@ export default {
             if (this.suppressFocus) {
                 return;
             }
-            const scrollContainer = this.$el.closest(".scroller");
-            const el = scrollContainer.querySelector(`[tabindex="${index}"]`);
+            const ul = this.$el.closest(".scroller");
+            const el = ul.querySelector(`[tabindex="${index}"]`);
             if (el) {
                 el.focus();
             }
diff --git a/client/src/components/History/ContentItem/Dataset/Dataset.vue b/client/src/components/History/ContentItem/Dataset/Dataset.vue
index 14b3c358276..cc00d07b0d4 100644
--- a/client/src/components/History/ContentItem/Dataset/Dataset.vue
+++ b/client/src/components/History/ContentItem/Dataset/Dataset.vue
@@ -4,7 +4,6 @@
         v-bind="$attrs"
         v-on="$listeners"
         :dataset="dataset"
-        :row-key="rowKey"
         @update:dataset="onUpdate"
         @delete="onDelete"
         @undelete="onUndelete"
@@ -27,7 +26,6 @@ export default {
     },
     props: {
         item: { type: Object, required: true },
-        rowKey: { type: [Number, String], required: false, default: "" },
     },
     computed: {
         dataset() {
diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 2b15be928e9..e36d2c8d2e1 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -8,7 +8,7 @@ either through the props, and make updates through the events -->
 <template>
     <div
         class="dataset history-content"
-        :data-content-type-id="typeId"
+        :id="typeId"
         :class="{ expanded, collapsed, selected }"
         :data-state="dataset.state"
         @keydown.arrow-left.self.stop="$emit('update:expanded', false)"
diff --git a/client/src/components/History/ContentItem/index.js b/client/src/components/History/ContentItem/index.js
index 5543ce9497a..32d866ab9d0 100644
--- a/client/src/components/History/ContentItem/index.js
+++ b/client/src/components/History/ContentItem/index.js
@@ -1,6 +1,9 @@
 /**
  * Scrolling components for history contents and collection contents.
  */
+
+import "./styles.scss";
+
 export { default as ContentItem } from "./ContentItem";
 export { default as CollectionContentItem } from "./CollectionContentItem";
 export { default as HistoryContentItem } from "./HistoryContentItem";
diff --git a/client/src/components/History/CurrentHistoryPanel.vue b/client/src/components/History/CurrentHistoryPanel.vue
index b54c8619a55..86011f0b5a4 100644
--- a/client/src/components/History/CurrentHistoryPanel.vue
+++ b/client/src/components/History/CurrentHistoryPanel.vue
@@ -12,7 +12,6 @@
                 </template>
             </HistoryPanel>
 
-            <!-- still loading -->
             <div v-else class="flex-grow-1 loadingBackground h-100">
                 <span class="sr-only" v-localize>Loading History...</span>
             </div>
diff --git a/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js b/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js
index 41bcad3d798..8024fd20b7b 100644
--- a/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js
+++ b/client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js
@@ -38,11 +38,6 @@ export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
         keyField = "hid",
         keyDirection = SEEK.DESC,
 
-        // interval in which we create new monitors
-        // if we chunk the inputs then we won't make as many
-        // might be easier on memory
-        // inputChunk = Math.floor(pageSize / 2),
-
         // reduces duplicate updates to the skiplist
         // which can be expensive
         updateGroupLifetime = 2 * debouncePeriod,
@@ -59,7 +54,6 @@ export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
     // avoids creating a new monitor for every single key emission
     // chunk to ceiling if descending, floor if ascending
     const monitorInputKey$ = targetKey$.pipe(
-        // chunk(inputChunk, keyDirection == SEEK.DESC, "monitorInputKey"),
         distinctUntilChanged(),
         show(debug, (key) => console.log("monitorInputKey", key)),
     );
diff --git a/client/src/components/History/providers/ContentProvider/processContentStreams.js b/client/src/components/History/providers/ContentProvider/processContentStreams.js
index 6b28650ee2b..79baa25a528 100644
--- a/client/src/components/History/providers/ContentProvider/processContentStreams.js
+++ b/client/src/components/History/providers/ContentProvider/processContentStreams.js
@@ -23,7 +23,7 @@ import { Subject } from "rxjs";
  */
 // prettier-ignore
 export function processContentStreams(payloadOperator, sources = {}, settings = {}) {
-    const { debouncePeriod, debug = true } = settings;
+    const { debouncePeriod, debug = false } = settings;
 
     // clean incoming source streams
     const parent$ = sources.parent$.pipe(
@@ -35,7 +35,6 @@ export function processContentStreams(payloadOperator, sources = {}, settings =
         show(debug, (params) => console.log('processContentStreams: params changed', params)),
     );
     const pos$ = sources.scrollPos$.pipe(
-        // debounceTime(debouncePeriod),
         show(debug, (pos) => console.log('processContentStreams: pos changed', pos)),
         distinctUntilChanged(ScrollPos.equals),
     );
diff --git a/client/src/components/History/providers/DatasetProvider.js b/client/src/components/History/providers/DatasetProvider.js
deleted file mode 100644
index e69de29bb2d..00000000000

From 622ce6e64fdf5787fcb796037c14dfb063ef73bc Mon Sep 17 00:00:00 2001
From: Adrian Zimmer <z.adrian1995@gmail.com>
Date: Wed, 18 Aug 2021 16:21:41 +0200
Subject: [PATCH 378/452] Refactor

---
 lib/galaxy/managers/group_roles.py           | 11 +++++------
 lib/galaxy/webapps/galaxy/api/group_roles.py | 15 +++++++++++----
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py
index e6a484c26e9..2a149dd7df9 100644
--- a/lib/galaxy/managers/group_roles.py
+++ b/lib/galaxy/managers/group_roles.py
@@ -37,10 +37,9 @@ class GroupRolesManager:
         group = self._get_group(trans, group_id)
         role = self._get_role(trans, role_id)
         group_role = self._get_group_role(trans, group, role)
-        if group_role is None:
+        if not group_role:
             raise ObjectNotFound(f"Role {role.name} not in group {group.name}")
         return role
-        # return self._serialize_group_role(group_id, role)
 
     def update(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField):
         """
@@ -50,7 +49,7 @@ class GroupRolesManager:
         group = self._get_group(trans, group_id)
         role = self._get_role(trans, role_id)
         group_role = self._get_group_role(trans, group, role)
-        if group_role is None:
+        if not group_role:
             self._add_role_to_group(trans, group, role)
         return role
 
@@ -62,7 +61,7 @@ class GroupRolesManager:
         group = self._get_group(trans, group_id)
         role = self._get_role(trans, role_id)
         group_role = self._get_group_role(trans, group, role)
-        if group_role is None:
+        if not group_role:
             raise ObjectNotFound(f"Role {role.name} not in group {group.name}")
         self._remove_role_from_group(trans, group_role)
         return role
@@ -70,14 +69,14 @@ class GroupRolesManager:
     def _get_group(self, trans: ProvidesAppContext, encoded_group_id: EncodedDatabaseIdField) -> Any:
         decoded_group_id = decode_id(self._app, encoded_group_id)
         group = trans.sa_session.query(model.Group).get(decoded_group_id)
-        if group is None:
+        if not group:
             raise ObjectNotFound(f"Group with id {encoded_group_id} was not found.")
         return group
 
     def _get_role(self, trans: ProvidesAppContext, encoded_role_id: EncodedDatabaseIdField) -> model.Role:
         decoded_role_id = decode_id(self._app, encoded_role_id)
         role = trans.sa_session.query(model.Role).get(decoded_role_id)
-        if role is None:
+        if not role:
             raise ObjectNotFound(f"Role with id {encoded_role_id} was not found.")
         return role
 
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 4d73f0f511a..2aeebfdf53a 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -62,28 +62,35 @@ class FastAPIGroupRoles:
     @router.get('/api/groups/{group_id}/roles',
                 require_admin=True,
                 summary='Displays a collection (list) of groups.')
-    def index(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
+    def index(self, trans: ProvidesAppContext = DependsOnTrans,
+              group_id: EncodedDatabaseIdField = GroupIDParam) -> GroupRoleListModel:
         group_roles = self.manager.index(trans, group_id)
         return GroupRoleListModel(__root__=[group_role_to_model(trans, group_id, gr.role) for gr in group_roles])
 
     @router.get('/api/groups/{group_id}/roles/{role_id}',
                 require_admin=True,
                 summary='Displays information about a group role.')
-    def show(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
+    def show(self, trans: ProvidesAppContext = DependsOnTrans,
+             group_id: EncodedDatabaseIdField = GroupIDParam,
+             role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
         role = self.manager.show(trans, role_id, group_id)
         return group_role_to_model(trans, group_id, role)
 
     @router.put('/api/groups/{group_id}/roles/{role_id}',
                 require_admin=True,
                 summary='Adds a role to a group')
-    def update(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
+    def update(self, trans: ProvidesAppContext = DependsOnTrans,
+               group_id: EncodedDatabaseIdField = GroupIDParam,
+               role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
         role = self.manager.update(trans, role_id, group_id)
         return group_role_to_model(trans, group_id, role)
 
     @router.delete('/api/groups/{group_id}/roles/{role_id}',
                    require_admin=True,
                    summary='Removes a role from a group')
-    def delete(self, trans: ProvidesAppContext = DependsOnTrans, group_id: EncodedDatabaseIdField = GroupIDParam, role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
+    def delete(self, trans: ProvidesAppContext = DependsOnTrans,
+               group_id: EncodedDatabaseIdField = GroupIDParam,
+               role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
         role = self.manager.delete(trans, role_id, group_id)
         return group_role_to_model(trans, group_id, role)
 

From 84fa44a12451ec71d01ee3ec01c2777dc1e79b34 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Wed, 18 Aug 2021 18:44:06 +0200
Subject: [PATCH 379/452] webpack dev server on remote galaxy server

---
 client/README.md         | 4 ++++
 client/webpack.config.js | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/client/README.md b/client/README.md
index e091c142848..1483f95d857 100644
--- a/client/README.md
+++ b/client/README.md
@@ -83,6 +83,10 @@ environment variable:
 
     GALAXY_URL="http://localhost:8000" make client-dev-server
 
+Sometimes you want to run your UI against a remote Galaxy server. This is also possible, if you enable `CHANGE_ORIGIN` flag 
+
+    CHANGE_ORIGIN=true GALAXY_URL="https://usegalaxy.org/" make client-dev-server
+
 ## Changing Styles/CSS
 
 Galaxy uses Sass for its styling, which is a superset of CSS that compiles down
diff --git a/client/webpack.config.js b/client/webpack.config.js
index 1fd253866ae..4eb2694f59c 100644
--- a/client/webpack.config.js
+++ b/client/webpack.config.js
@@ -232,6 +232,8 @@ module.exports = (env = {}, argv = {}) => {
             proxy: {
                 "/": {
                     target: process.env.GALAXY_URL || "http://localhost:8080",
+                    secure: process.env.CHANGE_ORIGIN ? !process.env.CHANGE_ORIGIN : true,
+                    changeOrigin: !!process.env.CHANGE_ORIGIN,
                 },
             },
         },

From d96ee3c14c0e02c386a630c0e2e8e364d18cc5b7 Mon Sep 17 00:00:00 2001
From: Oleg Zharkov <oleg.zharkov22@gmail.com>
Date: Wed, 18 Aug 2021 18:47:35 +0200
Subject: [PATCH 380/452] wording

---
 client/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/README.md b/client/README.md
index 1483f95d857..1295ec3a169 100644
--- a/client/README.md
+++ b/client/README.md
@@ -83,7 +83,7 @@ environment variable:
 
     GALAXY_URL="http://localhost:8000" make client-dev-server
 
-Sometimes you want to run your UI against a remote Galaxy server. This is also possible, if you enable `CHANGE_ORIGIN` flag 
+Sometimes you want to run your local UI against a remote Galaxy server. This is also possible, if you enable `CHANGE_ORIGIN` flag 
 
     CHANGE_ORIGIN=true GALAXY_URL="https://usegalaxy.org/" make client-dev-server
 

From 853d7e978d765cf0fefe8f8285e62a9a4b451287 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 18 Aug 2021 12:52:06 -0400
Subject: [PATCH 381/452] API test case to ensure dataset names get escaped
 during tool evaluation.

I think other sanitization properties aren't testing that model properties are escaped.
---
 lib/galaxy_test/api/test_tools.py           | 36 ++++++++++++++++++++-
 test/functional/tools/model_attributes.xml  | 15 +++++++++
 test/functional/tools/samples_tool_conf.xml |  1 +
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 test/functional/tools/model_attributes.xml

diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 4d93a920943..1d583c4d6da 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -6,7 +6,7 @@ import zipfile
 from io import BytesIO
 
 import pytest
-from requests import get
+from requests import get, put
 
 from galaxy.util import galaxy_root_path
 from galaxy_test.base import rules_test_data
@@ -308,6 +308,40 @@ class ToolsTestCase(ApiTestCase, TestsTools):
         self._assert_has_keys(tool_info, "inputs", "outputs", "panel_section_id")
         return tool_info
 
+    @skip_without_tool("model_attributes")
+    @uses_test_history(require_new=False)
+    def test_model_attributes_sanitization(self, history_id):
+        cool_name_with_quote = "cool name with a quo\"te"
+        cool_name_without_quote = "cool name with a quo__dq__te"
+
+        current_user = self._get("users/current").json()
+        user_info_url = self._api_url(f"users/{current_user['id']}/information/inputs", use_key=True)
+        put_response = put(user_info_url, data=json.dumps({"address_0|desc": cool_name_with_quote}))
+        put_response.raise_for_status()
+
+        response = get(user_info_url).json()
+        self.assertEqual(len(response["addresses"]), 1)
+        self.assertEqual(response["addresses"][0]["desc"], cool_name_with_quote)
+
+        hda1 = self.dataset_populator.new_dataset(history_id, content='1\t2\t3', name=cool_name_with_quote)
+        assert hda1["name"] == cool_name_with_quote
+
+        rval = self._run(
+            tool_id="model_attributes",
+            inputs={"input1": dataset_to_param(hda1)},
+            history_id=history_id,
+            assert_ok=True,
+            wait_for_job=True,
+        )
+        sanitized_dataset_name = self.dataset_populator.get_history_dataset_content(history_id, dataset=rval["outputs"][0])
+        assert sanitized_dataset_name.strip() == cool_name_without_quote
+
+        sanitized_email = self.dataset_populator.get_history_dataset_content(history_id, dataset=rval["outputs"][1])
+        assert '"' not in sanitized_email
+
+        sanitized_address = self.dataset_populator.get_history_dataset_content(history_id, dataset=rval["outputs"][2])
+        assert sanitized_address.strip() == cool_name_without_quote
+
     @skip_without_tool("composite_output")
     def test_test_data_filepath_security(self):
         test_data_response = self._get(f"tools/{'composite_output'}/test_data_path?filename=../CONTRIBUTORS.md", admin=True)
diff --git a/test/functional/tools/model_attributes.xml b/test/functional/tools/model_attributes.xml
new file mode 100644
index 00000000000..a43b3de501f
--- /dev/null
+++ b/test/functional/tools/model_attributes.xml
@@ -0,0 +1,15 @@
+<tool id="model_attributes" name="model_attributes" version="0.1.0">
+    <command><![CDATA[
+        echo '$input1.name' > "$out_file1";
+        echo '$__user__.email' > "$out_file2";
+        echo '$__user__.addresses[0].desc' > "$out_file3";
+    ]]></command>
+    <inputs>
+        <param name="input1" type="data" label="Dataset with a name to output"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+        <data name="out_file2" format="txt" />
+        <data name="out_file3" format="txt" />
+    </outputs>
+</tool>
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index 7c82aca504e..9bef64673f0 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -195,6 +195,7 @@
   <tool file="cheetah_problem_unbound_var.xml" />
   <tool file="cheetah_problem_unbound_var_input.xml" />
   <tool file="cheetah_problem_syntax_error.xml" />
+  <tool file="model_attributes.xml" />
   <tool file="python_environment_problem.xml" />
   <tool file="config_vars.xml" />
 

From 341658c1eeb87b3068b7d2267c954e04c3fce366 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Wed, 18 Aug 2021 13:01:22 -0700
Subject: [PATCH 382/452] started fixing selenium tests in workflow_run tests

---
 client/src/components/History/EditorMenu.vue  |  2 +
 .../src/components/History/HistoriesMenu.vue  |  2 +-
 client/src/components/History/History.vue     |  1 +
 .../src/components/History/HistoryDetails.vue |  6 ++-
 lib/galaxy/selenium/navigates_galaxy.py       | 45 +++++++++++++------
 lib/galaxy/selenium/navigation.yml            |  2 +
 6 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/client/src/components/History/EditorMenu.vue b/client/src/components/History/EditorMenu.vue
index a3c84878860..8ab53810e1b 100644
--- a/client/src/components/History/EditorMenu.vue
+++ b/client/src/components/History/EditorMenu.vue
@@ -6,6 +6,7 @@
             :pressed="editing"
             icon="pen"
             @click="$emit('update:editing', !editing)"
+            data-description="history editor toggle"
         />
 
         <template v-if="editing">
@@ -16,6 +17,7 @@
                 :class="{ disabled: !valid }"
                 icon="save"
                 @click="$emit('save')"
+                data-description="history editor save button"
             />
 
             <!-- revert -->
diff --git a/client/src/components/History/HistoriesMenu.vue b/client/src/components/History/HistoriesMenu.vue
index 8e88a20cf3a..f9cb8d3d047 100644
--- a/client/src/components/History/HistoriesMenu.vue
+++ b/client/src/components/History/HistoriesMenu.vue
@@ -28,7 +28,7 @@
                 <span v-localize>Change the Current History</span>
             </b-dropdown-item>
 
-            <b-dropdown-item @click="$emit('createNewHistory')">
+            <b-dropdown-item data-description="create new history" @click="$emit('createNewHistory')">
                 <Icon class="mr-1" icon="plus" />
                 <span v-localize>Create a New History</span>
             </b-dropdown-item>
diff --git a/client/src/components/History/History.vue b/client/src/components/History/History.vue
index 48b1013848c..516147616a0 100644
--- a/client/src/components/History/History.vue
+++ b/client/src/components/History/History.vue
@@ -82,6 +82,7 @@
                                     :selected="isSelected(item)"
                                     @update:selected="setSelected(item, $event)"
                                     @viewCollection="$emit('viewCollection', item)"
+                                    :data-hid="item.hid"
                                 />
                             </template>
                         </Scroller>
diff --git a/client/src/components/History/HistoryDetails.vue b/client/src/components/History/HistoryDetails.vue
index a9bc27bf1e7..c443219632e 100644
--- a/client/src/components/History/HistoryDetails.vue
+++ b/client/src/components/History/HistoryDetails.vue
@@ -5,8 +5,8 @@ Emit new History model object when done via .sync syntax
 
 <template>
     <section class="history-details d-flex">
-        <div class="flex-grow-1">
-            <h3>{{ historyName || "(History Name)" }}</h3>
+        <div class="flex-grow-1 overflow-hidden">
+            <h3 data-description="history name display">{{ historyName || "(History Name)" }}</h3>
             <h5 class="history-size">{{ history.size | niceFileSize }}</h5>
 
             <!-- display title, annotation, tags -->
@@ -23,6 +23,7 @@ Emit new History model object when done via .sync syntax
                     placeholder="History Name"
                     trim
                     max-rows="4"
+                    data-description="history name input"
                 ></b-textarea>
 
                 <b-textarea
@@ -31,6 +32,7 @@ Emit new History model object when done via .sync syntax
                     placeholder="Annotation (optional)"
                     trim
                     max-rows="4"
+                    data-description="history annotation input"
                 ></b-textarea>
 
                 <HistoryTags class="mt-3" :history="history" />
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index a588542594a..9da66468f5f 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -372,9 +372,12 @@ class NavigatesGalaxy(HasDriver):
         return history_item_selector
 
     def history_panel_wait_for_hid_state(self, hid, state, allowed_force_refreshes=0, multi_history_panel=False):
-        history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes, multi_history_panel=multi_history_panel)
-        # history_item_selector_state = history_item_selector.with_class(f"state-{state}")
-        history_item_selector_state = history_item_selector.with_data("state", state)
+        if self.is_beta_history():
+            history_item_selector_state = self.components.history_panel.item.by_state_and_hid(state='ok', hid=hid)
+        else:
+            history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes, multi_history_panel=multi_history_panel)
+            # history_item_selector_state = history_item_selector.with_class(f"state-{state}")
+            history_item_selector_state = history_item_selector.with_data("state", state)
         try:
             self.history_item_wait_for(history_item_selector_state, allowed_force_refreshes)
         except self.TimeoutException as e:
@@ -1245,8 +1248,18 @@ class NavigatesGalaxy(HasDriver):
     def is_beta_history(self):
         return not self.components.history_panel.beta.is_absent
 
+    # avoids problematic ID and classes on markup
+    def beta_element(self, attribute_value, attribute_name="data-description"):
+        return self.components.history_panel.by_attribute(name=attribute_name, value=attribute_value)
+
     def history_click_create_new(self):
-        self.components.history_panel.new_history_button.wait_for_and_click()
+        if not self.is_beta_history():
+            self.components.history_panel.new_history_button.wait_for_and_click()
+        else:
+            dropdown = self.beta_element("histories operation menu")
+            dropdown.wait_for_and_click()
+            option = self.beta_element("create new history")
+            option.wait_for_and_click()
 
     def history_panel_click_copy_elements(self):
         if self.is_beta_history():
@@ -1287,21 +1300,25 @@ class NavigatesGalaxy(HasDriver):
             time.sleep(.5)
 
     def history_panel_rename(self, new_name):
-        editable_text_input_element = self.history_panel_click_to_rename()
         if self.is_beta_history():
+            toggle = self.beta_element("history editor toggle")
+            toggle.wait_for_and_click()
+            editable_text_input_element = self.beta_element("history name input").wait_for_visible()
+            self.sleep_for_seconds(1)
             editable_text_input_element.clear()
-        editable_text_input_element.send_keys(new_name)
-        self.send_enter(editable_text_input_element)
+            editable_text_input_element.send_keys(new_name)
+            self.send_enter(editable_text_input_element)
+            save_button = self.beta_element("history editor save button")
+            save_button.wait_for_and_click()
+        else:
+            editable_text_input_element = self.history_panel_click_to_rename()
+            editable_text_input_element.send_keys(new_name)
+            self.send_enter(editable_text_input_element)
 
     def history_panel_click_to_rename(self):
         history_panel = self.components.history_panel
-        if self.is_beta_history():
-            name = history_panel.name_beta
-            edit = history_panel.name_edit_input_beta
-        else:
-            name = history_panel.name
-            edit = history_panel.name_edit_input
-
+        name = history_panel.name
+        edit = history_panel.name_edit_input
         name.wait_for_and_click()
         return edit.wait_for_visible()
 
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index dcd5300db0d..4f766f55b91 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -116,6 +116,7 @@ history_panel:
       # These now appear other places :_( - e.g. in the invocation view so we'll try
       # prefixing ids with #current-history-panel but obviously we need to switch to classes or data
       _: '#current-history-panel #${history_content_type}-${id}'
+      by_state_and_hid: '.beta.history .listing [data-hid="${hid}"][data-state="${state}"]'
 
       title: '${_} .content-title'
       hid: '${_} .hid'
@@ -168,6 +169,7 @@ history_panel:
 
   selectors:
     beta: 'section.history'
+    by_attribute: '.beta.history [${name}="${value}"]'
     _: '#current-history-panel'
     search: '#current-history-panel input.search-query'
     refresh_button: '.history-refresh-button'

From a1399ce146d00045596d8d3001875439e912c130 Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Thu, 19 Aug 2021 15:40:33 -0400
Subject: [PATCH 383/452] Added healthcheck to irods object store

---
 .../sample/object_store_conf.xml.sample       |  2 +-
 lib/galaxy/objectstore/irods.py               | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample
index 3ed6341e74d..88eec71f996 100644
--- a/lib/galaxy/config/sample/object_store_conf.xml.sample
+++ b/lib/galaxy/config/sample/object_store_conf.xml.sample
@@ -48,7 +48,7 @@
             <auth username="rods" password="rods" />
             <resource name="demoResc" />
             <zone name="tempZone" />
-            <connection host="localhost" port="1247" timeout="30" poolsize="3" refresh_time="300"/>
+            <connection host="localhost" port="1247" timeout="30" poolsize="3" refresh_time="300" healthcheck_time="300" />
             <cache path="database/object_store_cache_irods" size="1000" />
             <extra_dir type="job_work" path="database/job_working_directory_irods"/>
             <extra_dir type="temp" path="database/tmp_irods"/>
diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py
index 8c8da670a47..f7ea604e0be 100644
--- a/lib/galaxy/objectstore/irods.py
+++ b/lib/galaxy/objectstore/irods.py
@@ -65,6 +65,7 @@ def parse_config_xml(config_xml):
         timeout = int(c_xml[0].get('timeout', 30))
         poolsize = int(c_xml[0].get('poolsize', 3))
         refresh_time = int(c_xml[0].get('refresh_time', 300))
+        healthcheck_time = int(c_xml[0].get('healthcheck_time', 300))
 
         c_xml = config_xml.findall('cache')
         if not c_xml:
@@ -95,6 +96,7 @@ def parse_config_xml(config_xml):
                 'timeout': timeout,
                 'poolsize': poolsize,
                 'refresh_time': refresh_time,
+                'healthcheck_time': healthcheck_time,
             },
             'cache': {
                 'size': cache_size,
@@ -128,6 +130,7 @@ class CloudConfigMixin:
                 'timeout': self.timeout,
                 'poolsize': self.poolsize,
                 'refresh_time': self.refresh_time,
+                'healthcheck_time': self.healthcheck_time,
             },
             'cache': {
                 'size': self.cache_size,
@@ -191,6 +194,9 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
         self.refresh_time = connection_dict.get('refresh_time')
         if self.refresh_time is None:
             _config_dict_error('connection->refresh_time')
+        self.healthcheck_time = connection_dict.get('healthcheck_time')
+        if self.healthcheck_time is None:
+            _config_dict_error('connection->healthcheck_time')
 
         cache_dict = config_dict['cache']
         if cache_dict is None:
@@ -207,6 +213,8 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
             _config_dict_error('extra_dirs')
         self.extra_dirs.update(extra_dirs)
 
+        self.last_healthcheck_time = datetime.now()
+
         if irods is None:
             raise Exception(IRODS_IMPORT_MESSAGE)
 
@@ -335,6 +343,10 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
 
     def _in_cache(self, rel_path):
         """ Check if the given dataset is in the local cache and return True if so. """
+        # Check every 'healthcheck_time' seconds if the irods server is down, and raise an exception, because if its 
+        # down, it can still serve files from cache. The raised exception notifies us of the server being down
+        self._server_healthcheck()
+
         cache_path = self._get_cache_path(rel_path)
         return os.path.exists(cache_path)
 
@@ -723,3 +735,14 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
 
     def _get_store_usage_percent(self):
         return 0.0
+
+    def _server_healthcheck(self):
+        # Every 'healthcheck_time' seconds, get a connection from the connection pool
+        # If the irods server is up, all is good -- release the connection
+        # If the irods server is down, a NetworkException is thrown
+        current_time = datetime.now()
+        if (current_time - self.last_healthcheck_time).total_seconds() > self.healthcheck_time:
+            # Update last_healthcheck_time
+            self.last_healthcheck_time = current_time
+            with self.session.pool.get_connection() as conn:
+                conn.release()

From 200d5b384eb76b090a2add461b3a045234b78277 Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Fri, 20 Aug 2021 15:53:50 -0400
Subject: [PATCH 384/452] Removed catch clause for NetworkException in
 irods.py, so if irods server is down, the exception percolates up.

---
 lib/galaxy/objectstore/irods.py | 36 ---------------------------------
 1 file changed, 36 deletions(-)

diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py
index f7ea604e0be..0e8e2b7a385 100644
--- a/lib/galaxy/objectstore/irods.py
+++ b/lib/galaxy/objectstore/irods.py
@@ -13,7 +13,6 @@ try:
     import irods.keywords as kw
     from irods.exception import CollectionDoesNotExist
     from irods.exception import DataObjectDoesNotExist
-    from irods.exception import NetworkException
     from irods.session import iRODSSession
 except ImportError:
     irods = None
@@ -313,9 +312,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
         except (DataObjectDoesNotExist, CollectionDoesNotExist):
             log.warning("Collection or data object (%s) does not exist", data_object_path)
             return -1
-        except NetworkException as e:
-            log.exception(e)
-            return -1
         finally:
             log.debug("irods_pt _get_size_in_irods: %s", ipt_timer)
 
@@ -335,18 +331,11 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
         except (DataObjectDoesNotExist, CollectionDoesNotExist):
             log.debug("Collection or data object (%s) does not exist", data_object_path)
             return False
-        except NetworkException as e:
-            log.exception(e)
-            return False
         finally:
             log.debug("irods_pt _data_object_exists: %s", ipt_timer)
 
     def _in_cache(self, rel_path):
         """ Check if the given dataset is in the local cache and return True if so. """
-        # Check every 'healthcheck_time' seconds if the irods server is down, and raise an exception, because if its 
-        # down, it can still serve files from cache. The raised exception notifies us of the server being down
-        self._server_healthcheck()
-
         cache_path = self._get_cache_path(rel_path)
         return os.path.exists(cache_path)
 
@@ -379,9 +368,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
         except (DataObjectDoesNotExist, CollectionDoesNotExist):
             log.warning("Collection or data object (%s) does not exist", data_object_path)
             return False
-        except NetworkException as e:
-            log.exception(e)
-            return False
         finally:
             log.debug("irods_pt _download: %s", ipt_timer)
 
@@ -458,9 +444,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
                 log.debug("Pushed cache file '%s' to collection '%s' (%s bytes transfered in %s sec)",
                         source_file, rel_path, os.path.getsize(source_file), (end_time - start_time).total_seconds())
             return True
-        except NetworkException as e:
-            log.exception(e)
-            return False
         finally:
             log.debug("irods_pt _push_to_irods: %s", ipt_timer)
 
@@ -584,9 +567,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
                 except CollectionDoesNotExist:
                     log.warning("Collection (%s) does not exist!", col_path)
                     return False
-                except NetworkException as e:
-                    log.exception(e)
-                    return False
 
                 cols = col.walk()
                 # Traverse the tree only one level deep
@@ -623,13 +603,8 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
                 except (DataObjectDoesNotExist, CollectionDoesNotExist):
                     log.info("Collection or data object (%s) does not exist", data_object_path)
                     return True
-                except NetworkException as e:
-                    log.exception(e)
-                    return False
         except OSError:
             log.exception('%s delete error', self._get_filename(obj, **kwargs))
-        except NetworkException as e:
-            log.exception(e)
         finally:
             log.debug("irods_pt _delete: %s", ipt_timer)
         return False
@@ -735,14 +710,3 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
 
     def _get_store_usage_percent(self):
         return 0.0
-
-    def _server_healthcheck(self):
-        # Every 'healthcheck_time' seconds, get a connection from the connection pool
-        # If the irods server is up, all is good -- release the connection
-        # If the irods server is down, a NetworkException is thrown
-        current_time = datetime.now()
-        if (current_time - self.last_healthcheck_time).total_seconds() > self.healthcheck_time:
-            # Update last_healthcheck_time
-            self.last_healthcheck_time = current_time
-            with self.session.pool.get_connection() as conn:
-                conn.release()

From c041a750b58ada95baabca67817605589d19c7c5 Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Fri, 20 Aug 2021 16:53:06 -0400
Subject: [PATCH 385/452] Removed healthcheck related code

---
 lib/galaxy/config/sample/object_store_conf.xml.sample | 2 +-
 lib/galaxy/objectstore/irods.py                       | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample
index 88eec71f996..3ed6341e74d 100644
--- a/lib/galaxy/config/sample/object_store_conf.xml.sample
+++ b/lib/galaxy/config/sample/object_store_conf.xml.sample
@@ -48,7 +48,7 @@
             <auth username="rods" password="rods" />
             <resource name="demoResc" />
             <zone name="tempZone" />
-            <connection host="localhost" port="1247" timeout="30" poolsize="3" refresh_time="300" healthcheck_time="300" />
+            <connection host="localhost" port="1247" timeout="30" poolsize="3" refresh_time="300"/>
             <cache path="database/object_store_cache_irods" size="1000" />
             <extra_dir type="job_work" path="database/job_working_directory_irods"/>
             <extra_dir type="temp" path="database/tmp_irods"/>
diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py
index 0e8e2b7a385..b536bba0c87 100644
--- a/lib/galaxy/objectstore/irods.py
+++ b/lib/galaxy/objectstore/irods.py
@@ -64,7 +64,6 @@ def parse_config_xml(config_xml):
         timeout = int(c_xml[0].get('timeout', 30))
         poolsize = int(c_xml[0].get('poolsize', 3))
         refresh_time = int(c_xml[0].get('refresh_time', 300))
-        healthcheck_time = int(c_xml[0].get('healthcheck_time', 300))
 
         c_xml = config_xml.findall('cache')
         if not c_xml:
@@ -95,7 +94,6 @@ def parse_config_xml(config_xml):
                 'timeout': timeout,
                 'poolsize': poolsize,
                 'refresh_time': refresh_time,
-                'healthcheck_time': healthcheck_time,
             },
             'cache': {
                 'size': cache_size,
@@ -129,7 +127,6 @@ class CloudConfigMixin:
                 'timeout': self.timeout,
                 'poolsize': self.poolsize,
                 'refresh_time': self.refresh_time,
-                'healthcheck_time': self.healthcheck_time,
             },
             'cache': {
                 'size': self.cache_size,
@@ -193,9 +190,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
         self.refresh_time = connection_dict.get('refresh_time')
         if self.refresh_time is None:
             _config_dict_error('connection->refresh_time')
-        self.healthcheck_time = connection_dict.get('healthcheck_time')
-        if self.healthcheck_time is None:
-            _config_dict_error('connection->healthcheck_time')
 
         cache_dict = config_dict['cache']
         if cache_dict is None:
@@ -212,8 +206,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
             _config_dict_error('extra_dirs')
         self.extra_dirs.update(extra_dirs)
 
-        self.last_healthcheck_time = datetime.now()
-
         if irods is None:
             raise Exception(IRODS_IMPORT_MESSAGE)
 

From bfe03bd4b9d5520220c6fc6a6ff7e9b770307594 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Sat, 21 Aug 2021 21:25:17 +0530
Subject: [PATCH 386/452] Fix variable name typo in watchers

---
 lib/galaxy/tool_util/toolbox/watcher.py | 6 +++---
 lib/galaxy/util/watcher.py              | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tool_util/toolbox/watcher.py b/lib/galaxy/tool_util/toolbox/watcher.py
index 8a374971487..777e3b16b90 100644
--- a/lib/galaxy/tool_util/toolbox/watcher.py
+++ b/lib/galaxy/tool_util/toolbox/watcher.py
@@ -32,7 +32,7 @@ def get_tool_watcher(toolbox, config):
     config_value = getattr(config, config_name, None)
     observer_class = get_observer_class(config_name, config_value, default="False", monitor_what_str="tools")
     if observer_class is not None:
-        return ToolWatcher(observer_class=observer_class, even_handler_class=ToolFileEventHandler, toolbox=toolbox)
+        return ToolWatcher(observer_class=observer_class, event_handler_class=ToolFileEventHandler, toolbox=toolbox)
     else:
         return NullWatcher()
 
@@ -169,8 +169,8 @@ class ToolConfWatcher:
 
 class ToolWatcher(BaseWatcher):
 
-    def __init__(self, observer_class, even_handler_class, toolbox):
-        super().__init__(observer_class, even_handler_class)
+    def __init__(self, observer_class, event_handler_class, toolbox):
+        super().__init__(observer_class, event_handler_class)
         self.toolbox = toolbox
         self.tool_file_ids = {}
         self.tool_dir_callbacks = {}
diff --git a/lib/galaxy/util/watcher.py b/lib/galaxy/util/watcher.py
index 1ed77c21512..3a2c56a67d5 100644
--- a/lib/galaxy/util/watcher.py
+++ b/lib/galaxy/util/watcher.py
@@ -65,10 +65,10 @@ def get_watcher(config, config_name, default="False", monitor_what_str=None, wat
 
 class BaseWatcher:
 
-    def __init__(self, observer_class, even_handler_class, **kwargs):
+    def __init__(self, observer_class, event_handler_class, **kwargs):
         self.observer = None
         self.observer_class = observer_class
-        self.event_handler = even_handler_class(self)
+        self.event_handler = event_handler_class(self)
         self.monitored_dirs = {}
 
     def start(self):

From 4b75be4e8e903c66d1565f80cd3881427b952533 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 23 Aug 2021 20:46:17 +0200
Subject: [PATCH 387/452] Deprecate url_for in FastAPI routes

This generalizes the error handling when trying to call `url_for` in ASGI mode.
---
 lib/galaxy/managers/base.py             |  4 ++--
 lib/galaxy/web/framework/__init__.py    | 17 ++++++++++++++++-
 lib/galaxy_test/api/test_group_roles.py |  6 +++++-
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index c95d0171ab4..f312693e75b 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -39,7 +39,6 @@ from typing import (
     Type,
 )
 
-import routes
 import sqlalchemy
 from sqlalchemy.orm.scoping import scoped_session
 
@@ -51,6 +50,7 @@ from galaxy.schema.fields import EncodedDatabaseIdField
 from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.structured_app import BasicApp, MinimalManagerApp
 from galaxy.util import namedtuple
+from galaxy.web import url_for as gx_url_for
 
 log = logging.getLogger(__name__)
 
@@ -554,7 +554,7 @@ class ModelSerializer(HasAModelManager):
         item_dict = MySerializer.serialize( my_item, keys_to_serialize )
     """
     #: 'service' to use for getting urls - use class var to allow overriding when testing
-    url_for = staticmethod(routes.url_for)
+    url_for = staticmethod(gx_url_for)
     default_view: Optional[str]
     views: Dict[str, List[str]]
 
diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py
index b4dc9cec0ab..64dabaf7933 100644
--- a/lib/galaxy/web/framework/__init__.py
+++ b/lib/galaxy/web/framework/__init__.py
@@ -4,4 +4,19 @@ Galaxy web application framework
 
 from . import base
 
-url_for = base.routes.url_for
+DEPRECATED_URL_ATTRIBUTE_MESSAGE = "*deprecated attribute, URL not filled in by server*"
+
+
+def handle_url_for(*args, **kargs) -> str:
+    """Tries to resolve the URL using the `routes` module.
+
+    This only works in a WSGI app so a deprecation message is returned
+    when running an ASGI app.
+    """
+    try:
+        return base.routes.url_for(*args, **kargs)
+    except AttributeError:
+        return DEPRECATED_URL_ATTRIBUTE_MESSAGE
+
+
+url_for = handle_url_for
diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py
index d445a900f79..ef2c1b95e2c 100644
--- a/lib/galaxy_test/api/test_group_roles.py
+++ b/lib/galaxy_test/api/test_group_roles.py
@@ -1,5 +1,6 @@
 from typing import List
 
+from galaxy.web.framework import DEPRECATED_URL_ATTRIBUTE_MESSAGE
 from galaxy_test.base.populators import DatasetPopulator
 from ._framework import ApiTestCase
 
@@ -67,7 +68,10 @@ class GroupRolesApiTestCase(ApiTestCase):
         self._assert_status_code_is_ok(update_response)
         group_role = update_response.json()
         self._assert_valid_group_role(group_role, assert_id=encoded_role_id)
-        assert (group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}" or group_role["url"] == "*deprecated attribute not filled in by FastAPI server*")
+        assert (
+            group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}"
+            or group_role["url"] == DEPRECATED_URL_ATTRIBUTE_MESSAGE
+        )
 
     def test_update_only_admin(self):
         encoded_group_id = "any-group-id"

From c7b996c2e7c407f754d980061ec2f3dc5030248e Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Tue, 24 Aug 2021 10:47:31 +0200
Subject: [PATCH 388/452] Refactor roles and group_roles api to use the default
 `url_for` result

---
 lib/galaxy/webapps/galaxy/api/common.py      | 4 ----
 lib/galaxy/webapps/galaxy/api/group_roles.py | 9 ++-------
 lib/galaxy/webapps/galaxy/api/roles.py       | 6 +-----
 3 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py
index dbd889c1f60..f8bc267c2c6 100644
--- a/lib/galaxy/webapps/galaxy/api/common.py
+++ b/lib/galaxy/webapps/galaxy/api/common.py
@@ -28,7 +28,3 @@ def parse_serialization_params(
     if keys:
         key_list = keys.split(',')
     return dict(view=view, keys=key_list, default_view=default_view)
-
-
-def fastapi_deprecation_message():
-    return "*deprecated attribute not filled in by FastAPI server*"
diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 2aeebfdf53a..0334d3f8e1d 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -5,7 +5,6 @@ API operations on Group objects.
 import logging
 
 from fastapi import Path
-from routes.util import url_for
 
 from galaxy.managers.context import ProvidesAppContext
 from galaxy.managers.group_roles import GroupRolesManager
@@ -14,8 +13,8 @@ from galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 from galaxy.web import (
     expose_api,
     require_admin,
+    url_for,
 )
-from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 from . import (
     BaseGalaxyAPIController,
     depends,
@@ -43,11 +42,7 @@ RoleIDParam: EncodedDatabaseIdField = Path(
 
 def group_role_to_model(trans, encoded_group_id, role):
     encoded_role_id = trans.security.encode_id(role.id)
-    try:
-        url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
-    except AttributeError:
-        url = fastapi_deprecation_message()
-
+    url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
     return GroupRoleModel(
         id=encoded_role_id,
         name=role.name,
diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py
index d427cc5f5f8..e3dbb545723 100644
--- a/lib/galaxy/webapps/galaxy/api/roles.py
+++ b/lib/galaxy/webapps/galaxy/api/roles.py
@@ -19,7 +19,6 @@ from galaxy.schema.schema import (
     RoleModel,
 )
 from galaxy.webapps.base.controller import url_for
-from galaxy.webapps.galaxy.api.common import fastapi_deprecation_message
 from . import (
     BaseGalaxyAPIController,
     depends,
@@ -38,10 +37,7 @@ router = Router(tags=["roles"])
 def role_to_model(trans, role):
     item = role.to_dict(view='element', value_mapper={'id': trans.security.encode_id})
     role_id = trans.security.encode_id(role.id)
-    try:
-        item['url'] = url_for('role', id=role_id)
-    except AttributeError:
-        item['url'] = fastapi_deprecation_message()
+    item['url'] = url_for('role', id=role_id)
     return RoleModel(**item)
 
 

From 46afba301e9efe240e177439fd6ae5fee31ad538 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Tue, 24 Aug 2021 18:26:14 +0200
Subject: [PATCH 389/452] Mark RelativeUrlField as deprecated in the schema

---
 lib/galaxy/schema/schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 5dcdcf8b9ec..12ea3aa53de 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -51,7 +51,7 @@ RelativeUrlField: RelativeUrl = Field(
     ...,
     title="URL",
     description="The relative URL to access this item.",
-    deprecated=False  # TODO Should this field be deprecated in FastAPI?
+    deprecated=True,
 )
 
 DownloadUrlField: RelativeUrl = Field(

From e13bd6a9fd3e42b8a683e3d36c8d0eef6edca633 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 02:27:05 -0400
Subject: [PATCH 390/452] Parse error details and inputs instead of complete
 form model object on changes

---
 client/src/components/Form/FormDisplay.vue |  4 ++--
 client/src/mvc/form/form-data.js           |  4 ++--
 client/src/mvc/form/form-view.js           | 10 +++++-----
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 23f95003bef..874e57a42b8 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -72,9 +72,9 @@ export default {
         },
         formConfig() {
             this.$nextTick(() => {
-                this.form.update(this.formConfig);
+                this.form.update(this.formConfig.inputs);
                 if (this.initialErrors) {
-                    this.form.errors(this.formConfig);
+                    this.form.errors(this.formConfig.errors);
                 }
             });
         },
diff --git a/client/src/mvc/form/form-data.js b/client/src/mvc/form/form-data.js
index ae7f5732d6a..b8cd86dfe64 100644
--- a/client/src/mvc/form/form-data.js
+++ b/client/src/mvc/form/form-data.js
@@ -124,9 +124,9 @@ export var Manager = Backbone.Model.extend({
 
     /** Matches a new tool model to the current input elements e.g. used to update dynamic options
      */
-    matchModel: function (model, callback) {
+    matchModel: function (inputs, callback) {
         var self = this;
-        visitInputs(model.inputs, (input, name) => {
+        visitInputs(inputs, (input, name) => {
             if (self.flat_dict[name]) {
                 callback(input, self.flat_dict[name]);
             }
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index a7590991442..ba0d8ad684a 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -19,9 +19,9 @@ export default Backbone.View.extend({
     },
 
     /** Update available options */
-    update: function (new_model) {
+    update: function (inputs) {
         var self = this;
-        this.data.matchModel(new_model, (node, input_id) => {
+        this.data.matchModel(inputs, (node, input_id) => {
             var field = self.field_list[input_id];
             if (field.update) {
                 field.update(node);
@@ -64,10 +64,10 @@ export default Backbone.View.extend({
     },
 
     /** Highlights errors */
-    errors: function (options) {
+    errors: function (details) {
         this.trigger("reset");
-        if (options && options.errors) {
-            var error_messages = this.data.matchResponse(options.errors);
+        if (details) {
+            var error_messages = this.data.matchResponse(details);
             for (var input_id in this.element_list) {
                 if (error_messages[input_id]) {
                     this.highlight(input_id, error_messages[input_id], true);

From 93437992572c35db941bed51f950415d8162ec82 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 02:28:07 -0400
Subject: [PATCH 391/452] Remove unused forms.js legacy module from workflow
 editor

---
 .../Workflow/Editor/modules/forms.js          | 362 ------------------
 1 file changed, 362 deletions(-)
 delete mode 100644 client/src/components/Workflow/Editor/modules/forms.js

diff --git a/client/src/components/Workflow/Editor/modules/forms.js b/client/src/components/Workflow/Editor/modules/forms.js
deleted file mode 100644
index 19a679ba399..00000000000
--- a/client/src/components/Workflow/Editor/modules/forms.js
+++ /dev/null
@@ -1,362 +0,0 @@
-import axios from "axios";
-import { getAppRoot } from "onload/loadConfig";
-import { getGalaxyInstance } from "app";
-import _l from "utils/localization";
-import Utils from "utils/utils";
-import ToolFormBase from "mvc/tool/tool-form-base";
-import WorkflowIcons from "components/Workflow/icons";
-
-/** Tool form wrapper for the workflow editor. */
-export class ToolForm {
-    constructor(options) {
-        const self = this;
-        const node = options.node;
-        this.workflow = options.workflow;
-        this.datatypes = options.datatypes;
-        this._customize(node);
-        this.form = new ToolFormBase({
-            ...node.config_form,
-            tool_version: node.config_form.version,
-            icon: WorkflowIcons[node.type],
-            text_enable: "Set in Advance",
-            text_disable: "Set at Runtime",
-            narrow: true,
-            initial_errors: true,
-            cls: "ui-portlet-section",
-            postchange(process, form) {
-                const Galaxy = getGalaxyInstance();
-                const options = form.model.attributes;
-                const current_state = {
-                    tool_id: options.id,
-                    tool_version: options.version,
-                    type: "tool",
-                    inputs: Object.assign({}, form.data.create()),
-                };
-                Galaxy.emit.debug("tool-form-workflow::postchange()", "Sending current state.", current_state);
-                axios
-                    .post(`${getAppRoot()}api/workflows/build_module`, current_state)
-                    .then(({ data }) => {
-                        node.updateData(data);
-                        self.form.model.set(node.config_form);
-                        self.form.update(node.config_form);
-                        self.form.errors(node.config_form);
-                        self._customize(node);
-                        Galaxy.emit.debug("tool-form-workflow::postchange()", "Received new model.", data);
-                        process.resolve();
-                    })
-                    .catch((response) => {
-                        Galaxy.emit.debug("tool-form-workflow::postchange()", "Refresh request failed.", response);
-                        process.reject();
-                    });
-            },
-        });
-    }
-    _customize(node) {
-        const inputs = node.config_form.inputs;
-        Utils.deepeach(inputs, (input) => {
-            if (input.type) {
-                if (["data", "data_collection"].indexOf(input.type) != -1) {
-                    input.hiddenInWorkflow = true;
-                    input.info = `Data input '${input.name}' (${Utils.textify(input.extensions)})`;
-                    input.value = { __class__: "RuntimeValue" };
-                } else if (!input.fixed) {
-                    input.connectable = true;
-                    input.collapsible_value = {
-                        __class__: "RuntimeValue",
-                    };
-                    input.is_workflow =
-                        (input.options && input.options.length === 0) || ["integer", "float"].indexOf(input.type) != -1;
-                }
-            }
-        });
-        Utils.deepeach(inputs, (input) => {
-            if (input.type === "conditional") {
-                input.connectable = false;
-                input.test_param.collapsible_value = undefined;
-            }
-        });
-        _addSections(this, node);
-        _addLabelAnnotation(this, node);
-    }
-}
-
-/** Augments the module form definition by adding label and annotation fields */
-function _addLabelAnnotation(self, node) {
-    const workflow = self.workflow;
-    const inputs = node.config_form.inputs;
-    inputs.unshift({
-        skipOnClone: true,
-        type: "text",
-        name: "__annotation",
-        label: "Step Annotation",
-        fixed: true,
-        value: node.annotation,
-        area: true,
-        onchange: function (new_annotation) {
-            node.setAnnotation(new_annotation);
-        },
-        help: "Add an annotation or notes to this step. Annotations are available when a workflow is viewed.",
-    });
-    inputs.unshift({
-        skipOnClone: true,
-        type: "text",
-        name: "__label",
-        label: "Label",
-        value: node.label,
-        help: _l("Add a step label."),
-        fixed: true,
-        onchange: function (new_label) {
-            node.setLabel(new_label);
-            let duplicate = false;
-            for (const i in workflow.nodes) {
-                const n = workflow.nodes[i];
-                if (n.label && n.label == new_label && n.id != node.id) {
-                    duplicate = true;
-                    break;
-                }
-            }
-            self.form.data.create();
-            const input_id = self.form.data.match("__label");
-            const input_element = self.form.element_list[input_id];
-            input_element.model.set(
-                "error_text",
-                duplicate && "Duplicate label. Please fix this before saving the workflow."
-            );
-        },
-    });
-}
-
-/** Visit input nodes and enrich by name/value pairs from server data */
-function _visit(head, head_list, output, node) {
-    const postJobActions = node.postJobActions;
-    head_list = head_list || [];
-    head_list.push(head);
-    for (const i in head.inputs) {
-        const input = head.inputs[i];
-        const action = input.action;
-        if (action) {
-            input.name = `pja__${output.name}__${input.action}`;
-            if (input.pja_arg) {
-                input.name += `__${input.pja_arg}`;
-            }
-            if (input.payload) {
-                for (const p_id in input.payload) {
-                    input.payload[`${input.name}__${p_id}`] = input.payload[p_id];
-                    delete input.payload[p_id];
-                }
-            }
-            const d = postJobActions[input.action + output.name];
-            if (d) {
-                for (const j in head_list) {
-                    head_list[j].expanded = true;
-                }
-                if (input.pja_arg) {
-                    input.value = (d.action_arguments && d.action_arguments[input.pja_arg]) || input.value;
-                } else {
-                    input.value = "true";
-                }
-            }
-        }
-        if (input.inputs) {
-            _visit(input, head_list.slice(0), output, node);
-        }
-    }
-}
-
-function _makeRenameHelp(name_labels) {
-    let help_section = `This action will rename the output dataset. Click <a href="https://galaxyproject.org/learn/advanced-workflow/variables/">here</a> for more information. Valid input variables are:`;
-    const li = `
-        <ul>
-            ${name_labels
-                .map(
-                    (name_label) => `<li><strong>${name_label.name.replace(/\|/g, ".")}</strong>
-                                                         ${name_label.label ? `(${name_label.label})` : ""}
-                                             </li>`
-                )
-                .join("")}
-        </ul>
-    `;
-    help_section += li;
-    return help_section;
-}
-
-/** Builds sub section with step actions/annotation */
-function _makeSection(self, node, output) {
-    const extensions = [];
-    const name_label_map = [];
-    const datatypes = self.datatypes;
-    for (const key in datatypes) {
-        extensions.push({ 0: datatypes[key], 1: datatypes[key] });
-    }
-    for (const input of node.inputs) {
-        name_label_map.push({ name: input.name, label: input.label });
-    }
-    const renameHelp = _makeRenameHelp(name_label_map);
-    extensions.sort((a, b) => (a.label > b.label ? 1 : a.label < b.label ? -1 : 0));
-    extensions.unshift({
-        0: "Sequences",
-        1: "Sequences",
-    });
-    extensions.unshift({
-        0: "Roadmaps",
-        1: "Roadmaps",
-    });
-    extensions.unshift({
-        0: "Leave unchanged",
-        1: "__empty__",
-    });
-    const activeOutput = node.activeOutputs.get(output.name);
-    const inputTitle = output.label || output.name;
-    const inputConfig = {
-        skipOnClone: true,
-        title: `Configure Output: '${inputTitle}'`,
-        type: "section",
-        flat: true,
-        inputs: [
-            {
-                label: "Label",
-                name: `__label__${output.name}`,
-                type: "text",
-                value: activeOutput && activeOutput.label,
-                help: "This will provide a short name to describe the output - this must be unique across workflows.",
-                fixed: true,
-                onchange: (newLabel) => {
-                    self.form.data.create();
-                    const oldLabel = node.labelOutput(output.name, newLabel);
-                    const input_id = self.form.data.match(`__label__${output.name}`);
-                    const input_element = self.form.element_list[input_id];
-                    if (oldLabel) {
-                        input_element.field.model.set("value", oldLabel);
-                        input_element.model.set("error_text", `Duplicate output label '${newLabel}' will be ignored.`);
-                    } else {
-                        input_element.model.set("error_text", "");
-                    }
-                    self.form.trigger("change");
-                },
-            },
-            {
-                action: "RenameDatasetAction",
-                pja_arg: "newname",
-                label: "Rename dataset",
-                type: "text",
-                value: "",
-                ignore: "",
-                help: renameHelp,
-            },
-            {
-                action: "ChangeDatatypeAction",
-                pja_arg: "newtype",
-                label: "Change datatype",
-                type: "select",
-                ignore: "__empty__",
-                value: "__empty__",
-                options: extensions,
-                help: "This action will change the datatype of the output to the indicated datatype.",
-                onchange: function (datatype) {
-                    node.changeOutputDatatype(output.name, datatype);
-                },
-            },
-            {
-                action: "TagDatasetAction",
-                pja_arg: "tags",
-                label: "Add Tags",
-                type: "text",
-                value: "",
-                ignore: "",
-                help: "This action will set tags for the dataset.",
-            },
-            {
-                action: "RemoveTagDatasetAction",
-                pja_arg: "tags",
-                label: "Remove Tags",
-                type: "text",
-                value: "",
-                ignore: "",
-                help: "This action will remove tags for the dataset.",
-            },
-            {
-                title: _l("Assign columns"),
-                type: "section",
-                flat: true,
-                inputs: [
-                    {
-                        action: "ColumnSetAction",
-                        pja_arg: "chromCol",
-                        label: "Chrom column",
-                        type: "integer",
-                        value: "",
-                        ignore: "",
-                    },
-                    {
-                        action: "ColumnSetAction",
-                        pja_arg: "startCol",
-                        label: "Start column",
-                        type: "integer",
-                        value: "",
-                        ignore: "",
-                    },
-                    {
-                        action: "ColumnSetAction",
-                        pja_arg: "endCol",
-                        label: "End column",
-                        type: "integer",
-                        value: "",
-                        ignore: "",
-                    },
-                    {
-                        action: "ColumnSetAction",
-                        pja_arg: "strandCol",
-                        label: "Strand column",
-                        type: "integer",
-                        value: "",
-                        ignore: "",
-                    },
-                    {
-                        action: "ColumnSetAction",
-                        pja_arg: "nameCol",
-                        label: "Name column",
-                        type: "integer",
-                        value: "",
-                        ignore: "",
-                    },
-                ],
-                help: "This action will set column assignments in the output dataset. Blank fields are ignored.",
-            },
-        ],
-    };
-    _visit(inputConfig, [], output, node);
-    return inputConfig;
-}
-
-/** Builds all sub sections */
-function _addSections(self, node) {
-    const inputs = node.config_form.inputs;
-    const postJobActions = node.postJobActions;
-    const outputFirst = node.outputs.length > 0 && node.outputs[0];
-    if (outputFirst) {
-        inputs.push({
-            name: `pja__${outputFirst.name}__EmailAction`,
-            label: "Email notification",
-            type: "boolean",
-            value: String(Boolean(postJobActions[`EmailAction${outputFirst.name}`])),
-            ignore: "false",
-            help: _l("An email notification will be sent when the job has completed."),
-            payload: {
-                host: window.location.host,
-            },
-            skipOnClone: true,
-        });
-        inputs.push({
-            name: `pja__${outputFirst.name}__DeleteIntermediatesAction`,
-            label: "Output cleanup",
-            type: "boolean",
-            value: String(Boolean(postJobActions[`DeleteIntermediatesAction${outputFirst.name}`])),
-            ignore: "false",
-            help: "Upon completion of this step, delete non-starred outputs from completed workflow steps if they are no longer required as inputs.",
-            skipOnClone: true,
-        });
-        for (const output of node.outputs) {
-            inputs.push(_makeSection(self, node, output));
-        }
-    }
-}

From b990725f2344578185fdefb74fe6bf9d70e3853e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 02:40:21 -0400
Subject: [PATCH 392/452] Avoid parsing complete model from tool form wrapper

---
 client/src/components/Form/FormDisplay.vue | 12 ++++++++----
 client/src/components/Tool/ToolForm.vue    |  7 ++-----
 client/src/mvc/form/form-view.js           |  2 +-
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 874e57a42b8..278c96c2b62 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -45,7 +45,7 @@ export default {
             type: Array,
             default: null,
         },
-        formConfig: {
+        errors: {
             type: Object,
             default: null,
         },
@@ -70,11 +70,15 @@ export default {
             this.onHighlight(this.validation, true);
             this.$emit("onValidation", this.validation);
         },
-        formConfig() {
+        inputs() {
+            this.$nextTick(() => {
+                this.form.update(this.inputs);
+            });
+        },
+        errors() {
             this.$nextTick(() => {
-                this.form.update(this.formConfig.inputs);
                 if (this.initialErrors) {
-                    this.form.errors(this.formConfig.errors);
+                    this.form.errors(this.errors);
                 }
             });
         },
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 60ecf918f8e..ddecee3ee9b 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -41,9 +41,9 @@
                         <template v-slot:body>
                             <FormDisplay
                                 :id="formConfig.id"
-                                :inputs="inputs"
+                                :inputs="formConfig.inputs"
                                 :validation-scroll-to="validationScrollTo"
-                                :form-config="formConfig"
+                                :errors="formConfig.errors"
                                 @onChange="onChange"
                                 @onValidation="onValidation"
                             />
@@ -182,9 +182,6 @@ export default {
         errorContentPretty() {
             return JSON.stringify(this.errorContent, null, 4);
         },
-        inputs() {
-            return this.formConfig.inputs;
-        },
         remapTitle() {
             if (this.remapAllowed === "job_produced_collection_elements") {
                 return "Replace elements in collection?";
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index ba0d8ad684a..4364479a760 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -92,7 +92,7 @@ export default Backbone.View.extend({
         this._renderForm();
         this.data.create();
         if (this.model.get("initial_errors")) {
-            this.errors(this.model.attributes);
+            this.errors(this.model.get("errors"));
         }
         // add listener which triggers on checksum change, and reset the form input wrappers
         var current_check = this.data.checksum();

From 1dadfe26a62041116aa931d993d22cc83745e68a Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 02:46:04 -0400
Subject: [PATCH 393/452] Remove form object parsing from workflow editor tool
 form

---
 client/src/components/Workflow/Editor/Forms/FormTool.vue | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index 70caeb2084d..fb8abe3d5d1 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -33,7 +33,7 @@
                     :id="id"
                     :inputs="inputs"
                     :initial-errors="true"
-                    :form-config="formConfig"
+                    :errors="errors"
                     text-enable="Set in Advance"
                     text-disable="Set at Runtime"
                     @onChange="onChange"
@@ -88,7 +88,6 @@ export default {
             sectionValues: {},
             messageVariant: "",
             messageText: "",
-            formConfig: {},
         };
     },
     computed: {
@@ -128,6 +127,9 @@ export default {
             });
             return inputs;
         },
+        errors() {
+            return this.node.config_form.errors;
+        },
     },
     methods: {
         onChangeOutputDatatype(outputName, newDatatype) {
@@ -162,7 +164,6 @@ export default {
                 inputs: Object.assign({}, this.mainValues, this.sectionValues),
             }).then((data) => {
                 this.$emit("onSetData", this.node.id, data);
-                this.formConfig = data;
                 if (newVersion) {
                     const options = data.config_form;
                     this.messageVariant = "success";

From f7eefdf36a2e61aeeb53440d09c2b246ba2560f4 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 02:58:05 -0400
Subject: [PATCH 394/452] Remove form config parsing from workflow run tool
 step wrapper

---
 .../src/components/Workflow/Run/WorkflowRunToolStep.vue   | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 5d28ea26698..a8f18e89f7f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -4,8 +4,8 @@
             <template v-slot:body>
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <FormDisplay
-                    :inputs="model.inputs"
-                    :form-config="formConfig"
+                    :inputs="formConfig.inputs"
+                    :errors="formConfig.errors"
                     :sustain-repeats="true"
                     :sustain-conditionals="true"
                     :replace-params="replaceParams"
@@ -58,8 +58,7 @@ export default {
     data() {
         return {
             collapsed: this.model.collapsed,
-            formConfig: {},
-            formData: {},
+            formConfig: this.model,
             replaceParams: {},
             errorText: null,
         };
@@ -132,7 +131,6 @@ export default {
                     this.errorText = errorText;
                 }
             );
-            this.formData = data;
             this.$emit("onChange", this.model.index, data);
         },
         onValidation(validation) {

From ee63fcc736da551dad9668f1445ad4828594e69b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 24 Aug 2021 17:16:56 +0300
Subject: [PATCH 395/452] Allow skipping sanitization of / char

and allow using collection name as outer directory
---
 lib/galaxy/tools/parameters/wrapped_json.py | 11 ++++----
 lib/galaxy/tools/wrappers.py                | 30 ++++++++++++---------
 lib/galaxy/util/__init__.py                 |  2 +-
 tools/data_export/export_remote.xml         | 13 ++++++---
 4 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py
index 954e78123d1..bb4b0c83f0d 100644
--- a/lib/galaxy/tools/parameters/wrapped_json.py
+++ b/lib/galaxy/tools/parameters/wrapped_json.py
@@ -33,8 +33,8 @@ def data_collection_input_to_path(v):
     return v.all_paths
 
 
-def data_collection_input_to_staging_path_and_source_path(v):
-    staging_paths = v.all_staging_paths
+def data_collection_input_to_staging_path_and_source_path(v, invalid_chars=('/',), include_collection_name=False):
+    staging_paths = v.all_staging_paths(invalid_chars=invalid_chars, include_collection_name=include_collection_name)
     source_paths = v.all_paths
     metadata_files = v.all_metadata_files
     return [
@@ -44,11 +44,12 @@ def data_collection_input_to_staging_path_and_source_path(v):
          } for staging_path, source_path, metadata_files in zip(staging_paths, source_paths, metadata_files)]
 
 
-def data_input_to_staging_path_and_source_path(v):
+def data_input_to_staging_path_and_source_path(v, invalid_chars=('/',)):
+    staging_path = v.staging_path(invalid_chars=invalid_chars)
     return {
-        'staging_path': v.staging_path,
+        'staging_path': staging_path,
         'source_path': data_input_to_path(v),
-        'metadata_files': [{'staging_path': f"{v.staging_path}.{mf[0]}", 'source_path': mf[1]} for mf in v.all_metadata_files]
+        'metadata_files': [{'staging_path': f"{staging_path}.{mf[0]}", 'source_path': mf[1]} for mf in v.all_metadata_files]
     }
 
 
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 9751d423211..38b36e0546f 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -315,23 +315,22 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
     def name_and_ext(self):
         return f"{self.element_identifier}.{self.file_ext}"
 
-    @property
-    def staging_path(self):
+    def staging_path(self, invalid_chars=('/',)):
         """
         Strip leading dots, unicode null chars, replace `/` with `_`, truncate at 255 characters.
 
         Not safe for commandline use, would need additional sanitization.
         """
         max_len = 254 - len(self.file_ext)
-        safe_element_identifier = filesystem_safe_string(self.element_identifier, max_len=max_len)
+        safe_element_identifier = filesystem_safe_string(self.element_identifier, max_len=max_len, invalid_chars=invalid_chars)
         return f"{safe_element_identifier}.{self.file_ext}"
 
     @property
     def all_metadata_files(self):
         return self.unsanitized.get_metadata_file_paths_and_extensions() if self else []
 
-    def serialize(self):
-        return data_input_to_staging_path_and_source_path(self) if self else {}
+    def serialize(self, invalid_chars=('/',)):
+        return data_input_to_staging_path_and_source_path(self, invalid_chars=invalid_chars) if self else {}
 
     @property
     def is_collection(self):
@@ -449,8 +448,8 @@ class DatasetListWrapper(list, ToolParameterValueWrapper, HasDatasets):
             self._dataset_elements_cache[group] = wrappers
         return self._dataset_elements_cache[group]
 
-    def serialize(self):
-        return [v.serialize() for v in self]
+    def serialize(self, invalid_chars=('/',)):
+        return [v.serialize(invalid_chars) for v in self]
 
     def __str__(self):
         return ','.join(map(str, self))
@@ -549,8 +548,7 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
                 return []
         return self._element_identifiers_extensions_paths_and_metadata_files
 
-    @property
-    def all_staging_paths(self):
+    def all_staging_paths(self, invalid_chars=('/',), include_collection_name=False):
         safe_element_identifiers = []
         for element_identifiers, extension, *_ in self.element_identifiers_extensions_paths_and_metadata_files:
             datatype = self.datatypes_registry.get_datatype_by_extension(extension)
@@ -558,12 +556,20 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
                 extension = getattr(datatype, 'file_ext_export_alias', extension)
             current_element_identifiers = []
             for element_identifier in element_identifiers:
-                current_element_identifiers.append(filesystem_safe_string(element_identifier, max_len=254 - len(extension)))
+                max_len = 254 - len(extension)
+                if include_collection_name:
+                    max_len = max_len - (len(self.name) + 1)
+                    assert max_len >= 1, 'Could not stage element, element identifier is too long'
+                current_element_identifier = filesystem_safe_string(element_identifier, max_len=max_len, invalid_chars=invalid_chars)
+                if include_collection_name and self.name:
+                    current_element_identifier = f"{filesystem_safe_string(self.name, invalid_chars=invalid_chars)}{os.path.sep}{current_element_identifier}"
+                current_element_identifiers.append(current_element_identifier)
+
             safe_element_identifiers.append(f'{os.path.sep.join(current_element_identifiers)}.{extension}')
         return safe_element_identifiers
 
-    def serialize(self):
-        return data_collection_input_to_staging_path_and_source_path(self)
+    def serialize(self, invalid_chars=('/',), include_collection_name=False):
+        return data_collection_input_to_staging_path_and_source_path(self, invalid_chars=invalid_chars, include_collection_name=include_collection_name)
 
     @property
     def is_input_supplied(self):
diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index 4c236fadfe7..fd4d2e56f55 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -1071,7 +1071,7 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals
     return value
 
 
-def filesystem_safe_string(s, max_len, truncation_chars='..', strip_leading_dot=True, invalid_chars=('/',), replacement_char='_'):
+def filesystem_safe_string(s, max_len=255, truncation_chars='..', strip_leading_dot=True, invalid_chars=('/',), replacement_char='_'):
     """
     Strip unicode null chars, truncate at 255 characters.
     Optionally replace additional ``invalid_chars`` with `replacement_char` .
diff --git a/tools/data_export/export_remote.xml b/tools/data_export/export_remote.xml
index 3ffaacfa4c4..bfa28422b84 100644
--- a/tools/data_export/export_remote.xml
+++ b/tools/data_export/export_remote.xml
@@ -18,9 +18,9 @@ python '$__tool_directory__/export_remote.py'
         <configfile name="files_to_export">#import json
 #from galaxy.util import filesystem_safe_string
 #if $export_type.export_type_selector == "datasets_auto":
-#set $fileconfig = json.dumps($export_type.infiles.serialize())
+#set $fileconfig = json.dumps($export_type.infiles.serialize(invalid_chars=tuple(str($invalid_chars))))
 #else if $export_type.export_type_selector == "collection_auto":
-#set $fileconfig = json.dumps($export_type.incollection.serialize())
+#set $fileconfig = json.dumps($export_type.incollection.serialize(invalid_chars=tuple(str($invalid_chars)), include_collection_name=bool($export_type.include_collection_name)))
 #else
 #set $fileconfig = [d['infile'].serialize() for d in $export_type.datasets]
 #for $i, $dataset in enumerate($export_type.datasets):
@@ -35,7 +35,7 @@ $fileconfig
         <conditional name="export_type">
             <param name="export_type_selector" type="select" label="What would you like to export?">
                 <option value="datasets_auto" selected="True">Datasets using their Galaxy name and extension.</option>
-                <option value="collection_auto">Dataset Collections using their name, structure and extension</option>
+                <option value="collection_auto">Dataset Collection using their name, structure and extension</option>
                 <option value="datasets_named">Datasets using specified name.</option>
                 <!-- Allow more options - e.g. HTML, collections preserving their structure... -->
             </param>
@@ -44,10 +44,11 @@ $fileconfig
             </when>
             <when value="collection_auto">
                 <param name="incollection" type="data_collection" format="txt,binary" label="Choose your dataset collection" help="Dataset collection will be exported. Collections will become folders and datasets will be files"/>
+                <param name="include_collection_name" type="boolean" label="Include the collection name as top level directory?"/>
             </when>
             <when value="datasets_named">
                 <repeat name="datasets" title="Dataset">
-                    <param type="data" name="infile" label="Input dataset" />
+                    <param type="data" format="txt,binary" name="infile" label="Input dataset" />
                     <param type="text" name="name" label="Enter file name" help="To create a file named &quot;f1.txt&quot; in directory named &quot;dir1&quot; enter &quot;dir1/f1.txt&quot;">
                         <!-- 🔥 turn off sanitizer, so unicode strings can be used. Is sanitized in configfile -->
                         <sanitizer sanitize="false"></sanitizer>
@@ -58,6 +59,10 @@ $fileconfig
         </conditional>
         <param type="directory_uri" name="d_uri" label="Directory URI" />
         <param name="include_metadata_files" type="boolean" checked="true" label="Include metadata files in export?" help="Examples of metadata files are bam, cram and vcf indexes that can be regenerated from raw data" />
+        <param name="invalid_chars" type="select" label="Select which characters should be stripped from the exported paths">
+            <option value="/">Sanitize /</option>
+            <option value="">Don't sanitize values</option>
+        </param>
     </inputs>
     <outputs>
         <data name="out" format="txt" label="Export logs"/>

From cfab5a352cfe87752166042e71a0e4fb4e7ded48 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 05:29:58 -0400
Subject: [PATCH 396/452] Fix qunit test, parse inputs instead of entire form
 model

---
 client/tests/qunit/tests/form_tests.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/tests/qunit/tests/form_tests.js b/client/tests/qunit/tests/form_tests.js
index 1968121230e..4c93ebf86ad 100644
--- a/client/tests/qunit/tests/form_tests.js
+++ b/client/tests/qunit/tests/form_tests.js
@@ -50,7 +50,7 @@ QUnit.test("form", function (assert) {
         "Created data correct"
     );
     var mapped_ids = [];
-    form.data.matchModel(form.model.attributes, function (input, id) {
+    form.data.matchModel(form.model.get("inputs"), function (input, id) {
         mapped_ids.push($("#" + id).attr("tour_id"));
     });
     assert.ok(

From 640eafa08450123afa177b2801321fa730e56bdc Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Wed, 25 Aug 2021 12:50:23 +0200
Subject: [PATCH 397/452] fix verify output for collections

For collections `planemo test` output for found file diff is `History item  different than expected...` which is not really helpful.

This PR adds the collection name and element identifier to the output.

Essentially a follow up to https://github.com/galaxyproject/galaxy/pull/9771
---
 lib/galaxy/tool_util/verify/interactor.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py
index dedc9b5b4fd..a0dce188875 100644
--- a/lib/galaxy/tool_util/verify/interactor.py
+++ b/lib/galaxy/tool_util/verify/interactor.py
@@ -142,13 +142,16 @@ class GalaxyInteractorApi:
 
         def verify_dataset(element, element_attrib, element_outfile):
             hda = element["object"]
-            self.verify_output_dataset(
-                history,
-                hda_id=hda["id"],
-                outfile=element_outfile,
-                attributes=element_attrib,
-                tool_id=tool_id
-            )
+            try:
+                self.verify_output_dataset(
+                    history,
+                    hda_id=hda["id"],
+                    outfile=element_outfile,
+                    attributes=element_attrib,
+                    tool_id=tool_id
+                )
+            except AssertionError as e:
+                raise AssertionError(f"Collection element {element.get('element_identifier', '')} of collection {output_collection_def.name}: {e}")
 
         verify_collection(output_collection_def, data_collection, verify_dataset)
 
@@ -869,7 +872,7 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
             element_type = element["element_type"]
             if element_type != "dataset_collection":
                 verify_dataset(element, element_attrib, element_outfile)
-            if element_type == "dataset_collection":
+            else:
                 elements = element["object"]["elements"]
                 verify_elements(elements, element_attrib.get("elements", {}))
 

From f9584509780ca5378b0df6c85c1c97d960d42349 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 25 Aug 2021 14:49:21 +0000
Subject: [PATCH 398/452] Bump tensorflow from 2.5.0 to 2.5.1 in
 /lib/galaxy/dependencies

Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/tensorflow/tensorflow/releases)
- [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md)
- [Commits](https://github.com/tensorflow/tensorflow/compare/v2.5.0...v2.5.1)

---
updated-dependencies:
- dependency-name: tensorflow
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/galaxy/dependencies/conditional-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt
index 758b8607d14..a1accf07738 100644
--- a/lib/galaxy/dependencies/conditional-requirements.txt
+++ b/lib/galaxy/dependencies/conditional-requirements.txt
@@ -38,4 +38,4 @@ pygithub
 influxdb
 
 # Deep learning packages for tool recommendation
-tensorflow==2.5.0
+tensorflow==2.5.1

From 94881812a05128ecfcc6c5dbfa15d4c0e2ecd641 Mon Sep 17 00:00:00 2001
From: Marius van den Beek <m.vandenbeek@gmail.com>
Date: Wed, 25 Aug 2021 18:03:19 +0300
Subject: [PATCH 399/452] Tweak tool options

Co-authored-by: Nicola Soranzo <nicola.soranzo@gmail.com>
---
 tools/data_export/export_remote.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/data_export/export_remote.xml b/tools/data_export/export_remote.xml
index bfa28422b84..839fd3dc609 100644
--- a/tools/data_export/export_remote.xml
+++ b/tools/data_export/export_remote.xml
@@ -34,9 +34,9 @@ $fileconfig
     <inputs>
         <conditional name="export_type">
             <param name="export_type_selector" type="select" label="What would you like to export?">
-                <option value="datasets_auto" selected="True">Datasets using their Galaxy name and extension.</option>
+                <option value="datasets_auto" selected="true">Datasets using their Galaxy name and extension</option>
                 <option value="collection_auto">Dataset Collection using their name, structure and extension</option>
-                <option value="datasets_named">Datasets using specified name.</option>
+                <option value="datasets_named">Datasets specifying their names</option>
                 <!-- Allow more options - e.g. HTML, collections preserving their structure... -->
             </param>
             <when value="datasets_auto">

From a8950564f457b8d58058d196d8a5c660893c86d8 Mon Sep 17 00:00:00 2001
From: Marius van den Beek <m.vandenbeek@gmail.com>
Date: Wed, 25 Aug 2021 18:05:43 +0300
Subject: [PATCH 400/452] Change method name to get_staging_path instead of
 staging_path

It's not a property anymore. Thanks Nuwan.

Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
---
 lib/galaxy/tools/parameters/wrapped_json.py | 2 +-
 lib/galaxy/tools/wrappers.py                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py
index bb4b0c83f0d..d596aeac600 100644
--- a/lib/galaxy/tools/parameters/wrapped_json.py
+++ b/lib/galaxy/tools/parameters/wrapped_json.py
@@ -45,7 +45,7 @@ def data_collection_input_to_staging_path_and_source_path(v, invalid_chars=('/',
 
 
 def data_input_to_staging_path_and_source_path(v, invalid_chars=('/',)):
-    staging_path = v.staging_path(invalid_chars=invalid_chars)
+    staging_path = v.get_staging_path(invalid_chars=invalid_chars)
     return {
         'staging_path': staging_path,
         'source_path': data_input_to_path(v),
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 38b36e0546f..dab52c64b79 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -315,7 +315,7 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
     def name_and_ext(self):
         return f"{self.element_identifier}.{self.file_ext}"
 
-    def staging_path(self, invalid_chars=('/',)):
+    def get_staging_path(self, invalid_chars=('/',)):
         """
         Strip leading dots, unicode null chars, replace `/` with `_`, truncate at 255 characters.
 

From 1a0cc6272d86cba1296bcfa9f971ae67cc8d9cc6 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 25 Aug 2021 11:02:48 -0400
Subject: [PATCH 401/452] Abstraction to reduce duplication in build_url_map in
 webapp.py.

---
 lib/galaxy/webapps/base/webapp.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py
index e0b749ac20d..c0adbab0d1c 100644
--- a/lib/galaxy/webapps/base/webapp.py
+++ b/lib/galaxy/webapps/base/webapp.py
@@ -1008,13 +1008,19 @@ def build_url_map(app, global_conf, local_conf):
         cache_time = int(cache_time)
     # Send to dynamic app by default
     urlmap["/"] = app
+
+    def get_static_from_config(option_name, default_path):
+        config_val = conf.get(option_name, default_url_path(default_path))
+        return Static(config_val, cache_time)
+
     # Define static mappings from config
-    urlmap["/static"] = Static(conf.get("static_dir", default_url_path("static/")), cache_time)
-    urlmap["/images"] = Static(conf.get("static_images_dir", default_url_path("static/images")), cache_time)
-    urlmap["/static/scripts"] = Static(conf.get("static_scripts_dir", default_url_path("static/scripts/")), cache_time)
-    urlmap["/static/welcome.html"] = Static(conf.get("static_welcome_html", default_url_path("static/welcome.html")), cache_time)
-    urlmap["/favicon.ico"] = Static(conf.get("static_favicon_dir", default_url_path("static/favicon.ico")), cache_time)
-    urlmap["/robots.txt"] = Static(conf.get("static_robots_txt", default_url_path("static/robots.txt")), cache_time)
+    urlmap["/static"] = get_static_from_config("static_dir", "static/")
+    urlmap["/images"] = get_static_from_config("static_images_dir", "static/images")
+    urlmap["/static/scripts"] = get_static_from_config("static_scripts_dir", "static/scripts/")
+    urlmap["/static/welcome.html"] = get_static_from_config("static_welcome_html", "static/welcome.html")
+    urlmap["/favicon.ico"] = get_static_from_config("static_favicon_dir", "static/favicon.ico")
+    urlmap["/robots.txt"] = get_static_from_config("static_robots_txt", "static/robots.txt")
+
     if 'static_local_dir' in conf:
         urlmap["/static_local"] = Static(conf["static_local_dir"], cache_time)
     return urlmap, cache_time

From 1765ed7c6d8ab2b5c318a5f839e8ed9e27338ac8 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 25 Aug 2021 11:44:16 -0400
Subject: [PATCH 402/452] per host configurable static site configs

---
 lib/galaxy/web/framework/middleware/static.py | 14 ++++++++++++--
 lib/galaxy/webapps/base/webapp.py             |  4 +++-
 lib/galaxy/webapps/galaxy/config_schema.yml   |  6 ++++++
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/web/framework/middleware/static.py b/lib/galaxy/web/framework/middleware/static.py
index d8aba84921d..65213d7af8b 100644
--- a/lib/galaxy/web/framework/middleware/static.py
+++ b/lib/galaxy/web/framework/middleware/static.py
@@ -8,9 +8,10 @@ from paste.urlparser import StaticURLParser
 
 class CacheableStaticURLParser(StaticURLParser):
 
-    def __init__(self, directory, cache_seconds=None):
+    def __init__(self, directory, cache_seconds=None, directory_per_host=None):
         StaticURLParser.__init__(self, directory)
         self.cache_seconds = cache_seconds
+        self.directory_per_host = directory_per_host
 
     def __call__(self, environ, start_response):
         path_info = environ.get('PATH_INFO', '')
@@ -27,7 +28,16 @@ class CacheableStaticURLParser(StaticURLParser):
             filename = 'index.html'
         else:
             filename = request.path_info_pop(environ)
-        full = os.path.join(self.directory, filename)
+
+        directory = self.directory
+        host = environ.get('HTTP_HOST')
+        if self.directory_per_host and host:
+            for host_key, host_val in self.directory_per_host.items():
+                if host_key in host:
+                    directory = host_val
+                    break
+
+        full = os.path.join(directory, filename)
         if not os.path.exists(full):
             return self.not_found(environ, start_response)
         if os.path.isdir(full):
diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py
index c0adbab0d1c..82d9c7c2873 100644
--- a/lib/galaxy/webapps/base/webapp.py
+++ b/lib/galaxy/webapps/base/webapp.py
@@ -1011,7 +1011,9 @@ def build_url_map(app, global_conf, local_conf):
 
     def get_static_from_config(option_name, default_path):
         config_val = conf.get(option_name, default_url_path(default_path))
-        return Static(config_val, cache_time)
+        per_host_config_option = f"{option_name}_by_host"
+        per_host_config = conf.get(per_host_config_option)
+        return Static(config_val, cache_time, directory_per_host=per_host_config)
 
     # Define static mappings from config
     urlmap["/static"] = get_static_from_config("static_dir", "static/")
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 3a42699e711..67b5e701d54 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -1479,6 +1479,7 @@ mapping:
       static_dir:
         type: str
         default: static/
+        per_host: true
         required: false
         desc: |
           Serve static content, which must be enabled if you're not serving it via a
@@ -1489,6 +1490,7 @@ mapping:
       static_images_dir:
         type: str
         default: static/images
+        per_host: true
         required: false
         desc: |
           Serve static content, which must be enabled if you're not serving it via a
@@ -1499,6 +1501,7 @@ mapping:
       static_favicon_dir:
         type: str
         default: static/favicon.ico
+        per_host: true
         required: false
         desc: |
           Serve static content, which must be enabled if you're not serving it via a
@@ -1509,6 +1512,7 @@ mapping:
       static_scripts_dir:
         type: str
         default: static/scripts/
+        per_host: true
         required: false
         desc: |
           Serve static content, which must be enabled if you're not serving it via a
@@ -1519,6 +1523,7 @@ mapping:
       static_style_dir:
         type: str
         default: static/style
+        per_host: true
         required: false
         desc: |
           Serve static content, which must be enabled if you're not serving it via a
@@ -1529,6 +1534,7 @@ mapping:
       static_robots_txt:
         type: str
         default: static/robots.txt
+        per_host: true
         required: false
         desc: |
           Serve static content, which must be enabled if you're not serving it via a

From ebf1e2435e52d05c109f8b6750924e915cf67ee5 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 25 Aug 2021 19:03:34 +0300
Subject: [PATCH 403/452] Rename all_staging_paths to get_all_staging_paths

---
 lib/galaxy/tools/parameters/wrapped_json.py | 2 +-
 lib/galaxy/tools/wrappers.py                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py
index d596aeac600..1e419204437 100644
--- a/lib/galaxy/tools/parameters/wrapped_json.py
+++ b/lib/galaxy/tools/parameters/wrapped_json.py
@@ -34,7 +34,7 @@ def data_collection_input_to_path(v):
 
 
 def data_collection_input_to_staging_path_and_source_path(v, invalid_chars=('/',), include_collection_name=False):
-    staging_paths = v.all_staging_paths(invalid_chars=invalid_chars, include_collection_name=include_collection_name)
+    staging_paths = v.get_all_staging_paths(invalid_chars=invalid_chars, include_collection_name=include_collection_name)
     source_paths = v.all_paths
     metadata_files = v.all_metadata_files
     return [
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index dab52c64b79..5da9940b158 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -548,7 +548,7 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
                 return []
         return self._element_identifiers_extensions_paths_and_metadata_files
 
-    def all_staging_paths(self, invalid_chars=('/',), include_collection_name=False):
+    def get_all_staging_paths(self, invalid_chars=('/',), include_collection_name=False):
         safe_element_identifiers = []
         for element_identifiers, extension, *_ in self.element_identifiers_extensions_paths_and_metadata_files:
             datatype = self.datatypes_registry.get_datatype_by_extension(extension)

From 944e28a50760c61d652c148484b72548a1ba1c4a Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Wed, 25 Aug 2021 21:38:52 +0530
Subject: [PATCH 404/452] Remove tool_conf.xml.main which is no longer used

---
 lib/galaxy/config/sample/tool_conf.xml.main | 139 --------------------
 run_tests.sh                                |   2 +-
 2 files changed, 1 insertion(+), 140 deletions(-)
 delete mode 100644 lib/galaxy/config/sample/tool_conf.xml.main

diff --git a/lib/galaxy/config/sample/tool_conf.xml.main b/lib/galaxy/config/sample/tool_conf.xml.main
deleted file mode 100644
index 7569b7ad0d7..00000000000
--- a/lib/galaxy/config/sample/tool_conf.xml.main
+++ /dev/null
@@ -1,139 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<toolbox monitor="true">
-  <section id="getext" name="Get Data">
-    <tool file="data_source/upload.xml" />
-    <tool file="data_source/ucsc_tablebrowser.xml" />
-    <tool file="data_source/ucsc_tablebrowser_archaea.xml" />
-    <tool file="data_source/ebi_sra.xml" />
-    <tool file="data_source/intermine.xml" />
-    <tool file="data_source/flymine.xml" />
-    <tool file="data_source/fly_modencode.xml" />
-    <tool file="data_source/modmine.xml" />
-    <tool file="data_source/mousemine.xml" />
-    <tool file="data_source/ratmine.xml" />
-    <tool file="data_source/yeastmine.xml" />
-    <tool file="data_source/worm_modencode.xml" />
-    <tool file="data_source/wormbase.xml" />
-    <tool file="data_source/zebrafishmine.xml" />
-    <tool file="data_source/eupathdb.xml" />
-  </section>
-  <section id="send" name="Send Data">
-    <tool file="cloud/send.xml" />
-  </section>
-  <section id="liftOver" name="Lift-Over">
-    <tool file="extract/liftOver_wrapper.xml" />
-  </section>
-  <section id="collection_operations" name="Collection Operations">
-    <tool file="${model_tools_path}/unzip_collection.xml" labels="new" />
-    <tool file="${model_tools_path}/zip_collection.xml" labels="new" />
-    <tool file="${model_tools_path}/filter_failed_collection.xml" labels="new" />
-    <tool file="${model_tools_path}/flatten_collection.xml" labels="new" />
-    <tool file="${model_tools_path}/merge_collection.xml" labels="new" />
-    <tool file="${model_tools_path}/relabel_from_file.xml" />
-    <tool file="${model_tools_path}/filter_from_file.xml" />
-    <tool file="${model_tools_path}/sort_collection_list.xml" />
-    <tool file="${model_tools_path}/apply_rules.xml" />
-  </section>
-  <section id="textutil" name="Text Manipulation">
-    <tool file="filters/fixedValueColumn.xml" />
-    <tool file="filters/catWrapper.xml" hidden="true" />
-    <tool file="filters/condense_characters.xml" />
-    <tool file="filters/convert_characters.xml" />
-    <tool file="filters/mergeCols.xml" />
-    <tool file="filters/CreateInterval.xml" />
-    <tool file="filters/cutWrapper.xml" hidden="true" />
-    <tool file="filters/changeCase.xml" />
-    <tool file="filters/pasteWrapper.xml" />
-    <tool file="filters/remove_beginning.xml" />
-    <tool file="filters/randomlines.xml" />
-    <tool file="filters/headWrapper.xml" />
-    <tool file="filters/tailWrapper.xml" />
-    <tool file="filters/trimmer.xml" />
-    <tool file="filters/wc_gnu.xml" />
-    <tool file="filters/secure_hash_message_digest.xml" />
-  </section>
-  <section id="convert" name="Convert Formats">
-    <tool file="filters/bed2gff.xml" />
-    <tool file="filters/gff2bed.xml" />
-    <tool file="maf/maf_to_bed.xml" />
-    <tool file="maf/maf_to_interval.xml" />
-    <tool file="maf/maf_to_fasta.xml" />
-    <tool file="filters/sff_extractor.xml" />
-    <tool file="filters/wig_to_bigwig.xml" />
-    <tool file="filters/bed_to_bigbed.xml" />
-  </section>
-  <section id="filter" name="Filter and Sort">
-    <tool file="stats/filtering.xml" />
-    <tool file="filters/sorter.xml" />
-    <tool file="filters/grep.xml" />
-    <label id="gff" text="GFF" />
-    <tool file="filters/gff/extract_GFF_Features.xml" />
-    <tool file="filters/gff/gff_filter_by_attribute.xml" />
-    <tool file="filters/gff/gff_filter_by_feature_count.xml" />
-    <tool file="filters/gff/gtf_filter_by_attribute_values_list.xml" />
-  </section>
-  <section id="group" name="Join, Subtract and Group">
-    <tool file="filters/joiner.xml" />
-    <tool file="filters/compare.xml" />
-    <tool file="stats/grouping.xml" />
-  </section>
-  <section id="fetchAlignSeq" name="Fetch Alignments/Sequences">
-    <tool file="maf/interval2maf_pairwise.xml" />
-    <tool file="maf/interval2maf.xml" />
-    <tool file="maf/interval_maf_to_merged_fasta.xml" />
-    <tool file="maf/genebed_maf_to_fasta.xml" />
-    <tool file="maf/maf_stats.xml" />
-    <tool file="maf/maf_thread_for_species.xml" />
-    <tool file="maf/maf_limit_to_species.xml" />
-    <tool file="maf/maf_limit_size.xml" />
-    <tool file="maf/maf_by_block_number.xml" />
-    <tool file="maf/maf_filter.xml" />
-    <tool file="maf/maf_reverse_complement.xml" />
-    <tool file="extract/extract_genomic_dna.xml" />
-  </section>
-  <section id="bxops" name="Operate on Genomic Intervals" version="">
-    <tool file="filters/wiggle_to_simple.xml" />
-    <tool file="stats/aggregate_binned_scores_in_intervals.xml" />
-    <tool file="filters/ucsc_gene_bed_to_exon_bed.xml" />
-  </section>
-  <section id="stats" name="Statistics">
-    <tool file="stats/gsummary.xml" />
-    <tool file="filters/uniq.xml" />
-  </section>
-  <section id="plots" name="Graph/Display Data">
-    <tool file="plotting/boxplot.xml" hidden="true" />
-    <tool file="maf/vcf_to_maf_customtrack.xml" />
-  </section>
-  <section id="hgv" name="Phenotype Association">
-    <tool file="evolution/codingSnps.xml" />
-    <tool file="evolution/add_scores.xml" />
-    <tool file="phenotype_association/sift.xml" />
-    <tool file="phenotype_association/linkToGProfile.xml" />
-    <tool file="phenotype_association/linkToDavid.xml" />
-    <tool file="phenotype_association/ldtools.xml" />
-    <tool file="phenotype_association/pass.xml" />
-    <tool file="phenotype_association/gpass.xml" />
-    <tool file="phenotype_association/beam.xml" />
-    <tool file="phenotype_association/lps.xml" />
-    <tool file="phenotype_association/master2pg.xml" />
-  </section>
-  <section id="cshl_library_information" name="NGS: QC and manipulation">
-    <label id="illumina" text="Illumina data" />
-    <label id="454" text="Roche-454 data" />
-    <label id="solid" text="AB-SOLiD data" />
-        <tool file="next_gen_conversion/solid2fastq.xml" />
-        <tool file="solid_tools/solid_qual_stats.xml" />
-        <tool file="solid_tools/solid_qual_boxplot.xml" />
-    <label id="generic_fastq" text="Generic FASTQ manipulation" />
-    <label id="fastx_toolkit_fastq" text="FASTX-Toolkit for FASTQ data" />
-  </section>
-  <section id="ngs_mapping" name="NGS: Mapping">
-    <label id="illumina" text="Illumina" />
-    <label id="roche_454" text="Roche-454" />
-    <label id="ab_solid" text="AB-SOLiD" />
-  </section>
-  <label id="deprecated" text="DEPRECATED" />
-  <section id="gatk" name="NGS: GATK Tools (beta)">
-  </section>
-
-</toolbox>
diff --git a/run_tests.sh b/run_tests.sh
index 414d315e370..e1023af1d56 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -388,7 +388,7 @@ do
           shift 1
           ;;
       -main|-main_tools|--main_tools)
-          GALAXY_TEST_TOOL_CONF="lib/galaxy/config/sample/tool_conf.xml.sample,config/tool_conf.xml.main"
+          GALAXY_TEST_TOOL_CONF="lib/galaxy/config/sample/tool_conf.xml.sample"
           marker="tool"
           test_script="pytest"
           report_file="run_framework_tests.html"

From 3bf8e59b0944c157437fb979c85e0fa4e236a179 Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Wed, 25 Aug 2021 12:20:36 -0400
Subject: [PATCH 405/452] Removing unused poolsize from code and config. This
 seems to be stale code for a previous effort to manage connection pool size
 by Galaxy (which was abandoned)

---
 lib/galaxy/config/sample/object_store_conf.xml.sample | 2 +-
 lib/galaxy/objectstore/irods.py                       | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample
index 3ed6341e74d..19043bab495 100644
--- a/lib/galaxy/config/sample/object_store_conf.xml.sample
+++ b/lib/galaxy/config/sample/object_store_conf.xml.sample
@@ -48,7 +48,7 @@
             <auth username="rods" password="rods" />
             <resource name="demoResc" />
             <zone name="tempZone" />
-            <connection host="localhost" port="1247" timeout="30" poolsize="3" refresh_time="300"/>
+            <connection host="localhost" port="1247" timeout="30" refresh_time="300"/>
             <cache path="database/object_store_cache_irods" size="1000" />
             <extra_dir type="job_work" path="database/job_working_directory_irods"/>
             <extra_dir type="temp" path="database/tmp_irods"/>
diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py
index b536bba0c87..9c504eafd82 100644
--- a/lib/galaxy/objectstore/irods.py
+++ b/lib/galaxy/objectstore/irods.py
@@ -62,7 +62,6 @@ def parse_config_xml(config_xml):
         host = c_xml[0].get('host', None)
         port = int(c_xml[0].get('port', 0))
         timeout = int(c_xml[0].get('timeout', 30))
-        poolsize = int(c_xml[0].get('poolsize', 3))
         refresh_time = int(c_xml[0].get('refresh_time', 300))
 
         c_xml = config_xml.findall('cache')
@@ -92,7 +91,6 @@ def parse_config_xml(config_xml):
                 'host': host,
                 'port': port,
                 'timeout': timeout,
-                'poolsize': poolsize,
                 'refresh_time': refresh_time,
             },
             'cache': {
@@ -125,7 +123,6 @@ class CloudConfigMixin:
                 'host': self.host,
                 'port': self.port,
                 'timeout': self.timeout,
-                'poolsize': self.poolsize,
                 'refresh_time': self.refresh_time,
             },
             'cache': {
@@ -184,9 +181,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin):
         self.timeout = connection_dict.get('timeout')
         if self.timeout is None:
             _config_dict_error('connection->timeout')
-        self.poolsize = connection_dict.get('poolsize')
-        if self.poolsize is None:
-            _config_dict_error('connection->poolsize')
         self.refresh_time = connection_dict.get('refresh_time')
         if self.refresh_time is None:
             _config_dict_error('connection->refresh_time')

From 9586a6c8da5904be651b822cc547900d5c182fcf Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 13:15:51 -0400
Subject: [PATCH 406/452] Remove skipOnClone flag, previously required to
 filter custom attributes before cloning a node

---
 client/src/components/Workflow/Editor/Forms/makeSection.js | 3 ---
 client/src/components/Workflow/Editor/Index.vue            | 6 +-----
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/client/src/components/Workflow/Editor/Forms/makeSection.js b/client/src/components/Workflow/Editor/Forms/makeSection.js
index c8a98e027c8..56d7e1b5de5 100644
--- a/client/src/components/Workflow/Editor/Forms/makeSection.js
+++ b/client/src/components/Workflow/Editor/Forms/makeSection.js
@@ -81,7 +81,6 @@ function _makeSection(node, output, datatypes, onOutputLabel, onOutputDatatype)
     const activeOutput = node.activeOutputs.get(output.name);
     const inputTitle = output.label || output.name;
     const inputConfig = {
-        skipOnClone: true,
         title: `Configure Output: '${inputTitle}'`,
         type: "section",
         flat: true,
@@ -207,7 +206,6 @@ export default function makeSection(node, datatypes, onOutputLabel, onOutputData
             payload: {
                 host: window.location.host,
             },
-            skipOnClone: true,
         });
         inputs.push({
             name: `pja__${outputFirst.name}__DeleteIntermediatesAction`,
@@ -216,7 +214,6 @@ export default function makeSection(node, datatypes, onOutputLabel, onOutputData
             value: String(Boolean(postJobActions[`DeleteIntermediatesAction${outputFirst.name}`])),
             ignore: "false",
             help: "Upon completion of this step, delete non-starred outputs from completed workflow steps if they are no longer required as inputs.",
-            skipOnClone: true,
         });
         for (const output of node.outputs) {
             inputs.push(_makeSection(node, output, datatypes, onOutputLabel, onOutputDatatype));
diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue
index 102fbe755ba..6a7722359c4 100644
--- a/client/src/components/Workflow/Editor/Index.vue
+++ b/client/src/components/Workflow/Editor/Index.vue
@@ -420,16 +420,12 @@ export default {
         onClone(node) {
             const newId = this.nodeIndex++;
             const stepCopy = JSON.parse(JSON.stringify(node.step));
-            const configFormCopy = JSON.parse(JSON.stringify(node.config_form));
             Vue.set(this.steps, newId, {
                 ...stepCopy,
                 id: newId,
-                config_form: {
-                    ...configFormCopy,
-                    inputs: configFormCopy.inputs.filter((input) => !input.skipOnClone),
-                },
                 uuid: null,
                 label: null,
+                config_form: JSON.parse(JSON.stringify(node.config_form)),
                 annotation: JSON.parse(JSON.stringify(node.annotation)),
                 tool_state: JSON.parse(JSON.stringify(node.tool_state)),
                 post_job_actions: JSON.parse(JSON.stringify(node.postJobActions)),

From 5d4d5a8cad9f2c54a1bb6e0ecc3d24f51f545a03 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Wed, 25 Aug 2021 12:18:51 -0700
Subject: [PATCH 407/452] added migration for job state view

---
 .../versions/0177_update_job_state_summary.py | 26 ++++++++++++++++
 lib/galaxy/model/view/__init__.py             | 30 ++++++++++++-------
 2 files changed, 46 insertions(+), 10 deletions(-)
 create mode 100644 lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py

diff --git a/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py b/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
new file mode 100644
index 00000000000..352c3bcb669
--- /dev/null
+++ b/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
@@ -0,0 +1,26 @@
+"""
+Update job-state-summary view for hdca elements to include job directly tied with the hdca
+"""
+
+import logging
+
+from galaxy.model.view import HistoryDatasetCollectionJobStateSummary
+from galaxy.model.view.utils import CreateView, DropView
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    print(__doc__)
+    # drop first because sqlite does not support or_replace
+    downgrade(migrate_engine)
+    view = HistoryDatasetCollectionJobStateSummary
+    create_view = CreateView(view.name, view.__view__)
+    # print(str(create_view.compile(migrate_engine)))
+    migrate_engine.execute(create_view)
+
+
+def downgrade(migrate_engine):
+    drop_view = DropView(HistoryDatasetCollectionJobStateSummary.name)
+    # print(str(drop_view.compile(migrate_engine)))
+    migrate_engine.execute(drop_view)
diff --git a/lib/galaxy/model/view/__init__.py b/lib/galaxy/model/view/__init__.py
index ae1958da9fb..7d53206e372 100644
--- a/lib/galaxy/model/view/__init__.py
+++ b/lib/galaxy/model/view/__init__.py
@@ -9,7 +9,7 @@ from galaxy.model.view.utils import View
 
 AGGREGATE_STATE_QUERY = """
 SELECT
-    hdca.id as hdca_id,
+    hdca_id,
     SUM(CASE WHEN state = 'new' THEN 1 ELSE 0 END) AS new,
     SUM(CASE WHEN state = 'resubmitted' THEN 1 ELSE 0 END) AS resubmitted,
     SUM(CASE WHEN state = 'waiting' THEN 1 ELSE 0 END) AS waiting,
@@ -22,15 +22,25 @@ SELECT
     SUM(CASE WHEN state = 'deleted' THEN 1 ELSE 0 END) AS deleted,
     SUM(CASE WHEN state = 'deleted_new' THEN 1 ELSE 0 END) AS deleted_new,
     SUM(CASE WHEN state = 'upload' THEN 1 ELSE 0 END) AS upload,
-    SUM(CASE WHEN job.id IS NOT NULL THEN 1 ELSE 0 END) AS all_jobs
-FROM history_dataset_collection_association hdca
-LEFT JOIN implicit_collection_jobs icj
-    ON icj.id = hdca.implicit_collection_jobs_id
-LEFT JOIN implicit_collection_jobs_job_association icjja
-    ON icj.id = icjja.implicit_collection_jobs_id
-LEFT JOIN job
-    ON icjja.job_id = job.id
-GROUP BY hdca.id
+    SUM(CASE WHEN job_id IS NOT NULL THEN 1 ELSE 0 END) AS all_jobs
+FROM (
+    SELECT hdca.id AS hdca_id, job.id AS job_id, job.state as state
+    FROM history_dataset_collection_association hdca
+    LEFT JOIN implicit_collection_jobs icj
+        ON icj.id = hdca.implicit_collection_jobs_id
+    LEFT JOIN implicit_collection_jobs_job_association icjja
+        ON icj.id = icjja.implicit_collection_jobs_id
+    LEFT JOIN job
+        ON icjja.job_id = job.id
+
+    UNION
+
+    SELECT hdca.id AS hdca_id, job.id AS job_id, job.state AS state
+    FROM history_dataset_collection_association hdca
+    LEFT JOIN job
+        ON hdca.job_id = job.id
+) jobstates
+GROUP BY jobstates.hdca_id
 """
 
 

From c53a39b3af68a93df803382ef00a8021da18accb Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Wed, 25 Aug 2021 13:03:19 -0700
Subject: [PATCH 408/452] added api test for contents/near since parameter

---
 lib/galaxy_test/api/test_history_contents.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 3919a6906d4..98591c466b2 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -597,6 +597,18 @@ class HistoryContentsApiTestCase(ApiTestCase):
             all_datasets_finished = first_time = datetime.utcnow().isoformat()
             assert len(self._get_content(history_id, update_time=all_datasets_finished)) == 0
 
+    def test_history_contents_near_with_since(self):
+        with self.dataset_populator.test_history() as history_id:
+            first_time = datetime.utcnow().isoformat()
+            assert len(self._get_content(history_id, update_time=first_time)) == 0
+            self.dataset_collection_populator.create_list_in_history(history_id=history_id)
+            assert len(self._get_content(history_id, update_time=first_time)) == 4  # 3 datasets
+            self.dataset_populator.wait_for_history(history_id)
+            all_datasets_finished = datetime.utcnow().isoformat()
+            # should return a 204 if history has not changed at all
+            response = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={all_datasets_finished}")
+            assert response.status_code == 204
+
     @skip_without_tool('cat_data_and_sleep')
     def test_history_contents_near_with_update_time_implicit_collection(self):
         with self.dataset_populator.test_history() as history_id:

From 89c5b723b5b57c80eb9d3e64cd13befb1dcfb8b3 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Wed, 25 Aug 2021 13:12:06 -0700
Subject: [PATCH 409/452] Fixed since parameter in contents/near

---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 2 +-
 lib/galaxy_test/api/test_history_contents.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 0f6dfdeff2c..7567b5f89c7 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -1772,7 +1772,7 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         filter_params = self._parse_rest_params(kwd)
         hid = int(hid)
         limit = int(limit)
-        since = kwd.get('update_time-gt', None)
+        since = kwd.get('since', None)
         if since:
             since = dateutil.parser.isoparse(since)
 
diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 98591c466b2..51918f5ee64 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -585,7 +585,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
             assert isinstance(c['job_state_summary'], dict)
 
     def _get_content(self, history_id, update_time):
-        return self._get(f"/api/histories/{history_id}/contents/near/100/100?update_time-ge={update_time}").json()
+        return self._get(f"/api/histories/{history_id}/contents/near/100/100?update_time-gt={update_time}").json()
 
     def test_history_contents_near_with_update_time(self):
         with self.dataset_populator.test_history() as history_id:

From 6499a8671e6566e3b138720d84bbae38099042a5 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Wed, 25 Aug 2021 16:17:06 -0700
Subject: [PATCH 410/452] fixed since date parameter processing on
 contents/near

---
 .../History/caching/loadHistoryContents.js    |  6 +-
 .../webapps/galaxy/api/history_contents.py    | 17 ++++--
 lib/galaxy_test/api/test_history_contents.py  | 55 ++++++++++++++++---
 3 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/client/src/components/History/caching/loadHistoryContents.js b/client/src/components/History/caching/loadHistoryContents.js
index 533081ca52e..0941297c382 100644
--- a/client/src/components/History/caching/loadHistoryContents.js
+++ b/client/src/components/History/caching/loadHistoryContents.js
@@ -22,7 +22,7 @@ export const clearHistoryDateStore = async () => {
  */
 // prettier-ignore
 export const loadHistoryContents = (cfg = {}) => (rawInputs$) => {
-    const { 
+    const {
         noInitial = false,
         windowSize = SearchParams.pageSize
     } = cfg;
@@ -39,7 +39,7 @@ export const loadHistoryContents = (cfg = {}) => (rawInputs$) => {
             return `${baseUrl}?${params.historyContentQueryString}`;
         }),
         map(prependPath),
-        requestWithUpdateTime({ dateStore, noInitial, responseQualifier }),
+        requestWithUpdateTime({ dateStore, noInitial, responseQualifier, dateFieldName: "since" }),
     );
 
     const validResponses$ = ajaxResponse$.pipe(
@@ -81,7 +81,7 @@ export const loadHistoryContents = (cfg = {}) => (rawInputs$) => {
 
             const matches = areDefined(matchesUp, matchesDown) ? matchesUp + matchesDown : undefined;
             const totalMatches = areDefined(totalMatchesUp, totalMatchesDown) ? totalMatchesUp + totalMatchesDown : undefined;
-            
+
             return {
                 summary,
                 matches,
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 7567b5f89c7..ad952fada65 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -790,8 +790,14 @@ class HistoriesContentsService(ServiceBase):
             # sqlalchemy DateTime columns are not timezone aware, so parse `since` into timezone-aware
             # datetime and then convert to naive datetime object representing UTC,
             # assuming history.update_time represents UTC time.
-            since_date = since.astimezone(datetime.timezone.utc).replace(tzinfo=None)
-            if history.update_time <= since_date:
+            # this is the right format, but changes the date itself resulting in logical errors
+
+            # Although this creates a date with the right format for SQLAlchemy, it changes the date
+            # data resulting in logical errors during testing. I fed back the history.update_time
+            # into this conversion and it fails the inequality
+            # since_date = since.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+
+            if history.update_time <= since:
                 trans.response.status = 204
                 return
 
@@ -1769,12 +1775,13 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         * GET /api/histories/{history_id}/contents/near/{hid}/{limit}
         """
         serialization_params = parse_serialization_params(default_view='betawebclient', **kwd)
+        since = kwd.pop('since', None)
+        if since:
+            since = since.replace("Z", "")
+            since = dateutil.parser.isoparse(since)
         filter_params = self._parse_rest_params(kwd)
         hid = int(hid)
         limit = int(limit)
-        since = kwd.get('since', None)
-        if since:
-            since = dateutil.parser.isoparse(since)
 
         return self.service.contents_near(
             trans, history_id, serialization_params, filter_params, hid, limit, since,
diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 51918f5ee64..3a29ed3ada8 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -599,15 +599,56 @@ class HistoryContentsApiTestCase(ApiTestCase):
 
     def test_history_contents_near_with_since(self):
         with self.dataset_populator.test_history() as history_id:
-            first_time = datetime.utcnow().isoformat()
-            assert len(self._get_content(history_id, update_time=first_time)) == 0
+            original_history = self._get(f"/api/histories/{history_id}").json()
+            original_history_stamp = original_history['update_time']
+
+            # check empty contents, with no since flag, should return an empty 200 result
+            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100")
+            assert history_contents.status_code == 200
+            assert len(history_contents.json()) == 0
+
+            # adding a since parameter, should return a 204 if history has not changed at all
+            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}")
+            assert history_contents.status_code == 204
+
+            # add some stuff
             self.dataset_collection_populator.create_list_in_history(history_id=history_id)
-            assert len(self._get_content(history_id, update_time=first_time)) == 4  # 3 datasets
             self.dataset_populator.wait_for_history(history_id)
-            all_datasets_finished = datetime.utcnow().isoformat()
-            # should return a 204 if history has not changed at all
-            response = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={all_datasets_finished}")
-            assert response.status_code == 204
+
+            # check to make sure the added stuff is there
+            changed_history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100")
+            assert changed_history_contents.status_code == 200
+            assert len(changed_history_contents.json()) == 4
+
+            # check to make sure the history date has actually changed due to changing the contents
+            changed_history = self._get(f"/api/histories/{history_id}").json()
+            changed_history_stamp = changed_history['update_time']
+            assert original_history_stamp != changed_history_stamp
+
+            # a repeated contents request with since=original_history_stamp should now return data
+            # because we have added datasets and the update_time should have been changed
+            changed_content = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}")
+            assert changed_content.status_code == 200
+            assert len(changed_content.json()) == 4
+
+    def test_history_contents_near_since_with_standard_iso8601_date(self):
+        with self.dataset_populator.test_history() as history_id:
+            original_history = self._get(f"/api/histories/{history_id}").json()
+            original_history_stamp = original_history['update_time']
+
+            # this is the standard date format that javascript will emit using .toISOString(), it
+            # should be the expected date format for any modern api
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString
+            valid_iso8601_date = original_history_stamp + 'Z'
+            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={valid_iso8601_date}")
+            status = history_contents.status_code
+            assert status == 204
+
+            # Javascript has less date precision than python, here's a common sample
+            javascript_iso_date = '2021-08-25T21:46:49.091Z'
+            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={javascript_iso_date}")
+            status = history_contents.status_code
+            assert status == 204 or status == 200
 
     @skip_without_tool('cat_data_and_sleep')
     def test_history_contents_near_with_update_time_implicit_collection(self):

From 054612e1ec3be2964e84345b9d29000d517531f5 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 21:41:13 -0400
Subject: [PATCH 411/452] Remove unnecessary initial-error prop, show errors if
 parsed directly, only used in workflow editor tool form

---
 client/src/components/Form/FormDisplay.vue      | 17 +++++------------
 client/src/components/Tool/ToolForm.vue         |  1 -
 .../Workflow/Editor/Forms/FormTool.vue          |  1 -
 .../Workflow/Run/WorkflowRunToolStep.vue        |  1 -
 client/src/mvc/form/form-view.js                |  3 ---
 5 files changed, 5 insertions(+), 18 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 278c96c2b62..682a9cb29fb 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -17,6 +17,10 @@ export default {
             type: Array,
             required: true,
         },
+        errors: {
+            type: Object,
+            default: null,
+        },
         sustainRepeats: {
             type: Boolean,
             default: false,
@@ -37,18 +41,10 @@ export default {
             type: String,
             default: null,
         },
-        initialErrors: {
-            type: Boolean,
-            default: false,
-        },
         validationScrollTo: {
             type: Array,
             default: null,
         },
-        errors: {
-            type: Object,
-            default: null,
-        },
         replaceParams: {
             type: Object,
             default: null,
@@ -77,9 +73,7 @@ export default {
         },
         errors() {
             this.$nextTick(() => {
-                if (this.initialErrors) {
-                    this.form.errors(this.errors);
-                }
+                this.form.errors(this.errors);
             });
         },
         replaceParams() {
@@ -177,7 +171,6 @@ export default {
                 this.form = new Form({
                     el,
                     inputs: this.inputs,
-                    initial_errors: this.initialErrors,
                     text_enable: this.textEnable,
                     text_disable: this.textDisable,
                     sustain_repeats: this.sustainRepeats,
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index ddecee3ee9b..3934194f2f1 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -43,7 +43,6 @@
                                 :id="formConfig.id"
                                 :inputs="formConfig.inputs"
                                 :validation-scroll-to="validationScrollTo"
-                                :errors="formConfig.errors"
                                 @onChange="onChange"
                                 @onValidation="onValidation"
                             />
diff --git a/client/src/components/Workflow/Editor/Forms/FormTool.vue b/client/src/components/Workflow/Editor/Forms/FormTool.vue
index fb8abe3d5d1..d75cc6a7fe2 100644
--- a/client/src/components/Workflow/Editor/Forms/FormTool.vue
+++ b/client/src/components/Workflow/Editor/Forms/FormTool.vue
@@ -32,7 +32,6 @@
                 <FormDisplay
                     :id="id"
                     :inputs="inputs"
-                    :initial-errors="true"
                     :errors="errors"
                     text-enable="Set in Advance"
                     text-disable="Set at Runtime"
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index a8f18e89f7f..14eb74f948f 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -5,7 +5,6 @@
                 <FormMessage :message="errorText" variant="danger" :persistent="true" />
                 <FormDisplay
                     :inputs="formConfig.inputs"
-                    :errors="formConfig.errors"
                     :sustain-repeats="true"
                     :sustain-conditionals="true"
                     :replace-params="replaceParams"
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index 4364479a760..527db13bede 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -91,9 +91,6 @@ export default Backbone.View.extend({
         this.data = new FormData.Manager(this);
         this._renderForm();
         this.data.create();
-        if (this.model.get("initial_errors")) {
-            this.errors(this.model.get("errors"));
-        }
         // add listener which triggers on checksum change, and reset the form input wrappers
         var current_check = this.data.checksum();
         this.on("change", (input_id) => {

From 2d2c6f687e5803d11c765f7e774028b468764dcb Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 21:45:33 -0400
Subject: [PATCH 412/452] Remove unused hide-operations prop from FormDisplay,
 operations are not rendered by FormDisplay anymore

---
 client/src/components/Form/FormDisplay.vue | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index 682a9cb29fb..ccba74bf1b8 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -29,10 +29,6 @@ export default {
             type: Boolean,
             default: false,
         },
-        hideOperations: {
-            type: Boolean,
-            default: false,
-        },
         textEnable: {
             type: String,
             default: null,

From c855a7ad13ff52ffa32ac998226026ac6202ac74 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Wed, 25 Aug 2021 22:02:00 -0400
Subject: [PATCH 413/452] Restore always refresh option in forms to reduce
 update request frequency

---
 client/src/components/Form/FormDisplay.vue    |  8 ++++----
 client/src/components/Tool/ToolForm.vue       |  6 ++++--
 .../Workflow/Run/WorkflowRunToolStep.vue      | 20 ++++++++++---------
 client/src/mvc/form/form-view.js              | 13 +++++-------
 4 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/client/src/components/Form/FormDisplay.vue b/client/src/components/Form/FormDisplay.vue
index ccba74bf1b8..d7dc8ac0778 100644
--- a/client/src/components/Form/FormDisplay.vue
+++ b/client/src/components/Form/FormDisplay.vue
@@ -157,9 +157,9 @@ export default {
                 this.form.trigger("change");
             }
         },
-        onChange() {
+        onChange(refreshRequest) {
             this.formData = this.form.data.create();
-            this.$emit("onChange", this.formData);
+            this.$emit("onChange", this.formData, refreshRequest);
         },
         onRender() {
             this.$nextTick(() => {
@@ -171,8 +171,8 @@ export default {
                     text_disable: this.textDisable,
                     sustain_repeats: this.sustainRepeats,
                     sustain_conditionals: this.sustainConditionals,
-                    onchange: () => {
-                        this.onChange();
+                    onchange: (refreshRequest) => {
+                        this.onChange(refreshRequest);
                     },
                 });
                 this.onChange();
diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index 3934194f2f1..c13a3cfd1db 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -206,9 +206,11 @@ export default {
         onValidation(validationInternal) {
             this.validationInternal = validationInternal;
         },
-        onChange(newData) {
+        onChange(newData, requestRefresh) {
             this.formData = newData;
-            this.onUpdate();
+            if (requestRefresh) {
+                this.onUpdate();
+            }
         },
         onUpdate() {
             updateToolFormData(this.formConfig.id, this.currentVersion, this.history_id, this.formData).then((data) => {
diff --git a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
index 14eb74f948f..56fcb9dcbc7 100644
--- a/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
+++ b/client/src/components/Workflow/Run/WorkflowRunToolStep.vue
@@ -121,15 +121,17 @@ export default {
                 }
             });
         },
-        onChange(data) {
-            getTool(this.model.id, this.model.version, data, this.historyId).then(
-                (formConfig) => {
-                    this.formConfig = formConfig;
-                },
-                (errorText) => {
-                    this.errorText = errorText;
-                }
-            );
+        onChange(data, refreshRequest) {
+            if (refreshRequest) {
+                getTool(this.model.id, this.model.version, data, this.historyId).then(
+                    (formConfig) => {
+                        this.formConfig = formConfig;
+                    },
+                    (errorText) => {
+                        this.errorText = errorText;
+                    }
+                );
+            }
             this.$emit("onChange", this.model.index, data);
         },
         onValidation(validation) {
diff --git a/client/src/mvc/form/form-view.js b/client/src/mvc/form/form-view.js
index 527db13bede..fa40d13b566 100644
--- a/client/src/mvc/form/form-view.js
+++ b/client/src/mvc/form/form-view.js
@@ -10,8 +10,6 @@ import FormData from "mvc/form/form-data";
 export default Backbone.View.extend({
     initialize: function (options) {
         this.model = new Backbone.Model({
-            initial_errors: false,
-            always_refresh: true,
             onchange: function () {},
         }).set(options);
         this.setElement(options.el || "<div/>");
@@ -95,12 +93,11 @@ export default Backbone.View.extend({
         var current_check = this.data.checksum();
         this.on("change", (input_id) => {
             var input = self.input_list[input_id];
-            if (!input || input.refresh_on_change || self.model.get("always_refresh")) {
-                var new_check = self.data.checksum();
-                if (new_check != current_check) {
-                    current_check = new_check;
-                    self.model.get("onchange")();
-                }
+            var refresh_request = !input || input.refresh_on_change;
+            var new_check = self.data.checksum();
+            if (new_check != current_check) {
+                current_check = new_check;
+                self.model.get("onchange")(refresh_request);
             }
         });
         this.on("reset", () => {

From 04f856a009255fa3664579a573831dd54f53500c Mon Sep 17 00:00:00 2001
From: melibleq <16723182+melibleq@users.noreply.github.com>
Date: Thu, 26 Aug 2021 10:31:37 -0300
Subject: [PATCH 414/452] Small fix in binary.py

The EDAM annotation wasn't working there because it had been added as "format", instead of "data".
---
 lib/galaxy/datatypes/binary.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 119a824b86b..5de44e55bf4 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -727,7 +727,7 @@ class BamQuerynameSorted(BamInputSorted):
 class CRAM(Binary):
     file_ext = "cram"
     edam_format = "format_3462"
-    edam_data = "format_0863"
+    edam_data = "data_0863"
 
     MetadataElement(name="cram_version", default=None, desc="CRAM Version", param=MetadataParameter, readonly=True, visible=False, optional=False, no_value=None)
     MetadataElement(name="cram_index", desc="CRAM Index File", param=metadata.FileParameter, file_ext="crai", readonly=True, no_value=None, visible=False, optional=True)

From 7ebec8a7b449f2bc3e9c69691025fd865cb6a092 Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Thu, 26 Aug 2021 08:34:52 -0500
Subject: [PATCH 415/452] Update string formats

---
 .../config/sample/datatypes_conf.xml.sample   |  10 +-
 lib/galaxy/datatypes/binary.py                | 153 ++++++++++++------
 2 files changed, 108 insertions(+), 55 deletions(-)

diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index e702ff930cb..10fcedf3767 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -272,6 +272,7 @@
     <datatype extension="mzml" type="galaxy.datatypes.proteomics:MzML" mimetype="application/xml" display_in_upload="true"/>
     <datatype extension="nmrml" type="galaxy.datatypes.proteomics:NmrML" mimetype="application/xml" display_in_upload="true" description="nmrML is an open mark-up language for NMR data." description_url="http://nmrml.org/schema/"/>
     <datatype extension="meryldb" type="galaxy.datatypes.binary:Meryldb" subclass="true" display_in_upload="true" description="MerylDB is a tar.gz archive containing 64 binaries + 64 indexes." />
+    <datatype extension="bref3" type="galaxy.datatypes.binary:Bref3" display_in_upload="true" description="Bref3 format is a binary format for storing phased, non-missing genotypes for a list of samples. More information in https://faculty.washington.edu/browning/beagle/bref3.14May18.pdf" />
     <datatype extension="mgf" type="galaxy.datatypes.proteomics:Mgf" display_in_upload="true"/>
     <datatype extension="wiff" type="galaxy.datatypes.proteomics:Wiff" display_in_upload="true"/>
     <datatype extension="mzxml" type="galaxy.datatypes.proteomics:MzXML" mimetype="application/xml" display_in_upload="true"/>
@@ -426,7 +427,7 @@
     </datatype>
     <!-- bcf_bgzip is just an alias for bcf for backward-compatibility -->
     <datatype extension="bcf_bgzip" type="galaxy.datatypes.binary:Bcf" subclass="true"/>
-    <datatype extension="bcf_uncompressed" type="galaxy.datatypes.binary:BcfUncompressed" mimetype="application/octet-stream">
+    <datatype extension="bcf_uncompressed" type="galaxy.datatypes.binary:BcfUncompressed" mimetype="application/octet-stream" display_in_upload="true">
       <converter file="bcf_uncompressed_to_bcf_converter.xml" target_datatype="bcf"/>
     </datatype>
     <datatype extension="velvet" type="galaxy.datatypes.assembly:Velvet" display_in_upload="true"/>
@@ -443,7 +444,7 @@
     <datatype extension="protobuf2" type="galaxy.datatypes.binary:Binary" subclass="true" description="Protocol Buffers (Protobuf) is data format for serializing structured data." display_in_upload="true"/>
     <datatype extension="protobuf3" type="galaxy.datatypes.binary:Binary" subclass="true" description="Protocol Buffers (Protobuf) is data format for serializing structured data." display_in_upload="true"/>
     <datatype extension="tabix" type="galaxy.datatypes.binary:Binary" subclass="true"/>
-    <datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="true"/>
+    <datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="true" display_in_upload="true"/>
     <datatype extension="vcf_bgzip" type="galaxy.datatypes.tabular:VcfGz" display_in_upload="true">
       <display file="igv/vcf.xml"/>
       <converter file="vcf_bgzip_to_tabix_converter.xml" target_datatype="tabix"/>
@@ -653,10 +654,10 @@
     <!-- Metagenomic Datatypes -->
     <datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="true" mimetype="application/json">
       <display file="biom/biom_simple.xml"/>
-      <converter file="biom1_to_biom2.xml" target_datatype="biom2"/>
+      <converter file="biom.xml" target_datatype="biom2"/>
     </datatype>
     <datatype extension="biom2" type="galaxy.datatypes.binary:Biom2" mimetype="application/octet-stream" display_in_upload="true">
-      <converter file="biom2_to_biom1.xml" target_datatype="biom1"/>
+      <converter file="biom.xml" target_datatype="biom1"/>
     </datatype>
     <datatype extension="msh" type="galaxy.datatypes.binary:MashSketch" display_in_upload="True" />
     <!-- Strand-specific Coordinate Count Datatype used by the Center for Eukaryotic Gene Regulation labs at Penn State -->
@@ -945,6 +946,7 @@
     <sniffer type="galaxy.datatypes.binary:Fast5ArchiveBz2"/>
     <sniffer type="galaxy.datatypes.binary:Fast5Archive"/>
     <sniffer type="galaxy.datatypes.binary:Meryldb" />
+    <sniffer type="galaxy.datatypes.binary:Bref3" />
     <sniffer type="galaxy.datatypes.binary:PostgresqlArchive"/>
     <sniffer type="galaxy.datatypes.binary:ICM"/>
     <sniffer type="galaxy.datatypes.binary:Idat"/>
diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 1faea327274..6c0d8ed1b12 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -92,7 +92,7 @@ class Ab1(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary ab1 sequence file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary ab1 sequence file ({nice_size(dataset.get_size())})"
 
 
 class Idat(Binary):
@@ -167,7 +167,7 @@ class Cel(Binary):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.blurb = "Cel version: %s" % dataset.metadata.version
+            dataset.blurb = f"Cel version: {dataset.metadata.version}"
             dataset.peek = get_file_peek(dataset.file_name)
         else:
             dataset.peek = 'file does not exist'
@@ -206,7 +206,7 @@ class CompressedArchive(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Compressed binary file (%s)" % (nice_size(dataset.get_size()))
+            return f"Compressed binary file ({nice_size(dataset.get_size())})"
 
 
 class Meryldb(CompressedArchive):
@@ -237,6 +237,33 @@ class Meryldb(CompressedArchive):
         return False
 
 
+class Bref3(Binary):
+    """Bref3 format is a binary format for storing phased, non-missing genotypes for a list of samples."""
+
+    file_ext = "bref3"
+
+    def __init__(self, **kwd):
+        super().__init__(**kwd)
+        self._magic = binascii.unhexlify("7a8874f400156272")
+
+    def sniff_prefix(self, sniff_prefix):
+        return sniff_prefix.startswith_bytes(self._magic)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Binary bref3 file"
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except Exception:
+            return f"Binary bref3 file ({nice_size(dataset.get_size())})"
+
+
 class DynamicCompressedArchive(CompressedArchive):
 
     def matches_any(self, target_datatypes):
@@ -284,7 +311,7 @@ class CompressedZipArchive(CompressedArchive):
         try:
             return dataset.peek
         except Exception:
-            return "Compressed zip file (%s)" % (nice_size(dataset.get_size()))
+            return f"Compressed zip file ({nice_size(dataset.get_size())})"
 
     def sniff(self, filename):
         with zipfile.ZipFile(filename) as zf:
@@ -376,14 +403,14 @@ class BamNative(CompressedArchive):
         try:
             return dataset.peek
         except Exception:
-            return "Binary bam alignments file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary bam alignments file ({nice_size(dataset.get_size())})"
 
     def to_archive(self, dataset, name=""):
         rel_paths = []
         file_paths = []
-        rel_paths.append("{}.{}".format(name or dataset.file_name, dataset.extension))
+        rel_paths.append(f"{name or dataset.file_name}.{dataset.extension}")
         file_paths.append(dataset.file_name)
-        rel_paths.append("{}.{}.bai".format(name or dataset.file_name, dataset.extension))
+        rel_paths.append(f"{name or dataset.file_name}.{dataset.extension}.bai")
         file_paths.append(dataset.metadata.bam_index.file_name)
         return zip(file_paths, rel_paths)
 
@@ -401,12 +428,12 @@ class BamNative(CompressedArchive):
             return
         tmp_dir = tempfile.mkdtemp()
         tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
-        sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix
+        sorted_file_name = f"{tmp_sorted_dataset_file_name_prefix}.bam"
         slots = os.environ.get('GALAXY_SLOTS', 1)
         sort_args = []
         if self.sort_flag:
             sort_args = [self.sort_flag]
-        sort_args.extend(["-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name])
+        sort_args.extend([f"-@{slots}", file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name])
         try:
             pysam.sort(*sort_args)
         except Exception:
@@ -448,7 +475,7 @@ class BamNative(CompressedArchive):
                         offset = -1
             except Exception as e:
                 offset = -1
-                ck_data = "Could not display BAM file, error was:\n%s" % e
+                ck_data = f"Could not display BAM file, error was:\n{e}"
         else:
             ck_data = ''
             offset = -1
@@ -709,7 +736,7 @@ class CRAM(Binary):
     def set_meta(self, dataset, overwrite=True, **kwd):
         major_version, minor_version = self.get_cram_version(dataset.file_name)
         if major_version != -1:
-            dataset.metadata.cram_version = str(major_version) + "." + str(minor_version)
+            dataset.metadata.cram_version = f"{str(major_version)}.{str(minor_version)}"
 
         if not dataset.metadata.cram_index:
             index_file = dataset.metadata.spec['cram_index'].param.new_file(dataset=dataset)
@@ -787,11 +814,11 @@ class Bcf(BaseBcf):
                                        '__dataset_%d_%s' % (dataset.id, os.path.basename(index_file.file_name)))
         os.symlink(dataset.file_name, dataset_symlink)
         try:
-            cmd = ['python', '-c', "import pysam.bcftools; pysam.bcftools.index('%s')" % (dataset_symlink)]
+            cmd = ['python', '-c', f"import pysam.bcftools; pysam.bcftools.index('{dataset_symlink}')"]
             subprocess.check_call(cmd)
-            shutil.move(dataset_symlink + '.csi', index_file.file_name)
+            shutil.move(f"{dataset_symlink}.csi", index_file.file_name)
         except Exception as e:
-            raise Exception('Error setting BCF metadata: %s' % util.unicodify(e))
+            raise Exception(f'Error setting BCF metadata: {util.unicodify(e)}')
         finally:
             # Remove temp file and symlink
             os.remove(dataset_symlink)
@@ -864,7 +891,7 @@ class H5(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary HDF5 file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary HDF5 file ({nice_size(dataset.get_size())})"
 
 
 class Loom(H5):
@@ -926,7 +953,7 @@ class Loom(H5):
         try:
             return dataset.peek
         except Exception:
-            return "Binary Loom file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary Loom file ({nice_size(dataset.get_size())})"
 
     def set_meta(self, dataset, overwrite=True, **kwd):
         super().set_meta(dataset, overwrite=overwrite, **kwd)
@@ -1161,7 +1188,7 @@ class Anndata(H5):
             peekstr += _makelayerstrings("uns", tmp.uns_count, tmp.uns_layers)
 
             dataset.peek = peekstr
-            dataset.blurb = "Anndata file (%s)" % nice_size(dataset.get_size())
+            dataset.blurb = f"Anndata file ({nice_size(dataset.get_size())})"
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'
@@ -1170,7 +1197,7 @@ class Anndata(H5):
         try:
             return dataset.peek
         except Exception:
-            return "Binary Anndata file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary Anndata file ({nice_size(dataset.get_size())})"
 
 
 @build_sniff_from_prefix
@@ -1188,7 +1215,7 @@ class GmxBinary(Binary):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "Binary GROMACS %s file" % (self.file_ext)
+            dataset.peek = f"Binary GROMACS {self.file_ext} file"
             dataset.blurb = nice_size(dataset.get_size())
         else:
             dataset.peek = 'file does not exist'
@@ -1198,7 +1225,7 @@ class GmxBinary(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary GROMACS {} trajectory file ({})".format(self.file_ext, nice_size(dataset.get_size()))
+            return f"Binary GROMACS {self.file_ext} trajectory file ({nice_size(dataset.get_size())})"
 
 
 class Trr(GmxBinary):
@@ -1331,7 +1358,7 @@ class Biom2(H5):
             try:
                 with h5py.File(dataset.file_name) as f:
                     for k, v in f.attrs.items():
-                        lines.append('{}:  {}'.format(k, util.unicodify(v)))
+                        lines.append(f'{k}:  {util.unicodify(v)}')
             except Exception as e:
                 log.warning('%s, set_peek Exception: %s', self, util.unicodify(e))
             dataset.peek = '\n'.join(lines)
@@ -1344,7 +1371,7 @@ class Biom2(H5):
         try:
             return dataset.peek
         except Exception:
-            return "Biom2 (HDF5) file (%s)" % (nice_size(dataset.get_size()))
+            return f"Biom2 (HDF5) file ({nice_size(dataset.get_size())})"
 
 
 class Cool(H5):
@@ -1397,7 +1424,7 @@ class Cool(H5):
         try:
             return dataset.peek
         except Exception:
-            return "Cool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
+            return f"Cool (HDF5) file ({nice_size(dataset.get_size())})."
 
 
 class MCool(H5):
@@ -1457,7 +1484,7 @@ class MCool(H5):
         try:
             return dataset.peek
         except Exception:
-            return "MCool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
+            return f"MCool (HDF5) file ({nice_size(dataset.get_size())})."
 
 
 class H5MLM(H5):
@@ -1546,6 +1573,8 @@ class HexrdMaterials(H5):
     edam_format = "format_3590"
 
     MetadataElement(name="materials", desc="materials", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None)
+    MetadataElement(name="SpaceGroupNumber", default={}, param=DictParameter, desc="SpaceGroupNumber", readonly=True, visible=True, no_value={})
+    MetadataElement(name="LatticeParameters", default={}, param=DictParameter, desc="LatticeParameters", readonly=True, visible=True, no_value={})
 
     def sniff(self, filename):
         if super().sniff(filename):
@@ -1561,13 +1590,29 @@ class HexrdMaterials(H5):
         try:
             with h5py.File(dataset.file_name, 'r') as mat_file:
                 dataset.metadata.materials = list(mat_file.keys())
+                sgn = dict()
+                lp = dict()
+                for m in mat_file.keys(): 
+                    if 'SpaceGroupNumber' in mat_file[m] and len(mat_file[m]['SpaceGroupNumber']) > 0:
+                        sgn[m] = mat_file[m]['SpaceGroupNumber'][0].item()
+                    if 'LatticeParameters' in mat_file[m]:
+                        lp[m] = mat_file[m]['LatticeParameters'][0:].tolist() 
+                dataset.metadata.SpaceGroupNumber = sgn
+                dataset.metadata.LatticeParameters = lp
         except Exception as e:
             log.warning('%s, set_meta Exception: %s', self, e)
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "Materials: %s" % (' '.join(dataset.metadata.materials))
-            dataset.blurb = dataset.peek
+            lines = ['Material SpaceGroup Lattice']
+            if dataset.metadata.materials:
+                for i, m in enumerate(dataset.metadata.materials):
+                    try:
+                        lines.append(f'{m} {dataset.metadata.SpaceGroupNumber[m]} {dataset.metadata.LatticeParameters[m]}')
+                    except Exception:
+                        continue
+            dataset.peek = '\n'.join(lines)
+            dataset.blurb = f"Materials: {' '.join(dataset.metadata.materials)}"
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'
@@ -1591,7 +1636,7 @@ class Scf(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary scf sequence file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary scf sequence file ({nice_size(dataset.get_size())})"
 
 
 @build_sniff_from_prefix
@@ -1618,7 +1663,7 @@ class Sff(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary sff file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary sff file ({nice_size(dataset.get_size())})"
 
 
 @build_sniff_from_prefix
@@ -1644,7 +1689,7 @@ class BigWig(Binary):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "Binary UCSC %s file" % self._name
+            dataset.peek = f"Binary UCSC {self._name} file"
             dataset.blurb = nice_size(dataset.get_size())
         else:
             dataset.peek = 'file does not exist'
@@ -1654,7 +1699,7 @@ class BigWig(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary UCSC {} file ({})".format(self._name, nice_size(dataset.get_size()))
+            return f"Binary UCSC {self._name} file ({nice_size(dataset.get_size())})"
 
 
 class BigBed(BigWig):
@@ -1692,7 +1737,7 @@ class TwoBit(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary TwoBit format nucleotide file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary TwoBit format nucleotide file ({nice_size(dataset.get_size())})"
 
 
 @dataproviders.decorators.has_dataproviders
@@ -1719,7 +1764,7 @@ class SQlite(Binary):
             for table, _ in rslt:
                 tables.append(table)
                 try:
-                    col_query = 'SELECT * FROM %s LIMIT 0' % table
+                    col_query = f'SELECT * FROM {table} LIMIT 0'
                     cur = conn.cursor().execute(col_query)
                     cols = [col[0] for col in cur.description]
                     columns[table] = cols
@@ -1727,7 +1772,7 @@ class SQlite(Binary):
                     log.warning('%s, set_meta Exception: %s', self, exc)
             for table in tables:
                 try:
-                    row_query = "SELECT count(*) FROM %s" % table
+                    row_query = f"SELECT count(*) FROM {table}"
                     rowcounts[table] = c.execute(row_query).fetchone()[0]
                 except Exception as exc:
                     log.warning('%s, set_meta Exception: %s', self, exc)
@@ -1771,7 +1816,7 @@ class SQlite(Binary):
             if dataset.metadata.tables:
                 for table in dataset.metadata.tables:
                     try:
-                        lines.append('{} [{}]'.format(table, dataset.metadata.table_row_count[table]))
+                        lines.append(f'{table} [{dataset.metadata.table_row_count[table]}]')
                     except Exception:
                         continue
             dataset.peek = '\n'.join(lines)
@@ -1784,7 +1829,7 @@ class SQlite(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "SQLite Database (%s)" % (nice_size(dataset.get_size()))
+            return f"SQLite Database ({nice_size(dataset.get_size())})"
 
     @dataproviders.decorators.dataprovider_factory('sqlite', dataproviders.dataset.SQliteDataProvider.settings)
     def sqlite_dataprovider(self, dataset, **settings):
@@ -2071,7 +2116,7 @@ class DlibSQlite(SQlite):
             c = conn.cursor()
             tables_query = "SELECT Value FROM metadata WHERE Key = 'version'"
             version = c.execute(tables_query).fetchall()[0]
-            dataset.metadata.dlib_version = '%s' % (version)
+            dataset.metadata.dlib_version = f'{version}'
         except Exception as e:
             log.warning('%s, set_meta Exception: %s', self, e)
 
@@ -2107,7 +2152,7 @@ class ElibSQlite(SQlite):
             c = conn.cursor()
             tables_query = "SELECT Value FROM metadata WHERE Key = 'version'"
             version = c.execute(tables_query).fetchall()[0]
-            dataset.metadata.dlib_version = '%s' % (version)
+            dataset.metadata.dlib_version = f'{version}'
         except Exception as e:
             log.warning('%s, set_meta Exception: %s', self, e)
 
@@ -2163,7 +2208,7 @@ class IdpDB(SQlite):
         try:
             return dataset.peek
         except Exception:
-            return "IDPickerDB SQLite file (%s)" % (nice_size(dataset.get_size()))
+            return f"IDPickerDB SQLite file ({nice_size(dataset.get_size())})"
 
 
 class GAFASQLite(SQlite):
@@ -2290,7 +2335,7 @@ class ExcelXls(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Microsoft Excel XLS file (%s)" % (data.nice_size(dataset.get_size()))
+            return f"Microsoft Excel XLS file ({data.nice_size(dataset.get_size())})"
 
 
 @build_sniff_from_prefix
@@ -2316,7 +2361,7 @@ class Sra(Binary):
         try:
             return dataset.peek
         except Exception:
-            return 'Binary sra file (%s)' % (nice_size(dataset.get_size()))
+            return f'Binary sra file ({nice_size(dataset.get_size())})'
 
 
 class RData(Binary):
@@ -2538,7 +2583,7 @@ class PostgresqlArchive(CompressedArchive):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "PostgreSQL Archive (%s)" % (nice_size(dataset.get_size()))
+            dataset.peek = f"PostgreSQL Archive ({nice_size(dataset.get_size())})"
             dataset.blurb = "PostgreSQL version %s" % (dataset.metadata.version or 'unknown')
         else:
             dataset.peek = 'file does not exist'
@@ -2548,7 +2593,7 @@ class PostgresqlArchive(CompressedArchive):
         try:
             return dataset.peek
         except Exception:
-            return "PostgreSQL Archive (%s)" % (nice_size(dataset.get_size()))
+            return f"PostgreSQL Archive ({nice_size(dataset.get_size())})"
 
 
 class Fast5Archive(CompressedArchive):
@@ -2592,7 +2637,7 @@ class Fast5Archive(CompressedArchive):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "FAST5 Archive (%s)" % (nice_size(dataset.get_size()))
+            dataset.peek = f"FAST5 Archive ({nice_size(dataset.get_size())})"
             dataset.blurb = "%s sequences" % (dataset.metadata.fast5_count or 'unknown')
         else:
             dataset.peek = 'file does not exist'
@@ -2602,7 +2647,7 @@ class Fast5Archive(CompressedArchive):
         try:
             return dataset.peek
         except Exception:
-            return "FAST5 Archive (%s)" % (nice_size(dataset.get_size()))
+            return f"FAST5 Archive ({nice_size(dataset.get_size())})"
 
 
 class Fast5ArchiveGz(Fast5Archive):
@@ -2718,7 +2763,7 @@ class NetCDF(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary netCDF file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary netCDF file ({nice_size(dataset.get_size())})"
 
     def sniff_prefix(self, sniff_prefix):
         return sniff_prefix.startswith_bytes(b'CDF')
@@ -2773,7 +2818,7 @@ class Dcd(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary CHARMM/NAMD dcd file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary CHARMM/NAMD dcd file ({nice_size(dataset.get_size())})"
 
 
 class Vel(Binary):
@@ -2824,7 +2869,7 @@ class Vel(Binary):
         try:
             return dataset.peek
         except Exception:
-            return "Binary CHARMM velocity file (%s)" % (nice_size(dataset.get_size()))
+            return f"Binary CHARMM velocity file ({nice_size(dataset.get_size())})"
 
 
 @build_sniff_from_prefix
@@ -2979,7 +3024,7 @@ class BafTar(CompressedArchive):
         try:
             return dataset.peek
         except Exception:
-            return "{} ({})".format(self.get_type(), nice_size(dataset.get_size()))
+            return f"{self.get_type()} ({nice_size(dataset.get_size())})"
 
 
 class YepTar(BafTar):
@@ -3141,6 +3186,7 @@ class Npz(CompressedArchive):
     # edam_format = "format_4003"
 
     MetadataElement(name="nfiles", default=0, desc="nfiles", readonly=True, visible=True, no_value=0)
+    MetadataElement(name="files", default=[], desc="files", readonly=True, visible=True, no_value=[])
 
     def __init__(self, **kwd):
         super().__init__(**kwd)
@@ -3160,12 +3206,13 @@ class Npz(CompressedArchive):
         try:
             with np.load(dataset.file_name) as npz:
                 dataset.metadata.nfiles = len(npz.files)
+                dataset.metadata.files = npz.files
         except Exception as e:
             log.warning('%s, set_meta Exception: %s', self, e)
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "Binary Numpy npz file"
+            dataset.peek = f"Binary Numpy npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})"
             dataset.blurb = nice_size(dataset.get_size())
         else:
             dataset.peek = 'file does not exist'
@@ -3226,7 +3273,9 @@ class HexrdImagesNpz(Npz):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "Panel: %s Frames: %s Shape: %s" % (dataset.metadata.panel_id, dataset.metadata.nframes, dataset.metadata.shape)
+            lines = [f"Binary Hexrd Image npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})",
+                     f"Panel: {dataset.metadata.panel_id} Frames: {dataset.metadata.nframes} Shape: {dataset.metadata.shape}"]
+            dataset.peek = '\n'.join(lines)
             dataset.blurb = nice_size(dataset.get_size())
         else:
             dataset.peek = 'file does not exist'
@@ -3281,7 +3330,9 @@ class HexrdEtaOmeNpz(Npz):
 
     def set_peek(self, dataset, is_multi_byte=False):
         if not dataset.dataset.purged:
-            dataset.peek = "Eta-Ome HKLs: %s Frames: %s" % (dataset.metadata.HKLsi, dataset.metadata.nframes)
+            lines = [f"Binary Hexrd Eta-Ome npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})",
+                     f"Eta-Ome HKLs: {dataset.metadata.HKLs} Frames: {dataset.metadata.nframes}"]
+            dataset.peek = '\n'.join(lines)
             dataset.blurb = nice_size(dataset.get_size())
         else:
             dataset.peek = 'file does not exist'

From 0610e2cee79f240149d369a7732665535c0b7693 Mon Sep 17 00:00:00 2001
From: Jim Johnson <jj@umn.edu>
Date: Thu, 26 Aug 2021 09:37:36 -0500
Subject: [PATCH 416/452] Update binary.py

Fix lint errors
---
 lib/galaxy/datatypes/binary.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 6c0d8ed1b12..5d0fef9738e 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -1592,11 +1592,11 @@ class HexrdMaterials(H5):
                 dataset.metadata.materials = list(mat_file.keys())
                 sgn = dict()
                 lp = dict()
-                for m in mat_file.keys(): 
+                for m in mat_file.keys():
                     if 'SpaceGroupNumber' in mat_file[m] and len(mat_file[m]['SpaceGroupNumber']) > 0:
                         sgn[m] = mat_file[m]['SpaceGroupNumber'][0].item()
                     if 'LatticeParameters' in mat_file[m]:
-                        lp[m] = mat_file[m]['LatticeParameters'][0:].tolist() 
+                        lp[m] = mat_file[m]['LatticeParameters'][0:].tolist()
                 dataset.metadata.SpaceGroupNumber = sgn
                 dataset.metadata.LatticeParameters = lp
         except Exception as e:
@@ -1606,7 +1606,7 @@ class HexrdMaterials(H5):
         if not dataset.dataset.purged:
             lines = ['Material SpaceGroup Lattice']
             if dataset.metadata.materials:
-                for i, m in enumerate(dataset.metadata.materials):
+                for m in dataset.metadata.materials:
                     try:
                         lines.append(f'{m} {dataset.metadata.SpaceGroupNumber[m]} {dataset.metadata.LatticeParameters[m]}')
                     except Exception:

From 569bffe9761c24c2c81070bb1852f8621801456a Mon Sep 17 00:00:00 2001
From: Mason <nerdinacan@gmail.com>
Date: Thu, 26 Aug 2021 07:49:05 -0700
Subject: [PATCH 417/452] Update
 lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py

Co-authored-by: Marius van den Beek <m.vandenbeek@gmail.com>
---
 .../model/migrate/versions/0177_update_job_state_summary.py      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py b/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
index 352c3bcb669..6391f94fc87 100644
--- a/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
+++ b/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
@@ -16,7 +16,6 @@ def upgrade(migrate_engine):
     downgrade(migrate_engine)
     view = HistoryDatasetCollectionJobStateSummary
     create_view = CreateView(view.name, view.__view__)
-    # print(str(create_view.compile(migrate_engine)))
     migrate_engine.execute(create_view)
 
 

From c1d4604151e34a0b4d9e3ef11c3f74b4abfe1236 Mon Sep 17 00:00:00 2001
From: Mason <nerdinacan@gmail.com>
Date: Thu, 26 Aug 2021 07:49:12 -0700
Subject: [PATCH 418/452] Update
 lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py

Co-authored-by: Marius van den Beek <m.vandenbeek@gmail.com>
---
 .../model/migrate/versions/0177_update_job_state_summary.py      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py b/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
index 6391f94fc87..f8e279f833e 100644
--- a/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
+++ b/lib/galaxy/model/migrate/versions/0177_update_job_state_summary.py
@@ -21,5 +21,4 @@ def upgrade(migrate_engine):
 
 def downgrade(migrate_engine):
     drop_view = DropView(HistoryDatasetCollectionJobStateSummary.name)
-    # print(str(drop_view.compile(migrate_engine)))
     migrate_engine.execute(drop_view)

From 563b3af7642c4802170826671eb65f87c1463f62 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 26 Aug 2021 11:28:43 -0400
Subject: [PATCH 419/452] Use consistent name for refresh flag in tool form

---
 client/src/components/Tool/ToolForm.vue | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue
index c13a3cfd1db..7b52422902d 100644
--- a/client/src/components/Tool/ToolForm.vue
+++ b/client/src/components/Tool/ToolForm.vue
@@ -206,9 +206,9 @@ export default {
         onValidation(validationInternal) {
             this.validationInternal = validationInternal;
         },
-        onChange(newData, requestRefresh) {
+        onChange(newData, refreshRequest) {
             this.formData = newData;
-            if (requestRefresh) {
+            if (refreshRequest) {
                 this.onUpdate();
             }
         },

From c965dd95a790a7afa3e906d3d46d603db29135d6 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Thu, 26 Aug 2021 09:56:21 -0700
Subject: [PATCH 420/452] added more ISO8601 tests and removed manual parsing
 procedure

---
 .../webapps/galaxy/api/history_contents.py    | 27 ++++++++++---------
 lib/galaxy_test/api/test_history_contents.py  | 16 ++++++-----
 2 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index ad952fada65..f7aea776a13 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -787,17 +787,15 @@ class HistoriesContentsService(ServiceBase):
         # while polling, check to see if the history has changed
         # if it hasn't then we can short-circuit the poll request
         if since:
-            # sqlalchemy DateTime columns are not timezone aware, so parse `since` into timezone-aware
-            # datetime and then convert to naive datetime object representing UTC,
-            # assuming history.update_time represents UTC time.
-            # this is the right format, but changes the date itself resulting in logical errors
+            # sqlalchemy DateTime columns are not timezone aware, but `since` may be a timezone aware
+            # DateTime object if a timezone offset is provided (https://github.com/samuelcolvin/pydantic/blob/5ccbdcb5904f35834300b01432a665c75dc02296/pydantic/datetime_parse.py#L179).
+            # To avoid having to manually process every IS08601 date format, simply convert
+            # history.update_time to UTC/timezone aware before performing the inequality
 
-            # Although this creates a date with the right format for SQLAlchemy, it changes the date
-            # data resulting in logical errors during testing. I fed back the history.update_time
-            # into this conversion and it fails the inequality
-            # since_date = since.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+            since = since if since.tzinfo is not None else since.replace(tzinfo=datetime.timezone.utc)
+            history_update_time_tz = history.update_time.replace(tzinfo=datetime.timezone.utc)
 
-            if history.update_time <= since:
+            if history_update_time_tz <= since:
                 trans.response.status = 204
                 return
 
@@ -1775,10 +1773,13 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems,
         * GET /api/histories/{history_id}/contents/near/{hid}/{limit}
         """
         serialization_params = parse_serialization_params(default_view='betawebclient', **kwd)
-        since = kwd.pop('since', None)
-        if since:
-            since = since.replace("Z", "")
-            since = dateutil.parser.isoparse(since)
+
+        since_str = kwd.pop('since', None)
+        if since_str:
+            since = dateutil.parser.isoparse(since_str)
+        else:
+            since = None
+
         filter_params = self._parse_rest_params(kwd)
         hid = int(hid)
         limit = int(limit)
diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 3a29ed3ada8..72ffa9f899d 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -639,16 +639,18 @@ class HistoryContentsApiTestCase(ApiTestCase):
             # this is the standard date format that javascript will emit using .toISOString(), it
             # should be the expected date format for any modern api
             # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString
+
+            # checking to make sure that the same exact history.update_time returns a "not changed"
+            # result after date parsing
             valid_iso8601_date = original_history_stamp + 'Z'
             history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={valid_iso8601_date}")
-            status = history_contents.status_code
-            assert status == 204
+            assert history_contents.status_code == 204
 
-            # Javascript has less date precision than python, here's a common sample
-            javascript_iso_date = '2021-08-25T21:46:49.091Z'
-            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={javascript_iso_date}")
-            status = history_contents.status_code
-            assert status == 204 or status == 200
+            # test parsing for other standard is08601 formats
+            sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z']
+            for date_str in sample_formats:
+                history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={date_str}")
+                assert history_contents.status_code != 400
 
     @skip_without_tool('cat_data_and_sleep')
     def test_history_contents_near_with_update_time_implicit_collection(self):

From 5ee9343c673cde184478af6529bf3c7d823af013 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Thu, 26 Aug 2021 10:32:42 -0700
Subject: [PATCH 421/452] addd url encoding for ISO8601 date formats that
 javascript will never emit for testing purposes

---
 lib/galaxy/webapps/galaxy/api/history_contents.py |  2 +-
 lib/galaxy_test/api/test_history_contents.py      | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index f7aea776a13..1c20c83facb 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -792,7 +792,7 @@ class HistoriesContentsService(ServiceBase):
             # To avoid having to manually process every IS08601 date format, simply convert
             # history.update_time to UTC/timezone aware before performing the inequality
 
-            since = since if since.tzinfo is not None else since.replace(tzinfo=datetime.timezone.utc)
+            since = since.replace(tzinfo=datetime.timezone.utc) if since.tzinfo is None else since
             history_update_time_tz = history.update_time.replace(tzinfo=datetime.timezone.utc)
 
             if history_update_time_tz <= since:
diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 72ffa9f899d..2f2e265adea 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -1,7 +1,7 @@
 import json
 import time
+import urllib
 from datetime import datetime
-
 from requests import delete, put
 
 from galaxy_test.base.populators import (
@@ -643,13 +643,15 @@ class HistoryContentsApiTestCase(ApiTestCase):
             # checking to make sure that the same exact history.update_time returns a "not changed"
             # result after date parsing
             valid_iso8601_date = original_history_stamp + 'Z'
-            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={valid_iso8601_date}")
+            encoded_valid_date = urllib.parse.quote_plus(valid_iso8601_date)
+            history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_valid_date}")
             assert history_contents.status_code == 204
 
             # test parsing for other standard is08601 formats
-            sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z']
+            sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z', '2002-10-10T12:00:00−05:00']
             for date_str in sample_formats:
-                history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={date_str}")
+                encoded_date = urllib.parse.quote_plus(date_str) # handles pluses, minuses
+                history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_date}")
                 assert history_contents.status_code != 400
 
     @skip_without_tool('cat_data_and_sleep')

From b424ff2c221c4341e2ac67967f857fde9d37ded3 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Thu, 26 Aug 2021 11:07:21 -0700
Subject: [PATCH 422/452] reversed date comparison strategy to use naive dates
 in comparison

---
 lib/galaxy/webapps/galaxy/api/history_contents.py | 10 +++-------
 lib/galaxy_test/api/test_history_contents.py      |  3 ++-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 1c20c83facb..5f6c37c9580 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -789,13 +789,9 @@ class HistoriesContentsService(ServiceBase):
         if since:
             # sqlalchemy DateTime columns are not timezone aware, but `since` may be a timezone aware
             # DateTime object if a timezone offset is provided (https://github.com/samuelcolvin/pydantic/blob/5ccbdcb5904f35834300b01432a665c75dc02296/pydantic/datetime_parse.py#L179).
-            # To avoid having to manually process every IS08601 date format, simply convert
-            # history.update_time to UTC/timezone aware before performing the inequality
-
-            since = since.replace(tzinfo=datetime.timezone.utc) if since.tzinfo is None else since
-            history_update_time_tz = history.update_time.replace(tzinfo=datetime.timezone.utc)
-
-            if history_update_time_tz <= since:
+            # If a timezone is provided (since.tzinfo is not None) we convert to UTC and remove tzinfo so that comparison with history.update_time is correct.
+            since = since if since.tzinfo is None else since.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+            if history.update_time <= since:
                 trans.response.status = 204
                 return
 
diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 2f2e265adea..2e0f8ab6d9c 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -2,6 +2,7 @@ import json
 import time
 import urllib
 from datetime import datetime
+
 from requests import delete, put
 
 from galaxy_test.base.populators import (
@@ -650,7 +651,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
             # test parsing for other standard is08601 formats
             sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z', '2002-10-10T12:00:00−05:00']
             for date_str in sample_formats:
-                encoded_date = urllib.parse.quote_plus(date_str) # handles pluses, minuses
+                encoded_date = urllib.parse.quote_plus(date_str)  # handles pluses, minuses
                 history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_date}")
                 assert history_contents.status_code != 400
 

From 3c7b07ae58a32cb34112e9634c446f2dc4224b56 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Thu, 26 Aug 2021 11:37:00 -0700
Subject: [PATCH 423/452] removed non-ascii character from sample data in test

---
 lib/galaxy_test/api/test_history_contents.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 2e0f8ab6d9c..c6a48ab9111 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -649,11 +649,11 @@ class HistoryContentsApiTestCase(ApiTestCase):
             assert history_contents.status_code == 204
 
             # test parsing for other standard is08601 formats
-            sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z', '2002-10-10T12:00:00−05:00']
+            sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z', '2002-10-10T12:00:00-05:00']
             for date_str in sample_formats:
                 encoded_date = urllib.parse.quote_plus(date_str)  # handles pluses, minuses
                 history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_date}")
-                assert history_contents.status_code != 400
+                self._assert_status_code_is_ok(history_contents)
 
     @skip_without_tool('cat_data_and_sleep')
     def test_history_contents_near_with_update_time_implicit_collection(self):

From 8bb21e5921c1ab5f80877be8336b81b057a41724 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 27 Aug 2021 15:28:44 -0400
Subject: [PATCH 424/452] Display Tabular datasets as sortable tables with
 correct metadata headers in pages

---
 .../Elements/HistoryDatasetDisplay.vue        | 96 +++++++++++++------
 .../providers/SingleQueryProvider.js          |  4 +
 lib/galaxy/managers/markdown_util.py          |  5 +-
 3 files changed, 75 insertions(+), 30 deletions(-)

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
index 5cc4e14bbb7..cf2836efef4 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
@@ -33,14 +33,35 @@
                 </span>
             </b-card-header>
             <b-card-body>
-                <LoadingSpan v-if="loading" message="Loading Dataset" />
-                <div v-else class="embedded-dataset content-height">
-                    <pre v-if="itemContent.item_data">
-                        <code class="text-normalwrap">{{ itemContent.item_data }}</code>
-                    </pre>
-                    <div v-else>No content found.</div>
-                    <b-link v-if="itemContent.truncated" :href="itemContent.item_url"> Show More... </b-link>
-                </div>
+                <UrlDataProvider :url="itemUrl" v-slot="{ result: itemContent, loading, error }">
+                    <LoadingSpan v-if="loading" message="Loading Dataset" />
+                    <div v-else-if="error">{{ error }}</div>
+                    <div v-else class="embedded-dataset content-height">
+                        <div v-if="itemContent.item_data">
+                            <pre v-if="notTabular">
+                                <code class="text-normalwrap">{{ itemContent.item_data }}</code>
+                            </pre>
+                            <div v-else>
+                                <UrlDataProvider
+                                    :url="metaUrl"
+                                    v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
+                                >
+                                    <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
+                                    <div v-else-if="metaError">{{ metaError }}</div>
+                                    <b-table
+                                        v-else
+                                        striped
+                                        hover
+                                        :fields="getFields(metaData)"
+                                        :items="getItems(itemContent.item_data, metaData)"
+                                    />
+                                </UrlDataProvider>
+                            </div>
+                        </div>
+                        <div v-else>No content found.</div>
+                        <b-link v-if="itemContent.truncated" :href="itemContent.item_url"> Show More... </b-link>
+                    </div>
+                </UrlDataProvider>
             </b-card-body>
         </b-card>
     </div>
@@ -49,10 +70,12 @@
 <script>
 import { getAppRoot } from "onload/loadConfig";
 import LoadingSpan from "components/LoadingSpan";
-import axios from "axios";
+import { UrlDataProvider } from "components/providers/UrlDataProvider";
+
 export default {
     components: {
         LoadingSpan,
+        UrlDataProvider,
     },
     props: {
         args: {
@@ -68,19 +91,11 @@ export default {
             default: false,
         },
     },
-    data() {
-        return {
-            itemContent: null,
-            loading: true,
-        };
-    },
-    created() {
-        this.getContent().then((data) => {
-            this.itemContent = data;
-            this.loading = false;
-        });
-    },
     computed: {
+        notTabular() {
+            const dataset = this.datasets[this.args.history_dataset_id];
+            return dataset && dataset.ext != "tabular";
+        },
         datasetName() {
             const dataset = this.datasets[this.args.history_dataset_id];
             return dataset && dataset.name;
@@ -92,17 +107,42 @@ export default {
             return `${getAppRoot()}dataset/imp?dataset_id=${this.args.history_dataset_id}`;
         },
         itemUrl() {
-            return `${getAppRoot()}api/datasets/${this.args.history_dataset_id}/get_content_as_text`;
+            return `api/datasets/${this.args.history_dataset_id}/get_content_as_text`;
+        },
+        metaUrl() {
+            return `api/datasets/${this.args.history_dataset_id}`;
         },
     },
     methods: {
-        async getContent() {
-            try {
-                const response = await axios.get(this.itemUrl);
-                return response.data;
-            } catch (e) {
-                return `Failed to retrieve content. ${e}`;
+        getFields(metaData) {
+            const fields = [];
+            const columnNames = metaData.metadata_column_names || [];
+            const columnCount = metaData.metadata_columns;
+            for (let i = 0; i < columnCount; i++) {
+                fields.push({
+                    key: `${i}`,
+                    label: columnNames[i] || i,
+                    sortable: true,
+                });
             }
+            return fields;
+        },
+        getItems(textData, metaData) {
+            const tableData = [];
+            const delimiter = metaData.metadata_delimiter || "\t";
+            const comments = metaData.metadata_comment_lines || 0;
+            const lines = textData.split("\n");
+            lines.forEach((line, i) => {
+                if (i >= comments) {
+                    const tabs = line.split(delimiter);
+                    const rowData = {};
+                    tabs.forEach((cellData, j) => {
+                        rowData[j] = cellData;
+                    });
+                    tableData.push(rowData);
+                }
+            });
+            return tableData;
         },
     },
 };
diff --git a/client/src/components/providers/SingleQueryProvider.js b/client/src/components/providers/SingleQueryProvider.js
index bee613787e0..76ec0cbb41d 100644
--- a/client/src/components/providers/SingleQueryProvider.js
+++ b/client/src/components/providers/SingleQueryProvider.js
@@ -21,6 +21,7 @@ export const SingleQueryProvider = (lookup) => {
         data() {
             return {
                 result: undefined,
+                error: undefined,
             };
         },
         computed: {
@@ -47,6 +48,8 @@ export const SingleQueryProvider = (lookup) => {
                     this.result = result;
                 },
                 (err) => {
+                    this.result = {};
+                    this.error = err;
                     this.$emit("error", err);
                 }
             );
@@ -55,6 +58,7 @@ export const SingleQueryProvider = (lookup) => {
             return this.$scopedSlots.default({
                 loading: this.loading,
                 result: this.result,
+                error: this.error,
             });
         },
     };
diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py
index 806281b5616..ae7e6927b80 100644
--- a/lib/galaxy/managers/markdown_util.py
+++ b/lib/galaxy/managers/markdown_util.py
@@ -270,10 +270,11 @@ class ReadyForExportMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHand
         self.ensure_rendering_data_for("history_datasets", obj)[key] = val or default_val
 
     def handle_dataset_display(self, line, hda):
-        self.extend_history_dataset_rendering_data(hda, "name", hda.name, "")
+        self.handle_dataset_name(line, hda)
+        self.handle_dataset_type(line, hda)
 
     def handle_dataset_embedded(self, line, hda):
-        self.extend_history_dataset_rendering_data(hda, "name", hda.name, "")
+        self.handle_dataset_name(line, hda)
 
     def handle_dataset_peek(self, line, hda):
         self.extend_history_dataset_rendering_data(hda, "peek", hda.peek, "*No Dataset Peek Available*")

From cab184eb95f904d8621cb63f0dbe9679ea8e3a92 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 27 Aug 2021 16:28:18 -0400
Subject: [PATCH 425/452] Show PDF and HTML content as embedded frames in
 pages, increase maximum content size to 20rem

---
 .../Elements/HistoryDatasetDisplay.vue        | 25 +++++++++++++------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
index cf2836efef4..147a27f4c78 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
@@ -37,11 +37,14 @@
                     <LoadingSpan v-if="loading" message="Loading Dataset" />
                     <div v-else-if="error">{{ error }}</div>
                     <div v-else class="embedded-dataset content-height">
-                        <div v-if="itemContent.item_data">
-                            <pre v-if="notTabular">
-                                <code class="text-normalwrap">{{ itemContent.item_data }}</code>
-                            </pre>
-                            <div v-else>
+                        <b-embed
+                            v-if="['pdf', 'html'].includes(datasetType)"
+                            type="iframe"
+                            aspect="16by9"
+                            :src="displayUrl"
+                        />
+                        <div v-else-if="itemContent.item_data">
+                            <div v-if="datasetType == 'tabular'">
                                 <UrlDataProvider
                                     :url="metaUrl"
                                     v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
@@ -57,6 +60,9 @@
                                     />
                                 </UrlDataProvider>
                             </div>
+                            <pre v-else>
+                                <code class="text-normalwrap">{{ itemContent.item_data }}</code>
+                            </pre>
                         </div>
                         <div v-else>No content found.</div>
                         <b-link v-if="itemContent.truncated" :href="itemContent.item_url"> Show More... </b-link>
@@ -92,9 +98,9 @@ export default {
         },
     },
     computed: {
-        notTabular() {
+        datasetType() {
             const dataset = this.datasets[this.args.history_dataset_id];
-            return dataset && dataset.ext != "tabular";
+            return dataset.ext;
         },
         datasetName() {
             const dataset = this.datasets[this.args.history_dataset_id];
@@ -103,6 +109,9 @@ export default {
         downloadUrl() {
             return `${getAppRoot()}dataset/display?dataset_id=${this.args.history_dataset_id}`;
         },
+        displayUrl() {
+            return `${getAppRoot()}datasets/${this.args.history_dataset_id}/display`;
+        },
         importUrl() {
             return `${getAppRoot()}dataset/imp?dataset_id=${this.args.history_dataset_id}`;
         },
@@ -149,6 +158,6 @@ export default {
 </script>
 <style scoped>
 .content-height {
-    max-height: 15rem;
+    max-height: 20rem;
 }
 </style>

From df7e40c550afb0647d2274f68fcf1eb817cc68bd Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Mon, 30 Aug 2021 10:19:32 +0530
Subject: [PATCH 426/452] Added missing triggers package to galaxy data

---
 packages/data/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/data/setup.py b/packages/data/setup.py
index 857666b8773..b0037cff0c1 100644
--- a/packages/data/setup.py
+++ b/packages/data/setup.py
@@ -40,6 +40,7 @@ PACKAGES = [
     'galaxy.model.dataset_collections',
     'galaxy.model.dataset_collections.types',
     'galaxy.model.migrate',
+    'galaxy.model.migrate.triggers',
     'galaxy.model.migrate.versions',
     'galaxy.model.orm',
     'galaxy.model.store',

From 938d807afffa920a7d8525d53c8f09c7dc157a9a Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Mon, 30 Aug 2021 00:13:59 -0700
Subject: [PATCH 427/452] selenium test updates

---
 .../src/components/History/ContentFilters.vue |  10 +-
 .../ContentItem/Dataset/DatasetMenu.vue       |   5 +-
 .../History/ContentItem/Dataset/DatasetUI.vue |  38 ++--
 .../ContentItem/Dataset/Summary/index.vue     |   2 +-
 .../ContentItem/DatasetCollection/DscUI.vue   |  13 +-
 .../History/ContentItem/Subcollection.vue     |  33 +--
 .../History/ContentItem/Subdataset.vue        |   1 -
 .../components/History/ContentOperations.vue  |  40 +++-
 client/src/components/History/ContentTags.vue |  54 -----
 .../History/CurrentCollection/Details.vue     |  36 +++-
 .../History/CurrentCollection/Panel.vue       |   2 +
 .../History/CurrentCollection/TopNav.vue      |  20 +-
 client/src/components/History/EditorMenu.vue  |   6 +-
 .../src/components/History/HistoriesMenu.vue  |   2 +-
 client/src/components/History/History.vue     |   2 +
 .../src/components/History/HistoryDetails.vue |  27 ++-
 client/src/components/History/HistoryMenu.vue |  10 +-
 client/src/components/History/StatusIcon.vue  |   2 +-
 .../operators/requestWithUpdateTime.js        |   2 +-
 .../History/model/DatasetCollection.js        |  10 +-
 .../History/model/JobStateSummary.js          |  59 ++---
 .../src/components/History/model/ScrollPos.js |   6 +-
 .../src/components/History/model/queries.js   |   2 +-
 client/src/components/History/model/states.js |   2 +
 .../ContentProvider/ContentProvider.js        |   6 +-
 .../HistoryContentProvider.test.js            |   2 +-
 .../HistoryContentProvider/contentPayload.js  |  24 +--
 .../components/History/providers/setCache.js  |   2 +-
 client/src/components/Nametags/Nametag.vue    |   2 +-
 .../src/components/RuleCollectionBuilder.vue  |  31 +--
 lib/galaxy/selenium/navigates_galaxy.py       | 156 ++++++++++----
 lib/galaxy/selenium/navigation.yml            |  60 +++++-
 .../selenium/test_collection_builders.py      | 180 +++++++++++-----
 .../selenium/test_history_copy_elements.py    |   5 +-
 .../selenium/test_history_export.py           |  31 +--
 .../selenium/test_history_panel.py            | 177 +++++++++------
 .../test_history_panel_collections.py         | 204 +++++++++++++-----
 .../selenium/test_history_panel_pagination.py |   5 +
 lib/galaxy_test/selenium/test_tool_form.py    |   1 +
 39 files changed, 807 insertions(+), 463 deletions(-)
 delete mode 100644 client/src/components/History/ContentTags.vue

diff --git a/client/src/components/History/ContentFilters.vue b/client/src/components/History/ContentFilters.vue
index da1663d41b8..a1fa851561e 100644
--- a/client/src/components/History/ContentFilters.vue
+++ b/client/src/components/History/ContentFilters.vue
@@ -1,7 +1,13 @@
 <template>
     <b-input-group v-if="params">
         <DebouncedInput v-model.trim="filterText" v-slot="{ value, input }">
-            <b-form-input size="sm" :value="value" @input="input" :placeholder="'Search Filter' | localize" />
+            <b-form-input
+                size="sm"
+                :value="value"
+                @input="input"
+                :placeholder="'Search Filter' | localize"
+                data-description="filter text input"
+            />
         </DebouncedInput>
 
         <b-input-group-append>
@@ -10,6 +16,7 @@
                 :pressed="showDeleted"
                 :variant="showDeleted ? 'info' : 'secondary'"
                 @click="showDeleted = !showDeleted"
+                data-description="show deleted filter toggle"
             >
                 {{ "Deleted" | localize }}
             </b-button>
@@ -18,6 +25,7 @@
                 :pressed="showHidden"
                 :variant="showHidden ? 'info' : 'secondary'"
                 @click="showHidden = !showHidden"
+                data-description="show hidden filter toggle"
             >
                 {{ "Hidden" | localize }}
             </b-button>
diff --git a/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue b/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue
index c037ddbc56a..2ef03d5f2ae 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue
@@ -11,7 +11,7 @@
                 class="px-1"
             />
             <IconButton
-                v-if="notIn(STATES.DISCARDED)"
+                v-if="writable && notIn(STATES.DISCARDED)"
                 icon="pen"
                 :title="editButtonTitle"
                 :disabled="dataset.deleted || isIn(STATES.UPLOAD, STATES.NEW)"
@@ -20,7 +20,7 @@
                 class="px-1"
             />
             <IconButton
-                v-if="dataset.accessible"
+                v-if="writable && dataset.accessible"
                 :icon="dataset.deleted ? 'trash-restore' : 'trash'"
                 :title="deleteButtonTitle"
                 :disabled="dataset.purged"
@@ -165,6 +165,7 @@ export default {
     props: {
         dataset: { type: Dataset, required: true },
         expanded: { type: Boolean, required: true },
+        writable: { type: Boolean, default: true },
     },
 
     data() {
diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index e36d2c8d2e1..375fc1b4a60 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -16,8 +16,8 @@ either through the props, and make updates through the events -->
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
         <!-- name, state buttons, menus -->
-        <nav class="p-1 d-flex align-items-center cursor-pointer" @click.stop="$emit('update:expanded', !expanded)">
-            <div class="d-flex align-items-center flex-grow-1 overflow-hidden">
+        <nav class="p-1 d-flex align-items-baseline cursor-pointer" @click.stop="$emit('update:expanded', !expanded)">
+            <div class="d-flex align-items-baseline flex-grow-1 overflow-hidden">
                 <div class="pl-1" v-if="showSelectionBox">
                     <b-check class="selector" :checked="selected" @change="$emit('update:selected', $event)"></b-check>
                 </div>
@@ -47,30 +47,30 @@ either through the props, and make updates through the events -->
                     @click.stop="$emit('undelete')"
                 />
 
-                <div class="content-title p-1 overflow-hidden">
+                <div class="content-title title p-1 overflow-hidden">
                     <h5 class="text-truncate" v-if="collapsed">
-                        <!-- <span class="hid">{{ dataset.hid }}</span> -->
-                        <span class="name">{{ dataset.title }}</span>
+                        <span class="hid sr-only" data-description="dataset hid">{{ dataset.hid }}</span>
+                        <span class="name" data-description="dataset name">{{ dataset.title }}</span>
                     </h5>
                 </div>
             </div>
 
             <div class="d-flex">
                 <slot name="menu">
-                    <DatasetMenu :dataset="dataset" :expanded="expanded" v-on="$listeners" />
+                    <DatasetMenu :dataset="dataset" :expanded="expanded" :writable="writable" v-on="$listeners" />
                 </slot>
             </div>
         </nav>
 
-        <!--- read-only tags with name: prefix
-        <div v-if="collapsed && dataset.nameTags.length" class="nametags p-2">
+        <!--- read-only tags with name: prefix -->
+        <div v-if="collapsed && dataset.nameTags.length" class="nametags px-2 pb-2">
             <Nametag v-for="tag in dataset.nameTags" :key="tag" :tag="tag" />
-        </div> -->
+        </div>
 
         <div v-if="expanded" class="p-3 details">
             <div class="d-flex">
                 <div class="flex-grow-1">
-                    <h4>{{ name || "(Dataset Name)" }}</h4>
+                    <h4 data-description="dataset name">{{ name || "(Dataset Name)" }}</h4>
 
                     <template v-if="!editing">
                         <p v-if="annotation">{{ annotation }}</p>
@@ -96,7 +96,7 @@ either through the props, and make updates through the events -->
                             max-rows="4"
                         ></b-textarea>
 
-                        <ContentTags class="mt-2" :content="dataset" />
+                        <StatelessTags class="mt-2" v-model="tags" />
                     </div>
                 </div>
 
@@ -114,7 +114,7 @@ either through the props, and make updates through the events -->
             </div>
 
             <div class="details">
-                <DatasetSummary :dataset="dataset" />
+                <DatasetSummary :dataset="dataset" class="summary" />
 
                 <div class="display-applications" v-if="dataset.displayLinks.length">
                     <div class="display-application" v-for="app in dataset.displayLinks" :key="app.label">
@@ -140,10 +140,10 @@ import { Nametag } from "components/Nametags";
 import StatusIcon from "../../StatusIcon";
 import DatasetMenu from "./DatasetMenu";
 import DatasetSummary from "./Summary";
-import ContentTags from "../../ContentTags";
 import { legacyNavigationMixin } from "components/plugins/legacyNavigation";
 import IconButton from "components/IconButton";
 import EditorMenu from "../../EditorMenu";
+import { StatelessTags } from "components/Tags";
 
 export default {
     mixins: [legacyNavigationMixin],
@@ -151,10 +151,10 @@ export default {
         StatusIcon,
         DatasetMenu,
         DatasetSummary,
-        ContentTags,
         Nametag,
         IconButton,
         EditorMenu,
+        StatelessTags,
     },
     props: {
         dataset: { type: Dataset, required: true },
@@ -202,6 +202,16 @@ export default {
                 this.patchTempContent({ annotation: newVal });
             },
         },
+        tags: {
+            get() {
+                return this.tempContent?.tags || [];
+            },
+            set(newTags) {
+                if (Array.isArray(newTags)) {
+                    this.patchTempContent({ tags: newTags });
+                }
+            },
+        },
         showSelectionBox() {
             return this.selectable && this.showSelection;
         },
diff --git a/client/src/components/History/ContentItem/Dataset/Summary/index.vue b/client/src/components/History/ContentItem/Dataset/Summary/index.vue
index e11d4ed7a55..d77aa511c2d 100644
--- a/client/src/components/History/ContentItem/Dataset/Summary/index.vue
+++ b/client/src/components/History/ContentItem/Dataset/Summary/index.vue
@@ -1,5 +1,5 @@
 <template>
-    <component :is="summaryComponent" :dataset="dataset" class="summary" />
+    <component :is="summaryComponent" :dataset="dataset" />
 </template>
 
 <script>
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index 8c712e6176a..2ed9276e8c8 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -10,8 +10,8 @@
         @keydown.arrow-right.self.stop="$emit('viewCollection')"
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
-        <nav class="p-1 d-flex align-items-center cursor-pointer">
-            <div class="d-flex align-items-center flex-grow-1 overflow-hidden">
+        <nav class="p-1 d-flex align-items-baseline cursor-pointer">
+            <div class="d-flex align-items-baseline flex-grow-1 overflow-hidden">
                 <div class="pl-1" v-if="showSelection">
                     <b-check
                         class="selector"
@@ -55,9 +55,9 @@
                     variant="link"
                 />
 
-                <div class="content-title flex-grow-1 overflow-hidden" @click.stop="$emit('viewCollection')">
+                <div class="content-title title flex-grow-1 overflow-hidden" @click.stop="$emit('viewCollection')">
                     <h5 class="text-truncate">
-                        <!-- <span class="hid">{{ dsc.hid }}</span> -->
+                        <span class="hid sr-only">{{ dsc.hid }}</span>
                         <span class="name">{{ dsc.name }}</span>
                         <span class="description">
                             ({{ dsc.collectionType | localize }} {{ dsc.collectionCountDescription | localize }})
@@ -74,7 +74,7 @@
         </nav>
 
         <!--- read-only tags with name: prefix -->
-        <div v-if="dsc.nameTags.length" class="nametags p-2">
+        <div v-if="collapsed && dsc.nameTags.length" class="nametags px-2 pb-2">
             <Nametag v-for="tag in dsc.nameTags" :key="tag" :tag="tag" />
         </div>
 
@@ -113,6 +113,9 @@ export default {
         typedId() {
             return this.dsc.type_id;
         },
+        collapsed() {
+            return !this.expanded;
+        },
     },
 };
 </script>
diff --git a/client/src/components/History/ContentItem/Subcollection.vue b/client/src/components/History/ContentItem/Subcollection.vue
index 45b4a7e8f78..0a7a632f405 100644
--- a/client/src/components/History/ContentItem/Subcollection.vue
+++ b/client/src/components/History/ContentItem/Subcollection.vue
@@ -2,42 +2,27 @@
     inside of other collections -->
 
 <template>
-    <div
-        v-bind="$attrs"
-        v-on="$listeners"
-        class="collapsed"
-        :data-state="state"
-        @keydown.arrow-right.self.stop.prevent="$emit('viewCollection')"
-        @click.stop="$emit('viewCollection')"
-    >
-        <nav class="content-top-menu d-flex align-items-center justify-content-between p-1">
-            <h5 class="flex-grow-1 overflow-hidden mr-auto text-nowrap text-truncate">
-                <span class="name">{{ dsc.name }}</span>
-                <span class="description">
-                    ({{ dsc.collectionType | localize }} {{ dsc.collectionCountDescription | localize }})
-                </span>
-            </h5>
-        </nav>
-    </div>
+    <DscUI v-if="dsc" v-bind="$attrs" v-on="$listeners" class="dataset-collection history-content" :dsc="dsc" />
 </template>
 
 <script>
 import { DatasetCollection } from "../model/DatasetCollection";
 import { STATES } from "../model";
+import DscUI from "./DatasetCollection/DscUI";
 
 export default {
+    components: {
+        DscUI,
+    },
     props: {
         item: { type: Object, required: true },
     },
     computed: {
         dsc() {
-            return new DatasetCollection(this.item);
-        },
-        state() {
-            // TODO: see if there are situations where this rule won't work
-            // subcollection doesn't have as much information as the collection
-            // did, but it's probably ok
-            return this.dsc.state || STATES.OK;
+            // TODO: I'm sure this is not the right way to set the state, but the collection api
+            // doesn't return any kind of state value at because the api is inconsistent
+            // /api/dataset_collections/d071e794759ab192/contents/e144222fed44799a
+            return new DatasetCollection({ ...this.item, populated_state: STATES.OK });
         },
     },
 };
diff --git a/client/src/components/History/ContentItem/Subdataset.vue b/client/src/components/History/ContentItem/Subdataset.vue
index 010dfa38315..89264e97b6c 100644
--- a/client/src/components/History/ContentItem/Subdataset.vue
+++ b/client/src/components/History/ContentItem/Subdataset.vue
@@ -10,7 +10,6 @@ cache first to see if we already have the data -->
         :dataset="dataset"
         :writable="false"
         :selectable="false"
-        :row-key="rowKey"
         @update:expanded="onExpand"
     />
 </template>
diff --git a/client/src/components/History/ContentOperations.vue b/client/src/components/History/ContentOperations.vue
index 33abb59142f..d81f5abc2cf 100644
--- a/client/src/components/History/ContentOperations.vue
+++ b/client/src/components/History/ContentOperations.vue
@@ -15,6 +15,7 @@
                     :disabled="!totalMatches"
                     :pressed="showFilter"
                     @click="showFilter = !showFilter"
+                    data-description="content filter toggle"
                 />
                 <IconButton
                     title="Collapse Expanded Items"
@@ -91,6 +92,7 @@
                     size="sm"
                     text="Create New Content"
                     :disabled="!totalMatches"
+                    data-description="new content menu"
                 >
                     <template v-slot:button-content>
                         <Icon icon="plus" />
@@ -104,23 +106,40 @@
                     <b-dropdown-item
                         aria-describedby="history-op-new-content"
                         @click="iframeRedirect('/dataset/copy_datasets')"
+                        data-description="copy datasets"
                     >
                         <span v-localize>Copy Datasets</span>
                     </b-dropdown-item>
 
-                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildDatasetList">
+                    <b-dropdown-item
+                        aria-describedby="history-op-new-content"
+                        @click="buildDatasetList"
+                        data-description="build list"
+                    >
                         <span v-localize>Build Dataset List</span>
                     </b-dropdown-item>
 
-                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildDatasetPair">
+                    <b-dropdown-item
+                        aria-describedby="history-op-new-content"
+                        @click="buildDatasetPair"
+                        data-description="build pair"
+                    >
                         <span v-localize>Build Dataset Pair</span>
                     </b-dropdown-item>
 
-                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildListOfPairs">
+                    <b-dropdown-item
+                        aria-describedby="history-op-new-content"
+                        @click="buildListOfPairs"
+                        data-description="build list of pairs"
+                    >
                         <span v-localize>Build List of Dataset Pairs</span>
                     </b-dropdown-item>
 
-                    <b-dropdown-item aria-describedby="history-op-new-content" @click="buildCollectionFromRules">
+                    <b-dropdown-item
+                        aria-describedby="history-op-new-content"
+                        @click="buildCollectionFromRules"
+                        data-description="build collection from rules"
+                    >
                         <span v-localize>Build Collection from Rules</span>
                     </b-dropdown-item>
                 </b-dropdown>
@@ -311,8 +330,19 @@ export default {
         async buildNewCollection(collectionTypeCode) {
             const modalResult = await buildCollectionModal(collectionTypeCode, this.history.id, this.contentSelection);
             const newCollection = await createDatasetCollection(this.history, modalResult);
+
+            // cache the collection
             await cacheContent(newCollection);
-            this.$emit("manualReload");
+
+            // have to hide the source items if that was requested
+            if (modalResult.hide_source_items) {
+                this.contentSelection.forEach(async (dataset) => {
+                    await cacheContent({ ...dataset, visible: false }, true);
+                });
+                this.$emit("resetSelection");
+            }
+
+            // this.$emit("manualReload");
         },
 
         // #endregion
diff --git a/client/src/components/History/ContentTags.vue b/client/src/components/History/ContentTags.vue
deleted file mode 100644
index 13bd5ef6feb..00000000000
--- a/client/src/components/History/ContentTags.vue
+++ /dev/null
@@ -1,54 +0,0 @@
-<template>
-    <StatelessTags :value="tags" @before-adding-tag="saveTag" @before-deleting-tag="deleteTag" />
-</template>
-
-<script>
-// TODO: rewrite tags with provider component instead of dependency injection
-
-import { Content } from "./model";
-import { updateContentFields } from "./model/queries";
-import { cacheContent } from "./caching";
-
-import { StatelessTags } from "components/Tags";
-import { createTag } from "components/Tags/model";
-
-export default {
-    components: {
-        StatelessTags,
-    },
-    props: {
-        content: { type: Content, required: true },
-    },
-    computed: {
-        tags() {
-            return this.content.tags;
-        },
-    },
-    methods: {
-        async saveTag({ tag, addTag }) {
-            const newTag = createTag(tag);
-            if (!newTag.valid) {
-                return;
-            }
-            const tagSet = new Set(this.tags);
-            tagSet.add(newTag.text);
-            const tags = Array.from(tagSet);
-            const ajaxResult = await updateContentFields(this.content, { tags });
-            await cacheContent(ajaxResult);
-            addTag(newTag);
-        },
-        async deleteTag({ tag, deleteTag }) {
-            const doomedTag = createTag(tag);
-            if (!doomedTag.valid) {
-                return;
-            }
-            const tagSet = new Set([...this.tags]);
-            tagSet.delete(doomedTag.text);
-            const tags = Array.from(tagSet);
-            const ajaxResult = await updateContentFields(this.content, { tags });
-            await cacheContent(ajaxResult);
-            deleteTag(tag);
-        },
-    },
-};
-</script>
diff --git a/client/src/components/History/CurrentCollection/Details.vue b/client/src/components/History/CurrentCollection/Details.vue
index 855d1567c20..d2b1b9bc546 100644
--- a/client/src/components/History/CurrentCollection/Details.vue
+++ b/client/src/components/History/CurrentCollection/Details.vue
@@ -1,18 +1,26 @@
 <!-- Edits a collection, if that collection is editable. -->
 
 <template>
-    <section class="history-details d-flex">
+    <section class="history-details d-flex" data-description="edit details">
         <div class="flex-grow-1">
-            <h3 class="history-title">{{ dscName || "(Collection Name)" }}</h3>
+            <h3 class="history-title" data-description="collection name display">
+                {{ dscName || "(Collection Name)" }}
+            </h3>
             <p class="mt-1">
                 <i class="fas fa-folder"></i>
                 a {{ dsc.collectionType | localize }}
                 {{ dsc.collectionCount | localize }}
             </p>
 
-            <div v-if="isEditing" class="mt-3">
-                <b-textarea v-model="dscName" placeholder="Collection Name" trim max-rows="4"></b-textarea>
-                <ContentTags :content="dsc" class="mt-3" />
+            <div v-if="isEditing" class="mt-3" @keydown.esc="revertAndCancel" data-description="edit form">
+                <b-textarea
+                    v-model="dscName"
+                    placeholder="Collection Name"
+                    trim
+                    max-rows="4"
+                    data-description="name input"
+                ></b-textarea>
+                <StatelessTags v-model="tags" class="mt-3 tags" />
             </div>
 
             <div v-else-if="dsc.tags && dsc.tags.length">
@@ -37,14 +45,14 @@
 <script>
 import { DatasetCollection } from "../model";
 import { Nametag } from "components/Nametags";
-import ContentTags from "../ContentTags";
 import EditorMenu from "../EditorMenu";
+import { StatelessTags } from "components/Tags";
 
 export default {
     components: {
-        ContentTags,
         Nametag,
         EditorMenu,
+        StatelessTags,
     },
     props: {
         dsc: { type: DatasetCollection, required: true },
@@ -77,6 +85,16 @@ export default {
                 this.patchTempDsc({ name: newName });
             },
         },
+        tags: {
+            get() {
+                return this.tempContent?.tags || [];
+            },
+            set(newTags) {
+                if (Array.isArray(newTags)) {
+                    this.patchTempDsc({ tags: newTags });
+                }
+            },
+        },
     },
     watch: {
         dscId(newId, oldId) {
@@ -103,6 +121,10 @@ export default {
                 this.reset();
             }
         },
+        revertAndCancel() {
+            this.revert();
+            this.editing = false;
+        },
     },
 };
 </script>
diff --git a/client/src/components/History/CurrentCollection/Panel.vue b/client/src/components/History/CurrentCollection/Panel.vue
index 9a259d92e05..1d9899a68f6 100644
--- a/client/src/components/History/CurrentCollection/Panel.vue
+++ b/client/src/components/History/CurrentCollection/Panel.vue
@@ -44,6 +44,8 @@
                                     :selectable="false"
                                     @update:expanded="setExpanded(item, $event)"
                                     @viewCollection="$emit('viewCollection', item)"
+                                    :data-index="index"
+                                    :data-row-key="rowKey"
                                 />
                             </template>
                         </Scroller>
diff --git a/client/src/components/History/CurrentCollection/TopNav.vue b/client/src/components/History/CurrentCollection/TopNav.vue
index cea439b63df..b63d4878287 100644
--- a/client/src/components/History/CurrentCollection/TopNav.vue
+++ b/client/src/components/History/CurrentCollection/TopNav.vue
@@ -1,11 +1,18 @@
 <template>
-    <b-dropdown v-if="breadCrumbOptions.length" size="sm" text="Return to..." boundary="viewport" no-caret>
+    <b-dropdown
+        v-if="breadCrumbOptions.length"
+        size="sm"
+        text="Return to..."
+        boundary="viewport"
+        data-description="collection breadcrumbs menu"
+        no-caret
+    >
         <template v-slot:button-content>
             <Icon icon="arrow-up" class="mr-1" />
             <b class="text-nowrap" v-localize>Return to...</b>
         </template>
 
-        <b-dropdown-item @click="close">
+        <b-dropdown-item @click="close" data-description="back to history">
             <span>History: {{ history.name }}</span>
         </b-dropdown-item>
 
@@ -14,7 +21,14 @@
         </b-dropdown-item>
     </b-dropdown>
 
-    <b-button v-else size="sm" title="`Return to: ${history.name}`" @click="close">
+    <b-button
+        v-else
+        class="back"
+        size="sm"
+        title="`Return to: ${history.name}`"
+        @click="close"
+        data-description="back to history"
+    >
         <Icon icon="arrow-up" class="mr-1" />
         <b class="text-nowrap">Return to: {{ history.name }}</b>
     </b-button>
diff --git a/client/src/components/History/EditorMenu.vue b/client/src/components/History/EditorMenu.vue
index 8ab53810e1b..7601b259cf9 100644
--- a/client/src/components/History/EditorMenu.vue
+++ b/client/src/components/History/EditorMenu.vue
@@ -6,7 +6,7 @@
             :pressed="editing"
             icon="pen"
             @click="$emit('update:editing', !editing)"
-            data-description="history editor toggle"
+            data-description="editor toggle"
         />
 
         <template v-if="editing">
@@ -17,7 +17,7 @@
                 :class="{ disabled: !valid }"
                 icon="save"
                 @click="$emit('save')"
-                data-description="history editor save button"
+                data-description="editor save button"
             />
 
             <!-- revert -->
@@ -27,6 +27,7 @@
                 :class="{ disabled: !dirty }"
                 icon="undo"
                 @click="$emit('revert')"
+                data-description="editor revert button"
             />
 
             <!-- form status -->
@@ -36,6 +37,7 @@
                 :class="{ valid: valid, disabled: !valid }"
                 :icon="valid ? 'check' : 'exclamation-triangle'"
                 variant="link"
+                data-description="editor status"
             />
         </template>
     </nav>
diff --git a/client/src/components/History/HistoriesMenu.vue b/client/src/components/History/HistoriesMenu.vue
index f9cb8d3d047..6e133da80ec 100644
--- a/client/src/components/History/HistoriesMenu.vue
+++ b/client/src/components/History/HistoriesMenu.vue
@@ -14,7 +14,7 @@
         >
             <template v-slot:button-content>
                 <Icon class="mr-1" icon="folder" />
-                <span id="click-me-fool" class="text-nowrap">{{ title | l }}</span>
+                <span class="text-nowrap">{{ title | l }}</span>
             </template>
 
             <b-dropdown-text>
diff --git a/client/src/components/History/History.vue b/client/src/components/History/History.vue
index 516147616a0..dda1fc58161 100644
--- a/client/src/components/History/History.vue
+++ b/client/src/components/History/History.vue
@@ -83,6 +83,8 @@
                                     @update:selected="setSelected(item, $event)"
                                     @viewCollection="$emit('viewCollection', item)"
                                     :data-hid="item.hid"
+                                    :data-index="index"
+                                    :data-row-key="rowKey"
                                 />
                             </template>
                         </Scroller>
diff --git a/client/src/components/History/HistoryDetails.vue b/client/src/components/History/HistoryDetails.vue
index c443219632e..8cae599ac71 100644
--- a/client/src/components/History/HistoryDetails.vue
+++ b/client/src/components/History/HistoryDetails.vue
@@ -4,26 +4,29 @@ Emit new History model object when done via .sync syntax
 -->
 
 <template>
-    <section class="history-details d-flex">
+    <section class="history-details d-flex" data-description="edit details">
         <div class="flex-grow-1 overflow-hidden">
             <h3 data-description="history name display">{{ historyName || "(History Name)" }}</h3>
-            <h5 class="history-size">{{ history.size | niceFileSize }}</h5>
+            <h5 class="history-size">
+                <span v-if="history.size">{{ history.size | niceFileSize }}</span>
+                <span v-else>(empty)</span>
+            </h5>
 
             <!-- display title, annotation, tags -->
             <div v-if="!editing">
-                <p v-if="annotation" class="mt-1">{{ annotation }}</p>
-                <HistoryTags class="mt-2" :history="history" :disabled="true" />
+                <p data-description="history annotation" v-if="annotation" class="mt-1">{{ annotation }}</p>
+                <StatelessTags class="mt-2 tags" :value="tags" :disabled="true" />
             </div>
 
             <!-- edit form, change title, annotation, or tags -->
-            <div v-else class="mt-3">
+            <div v-else class="mt-3" @keydown.esc="revertAndCancel" data-description="edit form">
                 <b-textarea
                     class="mb-2"
                     v-model="historyName"
                     placeholder="History Name"
                     trim
                     max-rows="4"
-                    data-description="history name input"
+                    data-description="name input"
                 ></b-textarea>
 
                 <b-textarea
@@ -32,10 +35,10 @@ Emit new History model object when done via .sync syntax
                     placeholder="Annotation (optional)"
                     trim
                     max-rows="4"
-                    data-description="history annotation input"
+                    data-description="annotation input"
                 ></b-textarea>
 
-                <HistoryTags class="mt-3" :history="history" />
+                <StatelessTags class="mt-3 tags" v-model="tags" />
             </div>
         </div>
 
@@ -56,13 +59,13 @@ Emit new History model object when done via .sync syntax
 <script>
 import prettyBytes from "pretty-bytes";
 import { History } from "./model";
-import HistoryTags from "./HistoryTags";
 import EditorMenu from "./EditorMenu";
+import { StatelessTags } from "components/Tags";
 
 export default {
     components: {
-        HistoryTags,
         EditorMenu,
+        StatelessTags,
     },
     filters: {
         niceFileSize(rawSize = 0) {
@@ -142,6 +145,10 @@ export default {
                 this.reset();
             }
         },
+        revertAndCancel() {
+            this.revert();
+            this.editing = false;
+        },
     },
 };
 </script>
diff --git a/client/src/components/History/HistoryMenu.vue b/client/src/components/History/HistoryMenu.vue
index a80bf9a9925..22133e24d36 100644
--- a/client/src/components/History/HistoryMenu.vue
+++ b/client/src/components/History/HistoryMenu.vue
@@ -1,6 +1,12 @@
 <template>
     <div>
-        <b-dropdown size="sm" variant="link" text="Histories" toggle-class="text-decoration-none">
+        <b-dropdown
+            size="sm"
+            variant="link"
+            text="Histories"
+            toggle-class="text-decoration-none"
+            data-description="history menu"
+        >
             <template v-slot:button-content>
                 <Icon icon="wrench" />
                 <span class="sr-only">Operations on the current history</span>
@@ -62,7 +68,7 @@
                 <span v-localize>Export Tool Citations</span>
             </b-dropdown-item>
 
-            <b-dropdown-item @click="backboneRoute(history.exportLink)">
+            <b-dropdown-item @click="backboneRoute(history.exportLink)" data-description="export to file">
                 <Icon icon="file-archive" class="mr-1" />
                 <span v-localize>Export History to File</span>
             </b-dropdown-item>
diff --git a/client/src/components/History/StatusIcon.vue b/client/src/components/History/StatusIcon.vue
index 033e6fc2739..b342daad634 100644
--- a/client/src/components/History/StatusIcon.vue
+++ b/client/src/components/History/StatusIcon.vue
@@ -25,7 +25,7 @@ export default {
     },
     computed: {
         stateIcon() {
-            if (this.inState(this.STATES.QUEUED, this.STATES.NEW)) {
+            if (this.inState(this.STATES.QUEUED, this.STATES.NEW, this.STATES.RUNNING)) {
                 return "clock";
             }
             if (this.inState(this.STATES.PAUSED)) {
diff --git a/client/src/components/History/caching/operators/requestWithUpdateTime.js b/client/src/components/History/caching/operators/requestWithUpdateTime.js
index 2a43afd7698..979dfb98fde 100644
--- a/client/src/components/History/caching/operators/requestWithUpdateTime.js
+++ b/client/src/components/History/caching/operators/requestWithUpdateTime.js
@@ -25,7 +25,7 @@ export const requestWithUpdateTime = (config = {}) => {
         requestTime = moment.utc(),
         // indicates we don't want initial results
         noInitial = false,
-        responseQualifier = () => true
+        responseQualifier = (response) => response.status == 200
     } = config;
 
     // mark and flag this update-time, append to next request with same base
diff --git a/client/src/components/History/model/DatasetCollection.js b/client/src/components/History/model/DatasetCollection.js
index 196bd186c31..f66cf6ccd24 100644
--- a/client/src/components/History/model/DatasetCollection.js
+++ b/client/src/components/History/model/DatasetCollection.js
@@ -17,19 +17,13 @@ export class DatasetCollection extends Content {
         super.loadProps(raw);
     }
 
-    // number of contained contents
-    // Need to do some manual handling, because once again, the API is
-    // inconsistent, element_count not returned in the case of a pair. It
-    // should be two, but we need to add that value in.
     get totalElements() {
         if ("element_count" in this) {
             return this.element_count;
         }
-
         if (this.collection_type == "paired") {
             return 2;
         }
-
         return undefined;
     }
 
@@ -58,11 +52,11 @@ export class DatasetCollection extends Content {
 
     // amalgam state value
     get state() {
-        return this.jobSummary.state || this.populated_state;
+        return this.jobSummary.state;
     }
 
     get jobSummary() {
-        return new JobStateSummary(this);
+        return Object.freeze(new JobStateSummary(this));
     }
 
     clone() {
diff --git a/client/src/components/History/model/JobStateSummary.js b/client/src/components/History/model/JobStateSummary.js
index 335c0d34542..93ed2c9cfa1 100644
--- a/client/src/components/History/model/JobStateSummary.js
+++ b/client/src/components/History/model/JobStateSummary.js
@@ -1,20 +1,13 @@
 /**
  * Read-only class takes raw data and applies some rules
  * to determine an amalgam state for a collection.
+ *
+ * Logic stolen from job-state-model.js and hdca-li.js
  */
 import { STATES } from "./states";
 
-// logic largely stolen from the butchered, degenerate mess at:
-// import nonsense from "mvc/history/job-states-model.js"
-
-// Job-state-summary lists
-
 const NON_TERMINAL_STATES = [STATES.NEW, STATES.WAITING, STATES.QUEUED, STATES.RUNNING];
-
-const ERROR_STATES = [
-    STATES.ERROR,
-    STATES.DELETED, // does this exist?
-];
+const ERROR_STATES = [STATES.ERROR, STATES.DELETED];
 
 export class JobStateSummary extends Map {
     constructor(dsc = {}) {
@@ -23,7 +16,6 @@ export class JobStateSummary extends Map {
         this.populated_state = populated_state;
     }
 
-    // state element has value > 0
     hasJobs(key) {
         return this.has(key) ? this.get(key) > 0 : false;
     }
@@ -33,48 +25,23 @@ export class JobStateSummary extends Map {
         return states.some((s) => this.hasJobs(s));
     }
 
-    // amalgam state, basically just for display purposes
     get state() {
-        // FROM mvc/history/hdca-li.js
-        //TODO: model currently has no state
-        // var state;
-        // var jobStatesSummary = this.model.jobStatesSummary;
-        // if (jobStatesSummary) {
-        //     if (jobStatesSummary.new()) {
-        //         state = "loading";
-        //     } else if (jobStatesSummary.errored()) {
-        //         state = "error";
-        //     } else if (jobStatesSummary.terminal()) {
-        //         state = "ok";
-        //     } else if (jobStatesSummary.running()) {
-        //         state = "running";
-        //     } else {
-        //         state = "queued";
-        //     }
-        // } else if (this.model.get("job_source_id")) {
-        //     // Initial rendering - polling will fill in more details in a bit.
-        //     state = "loading";
-        // } else {
-        //     state = this.model.get("populated_state") ? STATES.OK : STATES.RUNNING;
-        // }
-
         if (this.jobCount) {
             if (this.isErrored) {
                 return STATES.ERROR;
             }
-            if (this.isNew) {
-                return STATES.NEW;
-            }
             if (this.isRunning) {
                 return STATES.RUNNING;
             }
+            if (this.isNew) {
+                return STATES.LOADING;
+            }
             if (this.isTerminal) {
                 return STATES.OK;
             }
             return STATES.QUEUED;
         }
-
-        return null;
+        return this.populated_state;
     }
 
     // Flags
@@ -84,11 +51,7 @@ export class JobStateSummary extends Map {
     }
 
     get isErrored() {
-        return this.anyHasJobs(...ERROR_STATES);
-    }
-
-    get isRunning() {
-        return this.hasJobs(STATES.RUNNING);
+        return this.populated_state == STATES.ERROR || this.anyHasJobs(...ERROR_STATES);
     }
 
     get isTerminal() {
@@ -98,6 +61,10 @@ export class JobStateSummary extends Map {
         return !this.anyHasJobs(...NON_TERMINAL_STATES);
     }
 
+    get isRunning() {
+        return this.hasJobs(STATES.RUNNING);
+    }
+
     // Counts
 
     get jobCount() {
@@ -109,6 +76,6 @@ export class JobStateSummary extends Map {
     }
 
     get runningCount() {
-        return this.get(STATES.ERRUNNINGROR);
+        return this.get(STATES.RUNNING);
     }
 }
diff --git a/client/src/components/History/model/ScrollPos.js b/client/src/components/History/model/ScrollPos.js
index 05e44a764d4..dde873dfb64 100644
--- a/client/src/components/History/model/ScrollPos.js
+++ b/client/src/components/History/model/ScrollPos.js
@@ -1,7 +1,7 @@
 export class ScrollPos {
     constructor(props = {}) {
-        const { cursor = 0.0, key = null } = props;
-        this.cursor = cursor;
+        const { cursor = null, key = null } = props;
+        this.cursor = key === null && cursor === null ? 0 : cursor;
         this.key = key;
     }
 }
@@ -12,5 +12,5 @@ ScrollPos.create = function (props) {
 };
 
 ScrollPos.equals = function (a, b) {
-    return a.cursor == b.cursor && a.key == b.key;
+    return a.cursor === b.cursor && a.key === b.key;
 };
diff --git a/client/src/components/History/model/queries.js b/client/src/components/History/model/queries.js
index 3b96eab7918..6688082a220 100644
--- a/client/src/components/History/model/queries.js
+++ b/client/src/components/History/model/queries.js
@@ -359,7 +359,7 @@ export async function createDatasetCollection(history, inputs = {}) {
         copy_elements: true,
         name: "list",
         element_identifiers: [],
-        hide_source_items: "True",
+        hide_source_items: true,
         type: "dataset_collection",
     };
 
diff --git a/client/src/components/History/model/states.js b/client/src/components/History/model/states.js
index 832a493ef22..c344101ef2e 100644
--- a/client/src/components/History/model/states.js
+++ b/client/src/components/History/model/states.js
@@ -33,6 +33,8 @@ export const STATES = {
     // found in job-state summary model?
     // this an actual state value or something derived from deleted prop?
     DELETED: "deleted",
+
+    LOADING: "loading",
 };
 
 export const NOT_READY_STATES = [STATES.UPLOAD, STATES.QUEUED, STATES.RUNNING, STATES.SETTING_METADATA, STATES.NEW];
diff --git a/client/src/components/History/providers/ContentProvider/ContentProvider.js b/client/src/components/History/providers/ContentProvider/ContentProvider.js
index 733cf903f8d..88f3db4e787 100644
--- a/client/src/components/History/providers/ContentProvider/ContentProvider.js
+++ b/client/src/components/History/providers/ContentProvider/ContentProvider.js
@@ -83,7 +83,7 @@ export const ContentProvider = {
 
     methods: {
         resetScrollPos() {
-            this.setScrollPos(ScrollPos.create());
+            this.setScrollPos();
         },
 
         initStreams() {
@@ -96,8 +96,8 @@ export const ContentProvider = {
          * cursor (0-1 value representing how far down it is)
          * @param {object} Object consisting of key and cursor props
          */
-        setScrollPos({ cursor = 0.0, key = null } = {}) {
-            if (isValidNumber(cursor)) {
+        setScrollPos({ cursor = null, key = null } = {}) {
+            if (isValidNumber(cursor) || key !== null) {
                 this.scrollPos = ScrollPos.create({ cursor, key });
             }
         },
diff --git a/client/src/components/History/providers/HistoryContentProvider/HistoryContentProvider.test.js b/client/src/components/History/providers/HistoryContentProvider/HistoryContentProvider.test.js
index 362bc313ed6..9129710dab8 100644
--- a/client/src/components/History/providers/HistoryContentProvider/HistoryContentProvider.test.js
+++ b/client/src/components/History/providers/HistoryContentProvider/HistoryContentProvider.test.js
@@ -148,7 +148,7 @@ describe("HistoryContentProvider", () => {
 
         test("should show results when we pick a HID by scroller height", async () => {
             // set scroller to non-existent hid
-            wrapper.vm.setScrollPos({ cursor: 0.52323, key: null });
+            wrapper.vm.setScrollPos({ cursor: 0.52323 });
             const scrollPayload = await payloadChange({ vm: wrapper.vm });
             const { contents, startKey, topRows, bottomRows, totalMatches } = scrollPayload;
             const hids = payloadHids(scrollPayload);
diff --git a/client/src/components/History/providers/HistoryContentProvider/contentPayload.js b/client/src/components/History/providers/HistoryContentProvider/contentPayload.js
index 881d9b033eb..704f013d654 100644
--- a/client/src/components/History/providers/HistoryContentProvider/contentPayload.js
+++ b/client/src/components/History/providers/HistoryContentProvider/contentPayload.js
@@ -1,4 +1,4 @@
-import { concat, race, timer, merge, partition, Observable, BehaviorSubject, Subject } from "rxjs";
+import { combineLatest, concat, race, timer, merge, partition, Observable, BehaviorSubject, Subject } from "rxjs";
 import {
     debounceTime,
     distinctUntilChanged,
@@ -65,12 +65,15 @@ export const contentPayload = (cfg = {}) => {
             pluck('key')
         );
 
-        // only have cursor height, need to make a guess
-        const estimatedHid$ = noKey$.pipe(
+        const cursor$ = noKey$.pipe(
             pluck('cursor'),
-            withLatestFrom(cursorToHid$),
+        );
+
+        // only have cursor height, need to make a guess
+        const estimatedHid$ = combineLatest(cursor$, cursorToHid$).pipe(
             map((inputs) => estimateHid(...inputs)),
             map(hid => Math.round(hid)),
+            distinctUntilChanged(),
             show(debug, hid => console.log("estimatedHid", hid, history)),
         );
 
@@ -149,18 +152,13 @@ export const contentPayload = (cfg = {}) => {
             map(([[lastResponse, response], pos, hid]) => {
                 const updatesAtTop = response.maxHid > lastResponse.maxHid;
 
-                const scrollerInDefault = (pos.cursor == 0 && pos.key == null);
+                const scrollerExactlyAtTop = pos.cursor === 0 || pos.key === lastResponse.maxHid;
                 const fudge = 2;
-                const scrollNearLastTop = Math.abs(hid - lastResponse.maxContentHid) < fudge;
-                const scrollerAtTop = scrollerInDefault || scrollNearLastTop;
-
-                // console.group("should i reposition?");
-                // console.log("updatesAtTop", updatesAtTop);
-                // console.log("scrollerAtTop", scrollerAtTop);
-                // console.groupEnd();
+                const scrollNearLastTop = pos.key !== null ? Math.abs(pos.key - lastResponse.maxContentHid) < fudge : false;
+                const scrollerAtTop = scrollerExactlyAtTop || scrollNearLastTop;
 
                 if (updatesAtTop && scrollerAtTop) {
-                    return new ScrollPos({ cursor: 0, key: response.maxHid })
+                    return ScrollPos.create({ cursor: 0, key: response.maxContentHid });
                 }
                 return null;
             }),
diff --git a/client/src/components/History/providers/setCache.js b/client/src/components/History/providers/setCache.js
index 5ba3d51585c..e270600811f 100644
--- a/client/src/components/History/providers/setCache.js
+++ b/client/src/components/History/providers/setCache.js
@@ -17,6 +17,6 @@ export function loadSet(key) {
     } catch (err) {
         console.warn("Error loading cached list", key, err);
         sessionStorage.removeItem(key);
-        return null;
+        return new Set();
     }
 }
diff --git a/client/src/components/Nametags/Nametag.vue b/client/src/components/Nametags/Nametag.vue
index 3fe004fcc3f..9833e5e79ee 100644
--- a/client/src/components/Nametags/Nametag.vue
+++ b/client/src/components/Nametags/Nametag.vue
@@ -1,5 +1,5 @@
 <template>
-    <span class="badge badge-tags" :style="tagStyles" @click.prevent="$emit('tag-click', tag)"> {{ tagLabel }} </span>
+    <span class="badge badge-tags" :style="tagStyles" @click.prevent="$emit('tag-click', tag)">{{ tagLabel }}</span>
 </template>
 
 <script>
diff --git a/client/src/components/RuleCollectionBuilder.vue b/client/src/components/RuleCollectionBuilder.vue
index 5618587000f..1647ae694ea 100644
--- a/client/src/components/RuleCollectionBuilder.vue
+++ b/client/src/components/RuleCollectionBuilder.vue
@@ -611,7 +611,6 @@ import $ from "jquery";
 import _ from "underscore";
 import { getAppRoot } from "onload/loadConfig";
 import { getGalaxyInstance } from "app";
-import AjaxQueue from "utils/ajax-queue";
 import axios from "axios";
 import _l from "utils/localization";
 import HotTable from "@handsontable/vue";
@@ -640,6 +639,14 @@ Vue.use(BootstrapVue);
 const RULES = RuleDefs.RULES;
 const MAPPING_TARGETS = RuleDefs.MAPPING_TARGETS;
 
+// convert deferred backbone nonsense into a promise
+const deferredToPromise = (d) => {
+    return new Promise((resolve, reject) => {
+        d.done((_, result) => resolve(result));
+        d.fail((err) => reject(err));
+    });
+};
+
 export default {
     data: function () {
         let orientation = "vertical";
@@ -1353,21 +1360,19 @@ export default {
             }
             this.saveSession(JSON.stringify(asJson));
             this.state = "wait";
-            const name = this.collectionName;
-            const collectionType = this.collectionType;
+            const { collectionName: name, collectionType, hideSourceItems } = this;
             if (this.elementsType == "datasets" || this.elementsType == "library_datasets") {
                 const elements = this.creationElementsFromDatasets();
                 if (this.state !== "error") {
-                    new AjaxQueue.AjaxQueue(
-                        _.map(elements, (elements, name) => {
-                            return () => {
-                                const response = this.creationFn(elements, collectionType, name, this.hideSourceItems);
-                                return response;
-                            };
-                        })
-                    )
-                        .done(this.oncreate)
-                        .fail(this.renderFetchError);
+                    const deferreds = Object.entries(elements).map(([name, els]) => {
+                        // This looks like a promise but it is not one because creationFn and
+                        // oncreate are references to function from the backbone models which means
+                        // they are expecting their arguments in a different order. So, looks like,
+                        // jQuery.Deferred and therefore jQuery are still dependencies
+                        return this.creationFn(els, collectionType, name, hideSourceItems).then(this.oncreate);
+                    });
+                    const promises = deferreds.map(deferredToPromise);
+                    return Promise.all(promises).catch((err) => this.renderFetchError(err));
                 }
             } else if (this.elementsType == "collection_contents") {
                 this.resetSource();
diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 9da66468f5f..5cde6d1eb17 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -56,6 +56,8 @@ class WAIT_TYPES:
     SHED_SEARCH = WaitType('shed_search', 30)
     # Wait time for repository installation
     REPO_INSTALL = WaitType('repo_install', 60)
+    # Beta history Polling Duration
+    HISTORY_POLL = WaitType("history_poll", 3)
 
 
 # Choose a moderate wait type for operations that don't specify a type.
@@ -110,6 +112,25 @@ def retry_during_transitions(f, attempts=RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAU
     return _retry
 
 
+def edit_details(f, scope=".beta.history"):
+    """Open the editor, run the edits, hit the save button"""
+
+    @wraps(f)
+    def func_wrapper(self, *args, **kwds):
+        # open editor
+        if self.is_beta_history():
+            self.open_history_editor(scope=scope)
+        # run edits
+        result = f(self, *args, **kwds)
+        # save edits
+        if self.is_beta_history():
+            save_button = self.beta_history_element('editor save button')
+            save_button.wait_for_and_click()
+        return result
+
+    return func_wrapper
+
+
 class NavigatesGalaxy(HasDriver):
     """Class with helpers methods for driving components of the Galaxy interface.
 
@@ -228,7 +249,7 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_name_element(self):
         if self.is_beta_history():
-            component = self.components.history_panel.name_beta
+            component = self.beta_history_element("history name display")
         else:
             component = self.components.history_panel.name
         return component.wait_for_present()
@@ -291,9 +312,11 @@ class NavigatesGalaxy(HasDriver):
         self.sleep_for(WAIT_TYPES.UX_RENDER)
 
     def history_panel_wait_for_hid_ok(self, hid, allowed_force_refreshes=0):
-        self.history_panel_wait_for_hid_state(hid, 'ok', allowed_force_refreshes=allowed_force_refreshes)
+        return self.history_panel_wait_for_hid_state(hid, 'ok', allowed_force_refreshes=allowed_force_refreshes)
 
     def history_panel_item_component(self, history_item=None, hid=None, multi_history_panel=False):
+        if self.is_beta_history():
+            return self.content_item_by_attributes(hid=hid)
         if history_item is None:
             assert hid
             history_item = self.hid_to_history_item(hid)
@@ -373,7 +396,8 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_wait_for_hid_state(self, hid, state, allowed_force_refreshes=0, multi_history_panel=False):
         if self.is_beta_history():
-            history_item_selector_state = self.components.history_panel.item.by_state_and_hid(state='ok', hid=hid)
+            history_item_selector_state = self.content_item_by_attributes(hid=hid, state=state)
+            history_item_selector_state.wait_for_visible()
         else:
             history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes, multi_history_panel=multi_history_panel)
             # history_item_selector_state = history_item_selector.with_class(f"state-{state}")
@@ -564,7 +588,8 @@ class NavigatesGalaxy(HasDriver):
         self._perform_upload(paste_data=paste_data, **kwd)
 
     def _perform_upload(self, test_path=None, paste_data=None, ext=None, genome=None, ext_all=None, genome_all=None):
-        self.home()
+        if not self.is_beta_history():
+            self.home()
         self.upload_start_click()
 
         self.upload_set_footer_extension(ext_all)
@@ -1249,24 +1274,40 @@ class NavigatesGalaxy(HasDriver):
         return not self.components.history_panel.beta.is_absent
 
     # avoids problematic ID and classes on markup
-    def beta_element(self, attribute_value, attribute_name="data-description"):
-        return self.components.history_panel.by_attribute(name=attribute_name, value=attribute_value)
+    def beta_history_element(self, attribute_value, attribute_name="data-description", scope=".beta.history"):
+        return self.components._.by_attribute(name=attribute_name, value=attribute_value, scope=scope)
+
+    # join list of attrs into css attribute selectors and append to base beta_item selector
+    def content_item_by_attributes(self, **attrs):
+        suffix_list = [f'[data-{k}="{v}"]' for (k, v) in attrs.items()]
+        suffix = "".join(suffix_list)
+        return self.components.history_panel.content_item.selector(suffix=suffix)
 
     def history_click_create_new(self):
         if not self.is_beta_history():
             self.components.history_panel.new_history_button.wait_for_and_click()
         else:
-            dropdown = self.beta_element("histories operation menu")
+            dropdown = self.beta_history_element("histories operation menu")
             dropdown.wait_for_and_click()
-            option = self.beta_element("create new history")
+            option = self.beta_history_element("create new history")
             option.wait_for_and_click()
 
     def history_panel_click_copy_elements(self):
         if self.is_beta_history():
-            self.components.history_panel.options_button_copy_datasets_beta.wait_for_and_click()
+            self.use_bootstrap_dropdown(option="copy datasets", menu="new content menu")
         else:
             self.click_history_option("Copy Datasets")
 
+    def use_bootstrap_dropdown(self, option=None, menu=None):
+        """uses bootstrap dropdown by data-description attributes"""
+        if option is None:
+            raise TypeError
+        if menu is None:
+            raise TypeError
+        toggle = self.beta_history_element(menu).descendant("button")
+        self.wait_for_and_click(toggle)
+        return self.beta_history_element(option).wait_for_and_click()
+
     @retry_during_transitions
     def histories_click_advanced_search(self):
         search_selector = '#standard-search .advanced-search-toggle'
@@ -1284,36 +1325,38 @@ class NavigatesGalaxy(HasDriver):
                 names.append(name)
         return names
 
+    @edit_details
     def history_panel_add_tags(self, tags):
         tag_icon = self.components.history_panel.tag_icon
         tag_area = self.components.history_panel.tag_area
         tag_area_input = self.components.history_panel.tag_area_input
 
-        if tag_area.is_absent or not tag_area.is_displayed:
+        # if the tag editor is not present but the tag_icon is, then click it
+        if not tag_icon.is_absent and (tag_area.is_absent or not tag_area.is_displayed):
             tag_icon.wait_for_and_click()
 
-        tag_area = tag_area_input.wait_for_and_click()
+        input_element = tag_area_input.wait_for_and_click()
 
         for tag in tags:
-            tag_area.send_keys(tag)
-            self.send_enter(tag_area)
-            time.sleep(.5)
+            input_element.send_keys(tag)
+            self.send_enter(input_element)
+            self.sleep_for(self.wait_types.UX_RENDER)
 
+    @edit_details
     def history_panel_rename(self, new_name):
+        editable_text_input_element = self.history_panel_name_input()
         if self.is_beta_history():
-            toggle = self.beta_element("history editor toggle")
-            toggle.wait_for_and_click()
-            editable_text_input_element = self.beta_element("history name input").wait_for_visible()
-            self.sleep_for_seconds(1)
             editable_text_input_element.clear()
-            editable_text_input_element.send_keys(new_name)
-            self.send_enter(editable_text_input_element)
-            save_button = self.beta_element("history editor save button")
-            save_button.wait_for_and_click()
+        editable_text_input_element.send_keys(new_name)
+        self.send_enter(editable_text_input_element)
+        return editable_text_input_element
+
+    def history_panel_name_input(self):
+        if self.is_beta_history():
+            editable_text_input_element = self.beta_history_element("name input").wait_for_visible()
         else:
             editable_text_input_element = self.history_panel_click_to_rename()
-            editable_text_input_element.send_keys(new_name)
-            self.send_enter(editable_text_input_element)
+        return editable_text_input_element
 
     def history_panel_click_to_rename(self):
         history_panel = self.components.history_panel
@@ -1416,15 +1459,17 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_click_item_title(self, hid, **kwds):
         item_component = self.history_panel_item_component(hid=hid)
-        details_component = item_component.details
         item_component.title.wait_for_and_click()
-
         if kwds.get("wait", False):
-            details_displayed = details_component.is_displayed
-            if details_displayed:
-                details_component.wait_for_absent_or_hidden()
+            if self.is_beta_history():
+                item_component.details.wait_for_present()
             else:
-                details_component.wait_for_visible()
+                details_component = item_component.details
+                details_displayed = details_component.is_displayed
+                if details_displayed:
+                    details_component.wait_for_absent_or_hidden()
+                else:
+                    details_component.wait_for_visible()
         return item_component
 
     def history_panel_ensure_showing_item_details(self, hid):
@@ -1621,31 +1666,33 @@ class NavigatesGalaxy(HasDriver):
         return element
 
     def set_history_annotation(self, annotation, clear_text=False):
-        self.ensure_history_annotation_area_displayed()
         history_panel = self.components.history_panel
 
         if self.is_beta_history():
-            editable = history_panel.annotation_editable_text_beta
-            edit = history_panel.annotation_edit_beta
+            toggle = self.beta_history_element("editor toggle")
+            toggle.wait_for_and_click()
+            annotation_input = self.beta_history_element("annotation input").wait_for_visible()
+            if clear_text:
+                annotation_input.clear()
+            annotation_input.send_keys(annotation)
+            save_button = self.beta_history_element("editor save button")
+            save_button.wait_for_and_click()
         else:
+            self.ensure_history_annotation_area_displayed()
             editable = history_panel.annotation_editable_text
             edit = history_panel.annotation_edit
 
-        editable.wait_for_and_click()
-        edit_el = edit.wait_for_and_click()
-        if clear_text:
-            # previously this was just edit_el.clear() but
-            # .clear() doesn't work with beta history panel
-            action_chains = self.action_chains()
-            for _ in range(40):
-                action_chains.send_keys(Keys.BACKSPACE)
-            action_chains.perform()
-        edit_el.send_keys(annotation)
+            editable.wait_for_and_click()
+            edit_el = edit.wait_for_and_click()
+            if clear_text:
+                # previously this was just edit_el.clear() but
+                # .clear() doesn't work with beta history panel
+                action_chains = self.action_chains()
+                for _ in range(40):
+                    action_chains.send_keys(Keys.BACKSPACE)
+                action_chains.perform()
+            edit_el.send_keys(annotation)
 
-        if self.is_beta_history():
-            # no button, just click away
-            self.click_center()
-        else:
             history_panel.annotation_done.wait_for_and_click()
 
     def ensure_history_annotation_area_displayed(self):
@@ -1691,6 +1738,21 @@ class NavigatesGalaxy(HasDriver):
     def snapshot(self, description):
         """Test case subclass overrides this to provide detailed logging."""
 
+    def open_history_editor(self, scope=".beta.history"):
+        if self.is_beta_history():
+            panel = self.components.history_panel.editor.selector(scope=scope)
+            toggle = panel.toggle
+            toggle.wait_for_and_click()
+            editor = panel.form
+            editor.wait_for_present()
+
+    def close_history_editor(self, scope=".beta.history"):
+        if self.is_beta_history():
+            toggle = self.components.history_panel.edit_toggle
+            toggle.wait_for_and_click()
+            editor = self.components.history_panel.editor.selector(scope=scope)
+            self.assert_absent_or_hidden(editor)
+
 
 class NotLoggedInException(TimeoutException):
 
diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml
index 4f766f55b91..aa50651feb5 100644
--- a/lib/galaxy/selenium/navigation.yml
+++ b/lib/galaxy/selenium/navigation.yml
@@ -15,6 +15,7 @@ _:  # global stuff
     left_panel_collapse: '#left > .unified-panel-footer > .panel-collapse'
     right_panel_drag: '#right > .unified-panel-footer > .drag'
     right_panel_collapse: '#right > .unified-panel-footer > .panel-collapse'
+    by_attribute: '${scope} [${name}="${value}"]'
 
   messages:
     selectors:
@@ -116,7 +117,6 @@ history_panel:
       # These now appear other places :_( - e.g. in the invocation view so we'll try
       # prefixing ids with #current-history-panel but obviously we need to switch to classes or data
       _: '#current-history-panel #${history_content_type}-${id}'
-      by_state_and_hid: '.beta.history .listing [data-hid="${hid}"][data-state="${state}"]'
 
       title: '${_} .content-title'
       hid: '${_} .hid'
@@ -144,6 +144,58 @@ history_panel:
       rerun_button: '${_} .icon-btn.rerun-btn'
       nametags: '${_} .nametags .badge-tags'
 
+  # beta content item
+  content_item:
+    selectors:
+      # specific rows selectable with attributes via suffix
+      _: '.beta.history .content-item${suffix}'
+      title: '${_} .content-title'
+      details: '${_} .details'
+      summary: '${_} .summary'
+
+      hid: '${_} .hid'
+      name: '${_} .name'
+      title_button_area: '${_} .primary-actions'
+      primary_action_buttons: '${_} .actions .left'
+      secondary_action_buttons: '${_} .actions .right'
+      blurb: '${_} .blurb .value'
+      dbkey: '${_} .dbkey .value'
+      info: '${_} .info .value'
+      peek: '${_} .dataset-peek'
+      toolhelp_title: '${_} .toolhelp strong'
+
+      # Title buttons...
+      display_button: '${_} .icon-btn.display-btn'
+      edit_button: '${_} .icon-btn.edit-btn'
+      delete_button: '${_} .icon-btn.delete-btn'
+
+      # Action buttons...
+      download_button: '${_} .icon-btn.download-btn'
+      info_button: '${_} .icon-btn.params-btn'
+      dbkey_button: '${_} .fa.fa-question'
+      rerun_button: '${_} .icon-btn.rerun-btn'
+      nametags: '${_} .nametags .badge-tags'
+
+  # re-usable history editor, scoped for use in different layout scenarios (multi, etc.)
+  editor:
+    selectors:
+      _: '${scope} [data-description="edit details"]'
+      toggle: '${_} [data-description="editor toggle"]'
+      form: '${_} [data-description="edit form"]'
+      name_input: '${_} [data-description="name input"]'
+      annotation_input: '${_} [data-description="annotation input"]'
+      tags_input: '${_} .tags input'
+      save_button: '${_} [data-description="editor save button"]'
+
+  # beta history tags
+  tag_editor:
+    selectors:
+      _: '${scope} .history-details .tags'
+      toggle: '${_} .toggle-link'
+      display: '${_} .tag-area .tag-name'
+      input: '${_} input'
+      tag_close_btn: '${_} .ti-icon-close'
+
   multi_operations:
     selectors:
       show_button: '.show-history-content-selectors-btn'
@@ -159,6 +211,8 @@ history_panel:
   collection_view:
     selectors:
       _: '.dataset-collection-panel'
+      nav_menu: '[data-description="collection breadcrumbs menu"]'
+      back_button: '[data-description="collection breadcrumbs menu"] :last'
       back: '.navigation .back'
       title: '.dataset-collection-panel .controls .title .editable-text'
       title_input: '.dataset-collection-panel .controls .title input'
@@ -168,8 +222,7 @@ history_panel:
       list_items: '.dataset-collection-panel .list-items .list-item'
 
   selectors:
-    beta: 'section.history'
-    by_attribute: '.beta.history [${name}="${value}"]'
+    beta: '.beta.history'
     _: '#current-history-panel'
     search: '#current-history-panel input.search-query'
     refresh_button: '.history-refresh-button'
@@ -186,6 +239,7 @@ history_panel:
     tag_area_input: '.history-details .tags-display .tags-input input'
     tag_close_btn: '.tags-display .ti-icon-close'
     tags: 'li.ti-tag.ti-valid .tag-name'
+
     annotation_icon: '.actions .history-annotate-btn'
     annotation_area: '.history-details .history-annotation'
     annotation_editable_text: '.history-details .history-annotation .annotation.editable-text'
diff --git a/lib/galaxy_test/selenium/test_collection_builders.py b/lib/galaxy_test/selenium/test_collection_builders.py
index 0bf5ee41959..6abd33a4493 100644
--- a/lib/galaxy_test/selenium/test_collection_builders.py
+++ b/lib/galaxy_test/selenium/test_collection_builders.py
@@ -11,66 +11,66 @@ class CollectionBuildersTestCase(SeleniumTestCase):
     @selenium_test
     def test_build_list_simple_hidden(self):
         self.perform_upload(self.get_filename("1.fasta"))
-        self.history_panel_wait_for_hid_ok(1)
-        self.history_panel_multi_operations_show()
-        self.history_panel_muli_operation_select_hid(1)
-        self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
+
+        self._wait_for_and_select([1])
+
+        if self.is_beta_history():
+            self._collection_dropdown("build list")
+        else:
+            self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
 
         self.collection_builder_set_name("my cool list")
         self.screenshot("collection_builder_list")
         self.collection_builder_create()
-        self._collection_builder_wait_for_hid_ok(2)
+        self._wait_for_hid_visible(2)
 
     @selenium_test
     def test_build_list_and_show_items(self):
         self.perform_upload(self.get_filename("1.fasta"))
-        self.history_panel_wait_for_hid_ok(1)
-        self.history_panel_multi_operations_show()
-        self.history_panel_muli_operation_select_hid(1)
-        self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
+
+        self._wait_for_and_select([1])
+
+        if self.is_beta_history():
+            self._collection_dropdown("build list")
+        else:
+            self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
 
         # this toggles the checkbox to not hide originals
         self.collection_builder_hide_originals()
         self.collection_builder_set_name("my cool list")
-
         self.collection_builder_create()
-        self._collection_builder_wait_for_hid_ok(3)
+        self.home()  # this shouldn't be necessary, and it isn't with a real browser.
+        self._wait_for_hid_visible(3)
 
     @selenium_test
     def test_build_pair_simple_hidden(self):
         self.perform_upload(self.get_filename("1.tabular"))
         self.perform_upload(self.get_filename("2.tabular"))
-        self._wait_for_hid_visible(1)
-        self._wait_for_hid_visible(2)
-        self.history_panel_multi_operations_show()
-        self.history_panel_muli_operation_select_hid(1)
-        self.history_panel_muli_operation_select_hid(2)
-        self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_pair)
+
+        self._wait_for_and_select([1, 2])
+
+        if self.is_beta_history():
+            self._collection_dropdown("build pair")
+        else:
+            self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_pair)
+
         self.collection_builder_set_name("my awesome pair")
         self.screenshot("collection_builder_pair")
         self.collection_builder_create()
-        self._collection_builder_wait_for_hid_ok(3)
-
-    def _collection_builder_wait_for_hid_ok(self, hid):
-        # a hacked up version of super().history_panel_wait_for_hid_ok for beta history panel not refreshing correctly.
-        allowed_force_refreshes = 0
-        if self.use_beta_history:
-            # so collection builder no longer forces refreshes of history panel.
-            # Why doesn't the panel pick it up automatically - an sqlite limitation?
-            # Or maybe refresh doesn't work if things are selected in the panel?
-            allowed_force_refreshes = 2
-        self.history_panel_wait_for_hid_ok(hid, allowed_force_refreshes=allowed_force_refreshes)
+        self._wait_for_hid_visible(3)
 
     @selenium_test
     def test_build_paired_list_simple(self):
         self.perform_upload(self.get_filename("1.tabular"))
         self.perform_upload(self.get_filename("2.tabular"))
-        self._wait_for_hid_visible(1)
-        self._wait_for_hid_visible(2)
-        self.history_panel_multi_operations_show()
-        self.history_panel_muli_operation_select_hid(1)
-        self.history_panel_muli_operation_select_hid(2)
-        self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list_pairs)
+
+        # self.home()
+        self._wait_for_and_select([1, 2])
+
+        if self.is_beta_history():
+            self._collection_dropdown("build list of pairs")
+        else:
+            self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list_pairs)
 
         self.collection_builder_clear_filters()
         self.collection_builder_click_paired_item("forward", 0)
@@ -78,20 +78,30 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_set_name("my awesome paired list")
         self.screenshot("collection_builder_paired_list")
         self.collection_builder_create()
-        self._collection_builder_wait_for_hid_ok(3)
-        self.history_panel_wait_for_hid_hidden(1)
-        self.history_panel_wait_for_hid_hidden(2)
+
+        if self.is_beta_history():
+            self._wait_for_hid_visible(3)
+            # switch to hidden filters to see the hidden datasets appear
+            self._show_hidden_content()
+            self._wait_for_hid_visible(1)
+            self._wait_for_hid_visible(2)
+        else:
+            self.history_panel_wait_for_hid_ok(3)
+            self.history_panel_wait_for_hid_hidden(1)
+            self.history_panel_wait_for_hid_hidden(2)
 
     @selenium_test
     def test_build_paired_list_show_original(self):
         self.perform_upload(self.get_filename("1.tabular"))
         self.perform_upload(self.get_filename("2.tabular"))
-        self._wait_for_hid_visible(1)
-        self._wait_for_hid_visible(2)
-        self.history_panel_multi_operations_show()
-        self.history_panel_muli_operation_select_hid(1)
-        self.history_panel_muli_operation_select_hid(2)
-        self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list_pairs)
+
+        # self.home()  # should not be necessary, chromedriver fails without it though
+        self._wait_for_and_select([1, 2])
+
+        if self.is_beta_history():
+            self._collection_dropdown("build list of pairs")
+        else:
+            self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list_pairs)
 
         self.wait_for_and_click(self.navigation.collection_builders.selectors.clear_filters)
 
@@ -103,31 +113,83 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         second_dataset_reverse = reverse_column.find_elements_by_css_selector("li")[1]
         second_dataset_reverse.click()
 
-        # this toggles the checkbox to not hide originals
         self.collection_builder_hide_originals()
         self.collection_builder_set_name("my awesome paired list")
-
         self.collection_builder_create()
-        self._collection_builder_wait_for_hid_ok(5)
-        self.history_panel_refresh_click()
-        self.history_panel_wait_for_hid_ok(1)
-        self.history_panel_wait_for_hid_ok(2)
-        self.history_panel_wait_for_hid_hidden(3)
-        self.history_panel_wait_for_hid_hidden(4)
+
+        if self.is_beta_history():
+            self._wait_for_hid_visible(5)
+            self._wait_for_hid_visible(1)
+            self._wait_for_hid_visible(2)
+            self._show_hidden_content()
+            self._wait_for_hid_visible(3)
+            self._wait_for_hid_visible(4)
+        else:
+            self.history_panel_wait_for_hid_ok(5)
+            self.history_panel_refresh_click()
+            self.history_panel_wait_for_hid_ok(1)
+            self.history_panel_wait_for_hid_ok(2)
+            self.history_panel_wait_for_hid_hidden(3)
+            self.history_panel_wait_for_hid_hidden(4)
 
     @selenium_test
     def test_build_simple_list_via_rules_hidden(self):
         self.perform_upload(self.get_filename("1.fasta"))
-        self.history_panel_wait_for_hid_ok(1, allowed_force_refreshes=1)
-        self.history_panel_multi_operations_show()
-        self.history_panel_muli_operation_select_hid(1)
-        self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_from_rules)
+
+        self._wait_for_and_select([1])
+
+        if self.is_beta_history():
+            self._collection_dropdown("build collection from rules")
+        else:
+            self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_from_rules)
 
         self.collection_builder_set_name("my cool list")
         self.screenshot("collection_builder_rules_list")
         self.collection_builder_create()
-        self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1)
-        self.history_panel_wait_for_hid_hidden(1)
 
-    def _wait_for_hid_visible(self, hid):
-        self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=1)
+        self._wait_for_hid_visible(2)
+
+        if self.is_beta_history():
+            self._show_hidden_content()
+            self._wait_for_hid_visible(1)
+        else:
+            self.history_panel_wait_for_hid_hidden(1)
+
+    def _wait_for_hid_visible(self, hid, state="ok"):
+        if self.is_beta_history():
+            # takes a little while for these things to upload and end up in the history
+            timeout = self.wait_length(self.wait_types.JOB_COMPLETION)
+            row_selector = self.content_item_by_attributes(hid=hid, state=state)
+            self.wait_for_visible(row_selector, timeout=timeout)
+        else:
+            self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=1)
+
+    def _collection_dropdown(self, option_description):
+        return self.use_bootstrap_dropdown(option=option_description, menu="new content menu")
+
+    def _wait_for_and_select(self, hids):
+        """
+        Waits for uploads to pass through queued, running, ok. Not all the states are not guaranteed
+        depending on how fast the upload goes compared to the history polling updates, it might just
+        skip to the end for a really fast upload
+        """
+
+        if self.is_beta_history():
+            for hid in hids:
+                timeout = self.wait_length(self.wait_types.JOB_COMPLETION)
+                row_selector = self.content_item_by_attributes(hid=hid, state="ok")
+                row = self.wait_for_present(row_selector, timeout=timeout)
+                row.send_keys(" ")
+        else:
+            for hid in hids:
+                self.history_panel_wait_for_hid_ok(hid)
+            self.history_panel_multi_operations_show()
+            for hid in hids:
+                self.history_panel_muli_operation_select_hid(hid)
+
+    def _show_hidden_content(self):
+        """Switches the beta hidden filter toggle on """
+        self.sleep_for(self.wait_types.UX_RENDER)
+        self.beta_history_element("content filter toggle").wait_for_and_click()
+        self.beta_history_element("show hidden filter toggle").wait_for_and_click()
+        self.sleep_for(self.wait_types.UX_RENDER)
diff --git a/lib/galaxy_test/selenium/test_history_copy_elements.py b/lib/galaxy_test/selenium/test_history_copy_elements.py
index 43f1be123e3..3ccc6864c6c 100644
--- a/lib/galaxy_test/selenium/test_history_copy_elements.py
+++ b/lib/galaxy_test/selenium/test_history_copy_elements.py
@@ -18,7 +18,8 @@ class HistoryCopyElementsTestCase(SeleniumTestCase):
         failed_collection = failed_response["implicit_collections"][0]
         failed_hid = failed_collection["hid"]
 
-        self.home()
+        if not self.is_beta_history():
+            self.home()
 
         self.history_panel_wait_for_hid_state(input_hid, "ok")
         self.history_panel_wait_for_hid_state(failed_hid, "error")
@@ -34,6 +35,8 @@ class HistoryCopyElementsTestCase(SeleniumTestCase):
             self.sleep_for(self.wait_types.UX_TRANSITION)
             self.components.history_copy_elements.done_link.wait_for_and_click()
 
+        # I don't know why this sleep is necessary but it seems to be
+        self.sleep_for(self.wait_types.UX_RENDER)
         # Okay copied first
         self.history_panel_wait_for_hid_state(5, "ok")
         # Then 4 datasets and then the failed collection (this was six when coming from the original history)
diff --git a/lib/galaxy_test/selenium/test_history_export.py b/lib/galaxy_test/selenium/test_history_export.py
index d55d0105ffd..0a6b6caa678 100644
--- a/lib/galaxy_test/selenium/test_history_export.py
+++ b/lib/galaxy_test/selenium/test_history_export.py
@@ -9,15 +9,15 @@ class HistoryExportTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_history_export(self):
-        gx_selenium_context = self
-        gx_selenium_context.perform_upload_of_pasted_content("my cool content")
-        gx_selenium_context.history_panel_wait_for_hid_ok(1)
-        gx_selenium_context.click_history_options()
-        gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
-        history_export = gx_selenium_context.components.history_export
+        self.perform_upload_of_pasted_content("my cool content")
+        self.history_panel_wait_for_hid_ok(1)
+
+        self.select_export_to_file()
+
+        history_export = self.components.history_export
         history_export.export_link.wait_for_and_click()
         history_export.running.wait_for_visible()
-        history_export.running.wait_for_absent(wait_type=gx_selenium_context.wait_types.JOB_COMPLETION)
+        history_export.running.wait_for_absent(wait_type=self.wait_types.JOB_COMPLETION)
         history_export.generated_export_link.wait_for_visible()
         history_export.copy_export_link.wait_for_visible()
         history_export.job_table.assert_absent_or_hidden()
@@ -26,19 +26,24 @@ class HistoryExportTestCase(SeleniumTestCase):
         history_export.job_table_ok.wait_for_and_click()
         history_export.job_table.wait_for_absent()
 
-        gx_selenium_context.click_history_options()
-        gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
+        self.select_export_to_file()
 
         # this time the exported link is still there
         history_export.generated_export_link.wait_for_visible()
         history_export.export_link.assert_absent()
 
-        gx_selenium_context.perform_upload_of_pasted_content("my cool content part 2")
-        gx_selenium_context.history_panel_wait_for_hid_ok(2)
+        self.perform_upload_of_pasted_content("my cool content part 2")
+        self.history_panel_wait_for_hid_ok(2)
 
-        gx_selenium_context.click_history_options()
-        gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
+        self.select_export_to_file()
 
         # now we have a generated link and a link to update to the newest export
         history_export.generated_export_link.wait_for_visible()
         history_export.export_link.wait_for_visible()
+
+    def select_export_to_file(self):
+        if self.is_beta_history():
+            self.use_bootstrap_dropdown(option="export to file", menu="history menu")
+        else:
+            self.click_history_options()
+            self.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
diff --git a/lib/galaxy_test/selenium/test_history_panel.py b/lib/galaxy_test/selenium/test_history_panel.py
index 01206979646..42a8af26b4f 100644
--- a/lib/galaxy_test/selenium/test_history_panel.py
+++ b/lib/galaxy_test/selenium/test_history_panel.py
@@ -1,11 +1,15 @@
 import pytest
 
+from galaxy.selenium.navigates_galaxy import (
+    edit_details
+)
 from .framework import (
     retry_assertion_during_transitions,
     selenium_test,
     SeleniumTestCase,
 )
 
+
 NEW_HISTORY_NAME = "New History Name"
 
 
@@ -17,30 +21,35 @@ class HistoryPanelTestCase(SeleniumTestCase):
     def test_history_panel_landing_state(self):
         self.assert_initial_history_panel_state_correct()
 
-        tag_icon_selector = self.navigation.history_panel.selectors.tag_icon
-        annotation_icon_selector = self.navigation.history_panel.selectors.annotation_icon
-
-        self.wait_for_visible(tag_icon_selector)
-        self.wait_for_visible(annotation_icon_selector)
-
-        name_element = self.history_panel_name_element()
-        self.assert_tooltip_text(name_element, self.navigation.history_panel.text.tooltip_name)
+        if self.is_beta_history():
+            # look for the editor icon
+            editor = self.components.history_panel.editor.selector(scope=".beta.history")
+            toggle = editor.toggle
+            toggle.wait_for_visible()
+        else:
+            tag_icon_selector = self.navigation.history_panel.selectors.tag_icon
+            annotation_icon_selector = self.navigation.history_panel.selectors.annotation_icon
+            self.wait_for_visible(tag_icon_selector)
+            self.wait_for_visible(annotation_icon_selector)
+            name_element = self.history_panel_name_element()
+            self.assert_tooltip_text(name_element, self.navigation.history_panel.text.tooltip_name)
 
     @selenium_test
     def test_history_panel_rename(self):
-        editable_text_input_element = self.history_panel_click_to_rename()
         if self.is_beta_history():
-            editable_text_input_element.clear()
-        editable_text_input_element.send_keys(NEW_HISTORY_NAME)
-        self.send_enter(editable_text_input_element)
-
-        self.assert_name_changed()
+            self.history_panel_rename(NEW_HISTORY_NAME)
+            self.assert_name_changed()
+        else:
+            editable_text_input_element = self.history_panel_click_to_rename()
+            editable_text_input_element.send_keys(NEW_HISTORY_NAME)
+            self.send_enter(editable_text_input_element)
+            self.assert_name_changed()
 
     @selenium_test
     def test_history_rename_confirm_with_click(self):
-        editable_text_input_element = self.history_panel_click_to_rename()
         if self.is_beta_history():
-            editable_text_input_element.clear()
+            raise pytest.skip("Beta History Panel has explicit editing toggle mode, so can not click off to the side to save")
+        editable_text_input_element = self.history_panel_name_input()
         editable_text_input_element.send_keys(NEW_HISTORY_NAME)
         self.click_center()
         self.assert_absent(self.navigation.history_panel.selectors.name_edit_input)
@@ -48,45 +57,53 @@ class HistoryPanelTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_history_rename_cancel_with_escape(self):
-        editable_text_input_element = self.history_panel_click_to_rename()
-        if self.is_beta_history():
-            editable_text_input_element.clear()
+        self.open_history_editor()
+        editable_text_input_element = self.history_panel_name_input()
         editable_text_input_element.send_keys(NEW_HISTORY_NAME)
         self.send_escape(editable_text_input_element)
-        self.assert_absent(self.navigation.history_panel.selectors.name_edit_input)
+        if self.is_beta_history():
+            self.components.history_panel.editor.selector(scope=".beta.history").form.wait_for_absent_or_hidden()
+        else:
+            self.assert_absent(self.navigation.history_panel.selectors.name_edit_input)
         assert NEW_HISTORY_NAME not in self.history_panel_name()
 
     @selenium_test
+    @edit_details
     def test_history_tags_and_annotations_buttons(self):
-        tag_icon_selector = self.navigation.history_panel.selectors.tag_icon
-        annotation_icon_selector = self.navigation.history_panel.selectors.annotation_icon
+        if self.is_beta_history():
+            history_editor = self.components.history_panel.editor.selector(scope=".beta.history")
+            history_editor.annotation_input.wait_for_clickable()
+            history_editor.tags_input.wait_for_clickable()
+        else:
+            tag_icon_selector = self.navigation.history_panel.selectors.tag_icon
+            annotation_icon_selector = self.navigation.history_panel.selectors.annotation_icon
 
-        tag_area_selector = self.navigation.history_panel.selectors.tag_area
-        annotation_area_selector = self.navigation.history_panel.selectors.annotation_area
+            tag_area_selector = self.navigation.history_panel.selectors.tag_area
+            annotation_area_selector = self.navigation.history_panel.selectors.annotation_area
 
-        tag_icon = self.wait_for_clickable(tag_icon_selector)
-        annon_icon = self.wait_for_clickable(annotation_icon_selector)
+            tag_icon = self.wait_for_clickable(tag_icon_selector)
+            annon_icon = self.wait_for_clickable(annotation_icon_selector)
 
-        self.assert_absent_or_hidden(tag_area_selector)
-        self.assert_absent_or_hidden(annotation_area_selector)
+            self.assert_absent_or_hidden(tag_area_selector)
+            self.assert_absent_or_hidden(annotation_area_selector)
 
-        tag_icon.click()
+            tag_icon.click()
 
-        self.wait_for_visible(tag_area_selector)
-        self.assert_absent_or_hidden(annotation_area_selector)
+            self.wait_for_visible(tag_area_selector)
+            self.assert_absent_or_hidden(annotation_area_selector)
 
-        tag_icon.click()
-        self.sleep_for(self.wait_types.UX_TRANSITION)
-        annon_icon.click()
+            tag_icon.click()
+            self.sleep_for(self.wait_types.UX_TRANSITION)
+            annon_icon.click()
 
-        self.wait_for_visible(annotation_area_selector)
-        self.assert_absent_or_hidden(tag_area_selector)
+            self.wait_for_visible(annotation_area_selector)
+            self.assert_absent_or_hidden(tag_area_selector)
 
-        annon_icon.click()
-        self.sleep_for(self.wait_types.UX_TRANSITION)
+            annon_icon.click()
+            self.sleep_for(self.wait_types.UX_TRANSITION)
 
-        self.assert_absent_or_hidden(tag_area_selector)
-        self.assert_absent_or_hidden(annotation_area_selector)
+            self.assert_absent_or_hidden(tag_area_selector)
+            self.assert_absent_or_hidden(annotation_area_selector)
 
     @selenium_test
     def test_history_panel_annotations_change(self):
@@ -97,7 +114,7 @@ class HistoryPanelTestCase(SeleniumTestCase):
                                       is_equal=True):
 
             if self.is_beta_history():
-                text_component = history_panel.annotation_editable_text_beta
+                text_component = self.beta_history_element("history annotation")
             else:
                 text_component = history_panel.annotation_editable_text
             current_annotation = text_component.wait_for_visible()
@@ -133,37 +150,38 @@ class HistoryPanelTestCase(SeleniumTestCase):
     @selenium_test
     def test_history_panel_tags_change(self):
 
-        def add_tags(size):
+        def create_tags(size):
             history_panel_tags = list()
-
             for i in range(size):
                 history_panel_tags.append(self._get_random_name(prefix="arbitrary_tag_%s_") % i)
-
-            self.history_panel_add_tags(history_panel_tags)
             return history_panel_tags
 
+        def add_tags(tags_size):
+            tags = create_tags(tags_size)
+            self.history_panel_add_tags(tags)
+            return tags
+
+        # check tags against list
         def assert_current_tags(expected_tags):
-            current_tags = self.components.history_panel.tags
-            current_tags.wait_for_visible()
-            assert [tag.text for tag in
-                    current_tags.all()] == expected_tags, f"tags [{current_tags}] are not the same as expected [{expected_tags}]"
+            if self.is_beta_history():
+                current_tags = self.open_tags()
+            else:
+                current_tags = self.components.history_panel.tags
+            errmsg = f"tags [{current_tags}] are not the same as expected [{expected_tags}]"
+            assert [tag.text for tag in current_tags.all()] == expected_tags, errmsg
 
-        def clear_tags(expected_tags_size):
+        # looks like this is intended to check if the tag editor is open
+        def assert_no_tags():
+            if self.is_beta_history():
+                tags_component = self.components.history_panel.tag_editor.selector(scope=".beta.history")
+                tags_component.display.assert_absent_or_hidden()
+            else:
+                self.components.history_panel.tag_area.assert_absent_or_hidden()
 
-            close_tag_buttons = self.components.history_panel.tag_close_btn.all()
-            current_tags_size = len(close_tag_buttons)
-
-            assert expected_tags_size == current_tags_size, "there are more tags than expected! current {}, expected {}".format(
-                current_tags_size, expected_tags_size)
-            for close_btn in reversed(close_tag_buttons):
-                close_btn.click()
-                self.sleep_for(self.wait_types.UX_RENDER)
-
-        tags_size = 5
-
-        self.components.history_panel.tag_area.assert_absent_or_hidden()
+        assert_no_tags()
 
         # add new tags to empty tags area
+        tags_size = 5
         tags = add_tags(tags_size)
         assert_current_tags(tags)
 
@@ -173,8 +191,39 @@ class HistoryPanelTestCase(SeleniumTestCase):
 
         # delete all tags
         expected_tags_len = len(tags)
-        clear_tags(expected_tags_len)
-        self.components.history_panel.tags.assert_absent_or_hidden()
+        self.clear_tags(expected_tags_len)
+
+        assert_no_tags()
+
+    # after about 5 tags, a toggle link shows up and you have to click it to see the full list
+    def open_tags(self):
+        tags_component = self.components.history_panel.tag_editor.selector(scope=".beta.history")
+        if not tags_component.toggle.is_absent:
+            tags_component.toggle.wait_for_and_click()
+        tags_component.display.wait_for_visible()
+        return tags_component.display
+
+    @edit_details
+    def clear_tags(self, expected_tags_size):
+        if self.is_beta_history():
+            self.open_tags()
+            tags = self.components.history_panel.tag_editor.selector(scope=".beta.history")
+            close_tag_buttons = tags.tag_close_btn.all()
+        else:
+            close_tag_buttons = self.components.history_panel.tag_close_btn.all()
+
+        current_tags_size = len(close_tag_buttons)
+
+        errmsg = "there are more tags than expected! current {}, expected {}".format(current_tags_size, expected_tags_size)
+        assert expected_tags_size == current_tags_size, errmsg
+
+        for close_btn in reversed(close_tag_buttons):
+            close_btn.click()
+            self.sleep_for(self.wait_types.UX_RENDER)
+
+        if not self.is_beta_history():
+            self.components.history_panel.tag_icon.wait_for_and_click()
+            self.sleep_for(self.wait_types.UX_RENDER)
 
     @selenium_test
     def test_refresh_preserves_state(self):
diff --git a/lib/galaxy_test/selenium/test_history_panel_collections.py b/lib/galaxy_test/selenium/test_history_panel_collections.py
index b69378e947d..3658c63eae5 100644
--- a/lib/galaxy_test/selenium/test_history_panel_collections.py
+++ b/lib/galaxy_test/selenium/test_history_panel_collections.py
@@ -1,5 +1,11 @@
 import time
 
+import pytest
+
+from galaxy.selenium.navigates_galaxy import (
+    edit_details,
+    WAIT_TYPES
+)
 from galaxy_test.base.api_asserts import assert_status_code_is
 from galaxy_test.base.populators import flakey
 from .framework import (
@@ -30,10 +36,12 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
             assert_ok=True,
         )
         ok_hid = ok_response["implicit_collections"][0]["hid"]
-        # sleep really shouldn't be needed :(
-        time.sleep(1)
 
-        self.home()
+        if not self.is_beta_history():
+            # sleep really shouldn't be needed :(
+            time.sleep(1)
+
+            self.home()
 
         self.history_panel_wait_for_hid_state(input_hid, "ok")
         self.history_panel_wait_for_hid_state(failed_hid, "error")
@@ -84,6 +92,7 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
             history_id
         )
         ok_hid = ok_response["output_collections"][0]["hid"]
+        assert ok_hid > 0
 
         failed_response = self.dataset_populator.run_tool(
             "collection_creates_dynamic_nested_fail",
@@ -91,14 +100,40 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
             history_id,
         )
         failed_hid = failed_response["output_collections"][0]["hid"]
+        assert failed_hid > 0
 
-        # sleep really shouldn't be needed :(
-        time.sleep(1)
+        if self.is_beta_history():
+            # if we do not reload the page after the initialization, then we have to wait for the
+            # server to update history.update_time so that polling returns a change set. If the
+            # chron-job-ish mechanism they put in place a few weeks back does not work in a timely
+            # fashion, the polling will never render curremt results before the test times out
+            self.sleep_for(WAIT_TYPES.HISTORY_POLL)
 
-        self.home()
+            # Note: it might be worth making something similar to history_panel_wait_for_hid_state
+            # but with a poll count parameter instead of a page-refresh parameter, then you could
+            # get quantitative testing results as well as qualitative (test_xyz required 25 polls
+            # to complete, etc)
+
+            try:
+                # this is the most likely order, but it might go faster if polling responds quickly
+                self.content_item_by_attributes(hid=ok_hid).wait_for_present()
+                self.content_item_by_attributes(hid=failed_hid).wait_for_present()
+                self.content_item_by_attributes(hid=failed_hid, state="running").wait_for_present()
+                self.content_item_by_attributes(hid=ok_hid, state="ok").wait_for_present()
+                self.content_item_by_attributes(hid=failed_hid, state="error").wait_for_present()
+            finally:
+                self.sleep_for(WAIT_TYPES.HISTORY_POLL)
+                self.content_item_by_attributes(hid=ok_hid, state="ok").wait_for_present()
+                self.content_item_by_attributes(hid=failed_hid, state="error").wait_for_present()
+        else:
+            # sleep really shouldn't be needed :(
+            time.sleep(1)
+
+            self.home()
+
+            self.history_panel_wait_for_hid_state(ok_hid, "ok")
+            self.history_panel_wait_for_hid_state(failed_hid, "error")
 
-        self.history_panel_wait_for_hid_state(ok_hid, "ok")
-        self.history_panel_wait_for_hid_state(failed_hid, "error")
         self.screenshot("history_panel_collections_state_terminal")
 
     @selenium_test
@@ -114,41 +149,80 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
             history_id,
             assert_ok=False,
         )
+        assert_status_code_is(running_response, 200)
+
+        payload = running_response.json()
+        assert payload["output_collections"]
+        assert payload["jobs"]
+        assert len(payload["jobs"]) > 0
+        assert len(["output_collections"]) > 0
+
+        running_hid = payload["output_collections"][0]["hid"]
+        assert running_hid
+
         try:
-            assert_status_code_is(running_response, 200)
-            running_hid = running_response.json()["output_collections"][0]["hid"]
-
-            # sleep really shouldn't be needed :(
-            time.sleep(1)
-
-            self.home()
+            if not self.is_beta_history():
+                # sleep really shouldn't be needed :(
+                time.sleep(1)
+                self.home()
             self.history_panel_wait_for_hid_state(running_hid, "running")
             self.screenshot("history_panel_collections_state_running")
         finally:
-            for job in running_response.json()["jobs"]:
+            for job in payload["jobs"]:
                 self.dataset_populator.cancel_job(job["id"])
 
     @selenium_test
-    def test_collection_operations(self):
-        # Test back and rename.
+    def test_back_to_history_button(self):
         input_collection = self._populated_paired_and_wait_for_it()
         collection_hid = input_collection["hid"]
-        collection_view = self._click_and_wait_for_collection_view(collection_hid)
-
-        collection_view.back.wait_for_and_click()
+        self.history_panel_wait_for_hid_state(collection_hid, "ok")
+        self._click_and_wait_for_collection_view(collection_hid)
+        self._back_to_history()
         self.history_panel_wait_for_hid_state(collection_hid, "ok")
 
-        self._click_and_wait_for_collection_view(collection_hid)
+    def _back_to_history(self):
+        if self.is_beta_history():
+            # it's either a single back button or a dropdown with an option depending
+            # on how deeply nested the collection is
+            menu = self.beta_history_element("collection breadcrumbs menu")
+            if not menu.is_absent:
+                # menu.is_displayed results in an error but I don't understand why
+                menu.wait_for_and_click()
+            back = self.beta_history_element("back to history")
+        else:
+            back = self.components.history_panel.collection_view.back
+        back.wait_for_and_click()
+        self.wait(WAIT_TYPES.UX_RENDER)
 
-        title_element = collection_view.title.wait_for_visible()
-        assert title_element.text == input_collection["name"]
-        title_element.click()
-        title_rename_element = collection_view.title_input.wait_for_visible()
-        self.screenshot("history_panel_collection_view_rename")
-        title_rename_element.send_keys("My New Name")
-        self.send_enter(title_rename_element)
-        title_element = collection_view.title.wait_for_visible()
-        assert title_element.text == "My New Name"
+    @selenium_test
+    def test_rename_collection(self):
+        input_collection = self._populated_paired_and_wait_for_it()
+        collection_hid = input_collection["hid"]
+        self.history_panel_wait_for_hid_state(collection_hid, "ok")
+        collection_view = self._click_and_wait_for_collection_view(collection_hid)
+
+        new_name = "My New Name"
+
+        if self.is_beta_history():
+            self._rename_collection(new_name)
+            self.screenshot("history_panel_collection_view_rename_beta")
+            title_element = self.beta_history_element("collection name display").wait_for_present()
+        else:
+            title_element = collection_view.title.wait_for_visible()
+            assert title_element.text == input_collection["name"]
+            title_element.click()
+            title_rename_element = collection_view.title_input.wait_for_visible()
+            title_rename_element.send_keys(new_name)
+            self.send_enter(title_rename_element)
+            self.screenshot("history_panel_collection_view_rename")
+            title_element = collection_view.title.wait_for_visible()
+        assert title_element.text == new_name
+
+    @edit_details
+    def _rename_collection(self, new_name):
+        title_element = self.beta_history_element("name input").wait_for_clickable()
+        title_element.clear()
+        title_element.send_keys(new_name)
 
     @selenium_test
     def test_name_tags_display(self):
@@ -157,15 +231,18 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
         collection_hid = input_collection["hid"]
         collection_view = self._click_and_wait_for_collection_view(collection_hid)
 
-        # the space on the end of the parent_selector is important
-        self.tagging_add(["#moo"], parent_selector=".dataset-collection-panel .controls ")
-
-        self.screenshot("history_panel_collection_view_add_nametag")
-        collection_view.back.wait_for_and_click()
+        if self.is_beta_history():
+            self.history_panel_add_tags(["#moo"])
+            self.screenshot("history_panel_collection_view_add_nametag_beta")
+            self._back_to_history()
+        else:
+            # the space on the end of the parent_selector is important
+            self.tagging_add(["#moo"], parent_selector=".dataset-collection-panel .controls ")
+            self.screenshot("history_panel_collection_view_add_nametag")
+            collection_view.back.wait_for_and_click()
 
         self.history_panel_wait_for_hid_state(collection_hid, "ok")
         nametags = self.history_panel_item_get_nametags(collection_hid)
-
         assert nametags == ["moo"]
         self.screenshot("history_panel_collection_with_nametag")
 
@@ -175,29 +252,49 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
         collection_hid = input_collection["hid"]
         collection_view = self._click_and_wait_for_collection_view(collection_hid)
 
-        dataset_elements = collection_view.list_items.all()
+        if self.is_beta_history():
+            self.sleep_for(WAIT_TYPES.HISTORY_POLL)
+            dataset_elements = self.content_item_by_attributes().all()
+        else:
+            dataset_elements = collection_view.list_items.all()
+
         assert len(dataset_elements) == 2, dataset_elements
-        title_elements = [de.find_element_by_css_selector(".title .name").text for de in dataset_elements]
-        assert title_elements == ["forward", "reverse"]
+        titles = [de.find_element_by_css_selector(".title .name").text for de in dataset_elements]
+        assert titles == ["forward", "reverse"]
         self.screenshot("history_panel_collection_view_paired")
 
     @selenium_test
-    @flakey  # Fails only in Jenkins full suite - possibly due to #3782.
+    @flakey  # Fails only in Jenkins full suite - possibly due to #3782
     def test_list_display(self):
-        _, failed_collection = self._generate_partially_failed_collection_with_input()
+        input_collection, failed_collection = self._generate_partially_failed_collection_with_input()
+        input_collection_hid = input_collection['hid']
         failed_hid = failed_collection["hid"]
-        self.home()
-        self.history_panel_wait_for_hid_state(failed_hid, "error")
-        collection_view = self._click_and_wait_for_collection_view(failed_hid)
 
-        @retry_assertion_during_transitions
-        def check_four_datasets_shown():
-            dataset_elements = collection_view.list_items.all()
-            assert len(dataset_elements) == 4, dataset_elements
+        if self.is_beta_history():
+            self.content_item_by_attributes(hid=input_collection_hid).wait_for_present()
+            self.content_item_by_attributes(hid=failed_hid, state="error").wait_for_present()
+            self._click_and_wait_for_collection_view(failed_hid)
+            self.content_item_by_attributes(index=3).wait_for_present()
+
+            dataset_elements = self.content_item_by_attributes().all()
+            assert len(dataset_elements) == 4
             title_elements = [de.find_element_by_css_selector(".title .name").text for de in dataset_elements]
             assert title_elements == ["data1", "data2", "data3", "data4"]
 
-        check_four_datasets_shown()
+        else:
+            self.home()
+            self.history_panel_wait_for_hid_state(failed_hid, "error")
+            collection_view = self._click_and_wait_for_collection_view(failed_hid)
+
+            @retry_assertion_during_transitions
+            def check_four_datasets_shown():
+                dataset_elements = collection_view.list_items.all()
+                assert len(dataset_elements) == 4, dataset_elements
+                title_elements = [de.find_element_by_css_selector(".title .name").text for de in dataset_elements]
+                assert title_elements == ["data1", "data2", "data3", "data4"]
+
+            check_four_datasets_shown()
+
         self.screenshot("history_panel_collection_view_list")
 
     @selenium_test
@@ -222,6 +319,8 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_limiting_collection_rendering(self):
+        if self.is_beta_history():
+            raise pytest.skip("Beta history can scroll through collections of any length")
         history_id = self.current_history_id()
         collection = self.dataset_collection_populator.create_list_in_history(history_id, contents=["0", "1", "0", "1"]).json()
         collection_hid = collection["hid"]
@@ -246,14 +345,15 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
         self.history_panel_click_item_title(collection_hid)
 
         collection_view = self.components.history_panel.collection_view
-        collection_view._.wait_for_visible()
+        collection_view._.wait_for_present()
         return collection_view
 
     def _populated_paired_and_wait_for_it(self):
         history_id = self.current_history_id()
         input_collection = self.dataset_collection_populator.create_pair_in_history(history_id).json()
         collection_hid = input_collection["hid"]
-        self.home()
+        if not self.is_beta_history():
+            self.home()
         self.history_panel_wait_for_hid_state(collection_hid, "ok")
 
         return input_collection
diff --git a/lib/galaxy_test/selenium/test_history_panel_pagination.py b/lib/galaxy_test/selenium/test_history_panel_pagination.py
index 99d1fe20526..e5802e708db 100644
--- a/lib/galaxy_test/selenium/test_history_panel_pagination.py
+++ b/lib/galaxy_test/selenium/test_history_panel_pagination.py
@@ -1,3 +1,5 @@
+import pytest
+
 from galaxy_test.base.populators import flakey
 from .framework import (
     selenium_test,
@@ -12,6 +14,9 @@ class HistoryPanelPaginationTestCase(SeleniumTestCase):
     @selenium_test
     @flakey  # The next button doesn't always work - maybe a delay in JS callback registering for that.
     def test_pagination(self):
+        if self.is_beta_history():
+            raise pytest.skip("There is no pagination on the beta history panel")
+
         history_id = self.current_history_id()
 
         self.dataset_populator.new_dataset(history_id, content='1\t2\t3', name="data1")
diff --git a/lib/galaxy_test/selenium/test_tool_form.py b/lib/galaxy_test/selenium/test_tool_form.py
index fd851bbf9c0..2c2e8ec5588 100644
--- a/lib/galaxy_test/selenium/test_tool_form.py
+++ b/lib/galaxy_test/selenium/test_tool_form.py
@@ -359,6 +359,7 @@ https://raw.githubusercontent.com/jmchilton/galaxy/apply_rules_tutorials/test-da
         rule_builder.main_button_ok.wait_for_and_click()
         self.tool_form_execute()
         output_hid = example["output_hid"]
+        self.home()
         self.history_panel_wait_for_hid_ok(output_hid)
         output_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=output_hid, wait=False)
         example["check"](output_hdca, self.dataset_populator)

From 27bbb645694de474313cc211311d76a84de001f1 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Mon, 30 Aug 2021 00:50:06 -0700
Subject: [PATCH 428/452] test_workflow_run

---
 lib/galaxy_test/selenium/test_workflow_run.py | 23 +++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy_test/selenium/test_workflow_run.py b/lib/galaxy_test/selenium/test_workflow_run.py
index 7c33ad462ab..890452284d3 100644
--- a/lib/galaxy_test/selenium/test_workflow_run.py
+++ b/lib/galaxy_test/selenium/test_workflow_run.py
@@ -38,7 +38,7 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.workflow_run_submit()
         self.sleep_for(self.wait_types.UX_TRANSITION)
         self.screenshot("workflow_run_simple_submitted")
-        self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1)
+        self._wait_for_hid(2, "ok")
         self.history_panel_click_item_title(hid=2, wait=True)
         self.assert_item_summary_includes(2, "2 sequences")
         self.screenshot("workflow_run_simple_complete")
@@ -94,7 +94,7 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.screenshot("workflow_run_rename_simple_input")
         self.workflow_run_submit()
         output_hid = 2
-        self.history_panel_wait_for_hid_ok(output_hid, allowed_force_refreshes=1)
+        self._wait_for_hid(output_hid, "ok")
         history_id = self.current_history_id()
         details = self.dataset_populator.get_history_dataset_details(history_id, hid=output_hid)
         assert details["name"] == "moocow suffix", details
@@ -126,7 +126,7 @@ steps:
         self.screenshot("workflow_run_step_parameter_input")
         self.workflow_run_submit()
         output_hid = 2
-        self.history_panel_wait_for_hid_ok(output_hid, allowed_force_refreshes=1)
+        self._wait_for_hid(output_hid, "ok")
         history_id = self.current_history_id()
         content = self.dataset_populator.get_history_dataset_content(history_id, hid=output_hid)
         assert "12345" in content, content
@@ -143,7 +143,7 @@ steps:
         self.screenshot("workflow_run_rename_subworkflow_input")
         self.workflow_run_submit()
         output_hid = 2
-        self.history_panel_wait_for_hid_ok(output_hid, allowed_force_refreshes=1)
+        self._wait_for_hid(output_hid, "ok")
         history_id = self.current_history_id()
         details = self.dataset_populator.get_history_dataset_details(history_id, hid=output_hid)
         assert details["name"] == "moocow suffix", details
@@ -166,7 +166,7 @@ steps:
         self.screenshot("workflow_run_two_inputs")
         self.workflow_run_submit()
 
-        self.history_panel_wait_for_hid_ok(7, allowed_force_refreshes=1)
+        self._wait_for_hid(7, "ok")
         content = self.dataset_populator.get_history_dataset_content(history_id, hid=7)
         self.assertEqual("10.0\n30.0\n20.0\n40.0\n", content)
 
@@ -178,7 +178,7 @@ steps:
         self.workflow_run_specify_inputs(inputs)
         self.screenshot("workflow_run_rules")
         self.workflow_run_submit()
-        self.history_panel_wait_for_hid_ok(6, allowed_force_refreshes=1)
+        self._wait_for_hid(6, "ok")
         output_content = self.dataset_populator.get_history_collection_details(history_id, hid=6)
         rules_test_data.check_example_2(output_content, self.dataset_populator)
 
@@ -235,7 +235,7 @@ steps:
         self.workflow_click_option(".workflow-run")
 
     def _assert_has_3_lines_after_run(self, hid):
-        self.history_panel_wait_for_hid_ok(hid, allowed_force_refreshes=1)
+        self._wait_for_hid(hid, "ok")
         history_id = self.current_history_id()
         content = self.dataset_populator.get_history_dataset_content(history_id, hid=hid)
         assert len([x for x in content.split("\n") if x]) == 3, content
@@ -259,3 +259,12 @@ steps:
         assert initial_value == "", initial_value
         input_element.clear()
         input_element.send_keys(value)
+
+    def _wait_for_hid(self, hid, state="ok"):
+        if self.is_beta_history():
+            # takes a little while for these things to upload and end up in the history
+            timeout = self.wait_length(self.wait_types.JOB_COMPLETION)
+            row_selector = self.content_item_by_attributes(hid=hid, state=state)
+            self.wait_for_visible(row_selector, timeout=timeout)
+        else:
+            self.history_panel_wait_for_hid_ok(7, allowed_force_refreshes=1)

From 22cd8eb5492e86c1c00ca7e27bdafb265c755205 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 30 Aug 2021 17:35:39 +0200
Subject: [PATCH 429/452] Refactor QualifiedUrlBuilder to work also with
 relative URLs

---
 lib/galaxy/managers/context.py            | 10 ++++++++--
 lib/galaxy/tools/parameters/basic.py      |  3 ++-
 lib/galaxy/web/framework/__init__.py      |  4 ++--
 lib/galaxy/webapps/base/webapp.py         |  9 +++++----
 lib/galaxy/webapps/galaxy/api/__init__.py | 13 ++++++++-----
 lib/galaxy/work/context.py                | 11 ++++++-----
 test/unit/tools/test_execution.py         |  2 +-
 test/unit/unittest_utils/galaxy_mock.py   |  2 +-
 8 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py
index c258bac1ec6..e99d4f2151e 100644
--- a/lib/galaxy/managers/context.py
+++ b/lib/galaxy/managers/context.py
@@ -67,8 +67,14 @@ class ProvidesAppContext:
         """
 
     @abc.abstractproperty
-    def qualified_url_builder(self) -> Optional[Callable[[str], str]]:
-        """Provide access to fully qualified Galaxy URLs (if available)."""
+    def url_builder(self) -> Optional[Callable[..., str]]:
+        """
+        Provide access to Galaxy URLs (if available).
+
+        :type   qualified:  bool
+        :param  qualified:  if True, the fully qualified URL is returned,
+                            else a relative URL is returned (default False).
+        """
 
     @property
     def security(self) -> IdEncodingHelper:
diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index b4509d1ee3e..a68203afd46 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -800,7 +800,7 @@ class BaseURLToolParameter(HiddenToolParameter):
         try:
             if not self.value.startswith("/"):
                 raise Exception("baseurl value must start with a /")
-            return trans.qualified_url_builder(self.value)
+            return trans.url_builder(self.value, qualified=True)
         except Exception as e:
             log.debug('Url creation failed for "%s": %s', self.name, unicodify(e))
             return self.value
@@ -1108,6 +1108,7 @@ class SelectTagParameter(SelectToolParameter):
     """
     Select set that is composed of a set of tags available for an input.
     """
+
     def __init__(self, tool, input_source):
         input_source = ensure_input_source(input_source)
         super().__init__(tool, input_source)
diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py
index 64dabaf7933..1cd37a0cc07 100644
--- a/lib/galaxy/web/framework/__init__.py
+++ b/lib/galaxy/web/framework/__init__.py
@@ -7,14 +7,14 @@ from . import base
 DEPRECATED_URL_ATTRIBUTE_MESSAGE = "*deprecated attribute, URL not filled in by server*"
 
 
-def handle_url_for(*args, **kargs) -> str:
+def handle_url_for(*args, **kwargs) -> str:
     """Tries to resolve the URL using the `routes` module.
 
     This only works in a WSGI app so a deprecation message is returned
     when running an ASGI app.
     """
     try:
-        return base.routes.url_for(*args, **kargs)
+        return base.routes.url_for(*args, **kwargs)
     except AttributeError:
         return DEPRECATED_URL_ATTRIBUTE_MESSAGE
 
diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py
index 82d9c7c2873..5aca1611849 100644
--- a/lib/galaxy/webapps/base/webapp.py
+++ b/lib/galaxy/webapps/base/webapp.py
@@ -188,8 +188,9 @@ def config_allows_origin(origin_raw, config):
     return False
 
 
-def qualified_url_builder(path):
-    return url_for(path, qualified=True)
+def url_builder(*args, **kwargs) -> str:
+    """Wrapper around the uWSGI version of the function for reversing URLs."""
+    return url_for(*args, **kwargs)
 
 
 class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryContext):
@@ -283,8 +284,8 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, context.ProvidesHistoryCo
         return self._app
 
     @property
-    def qualified_url_builder(self):
-        return qualified_url_builder
+    def url_builder(self):
+        return url_builder
 
     def setup_i18n(self):
         locales = []
diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py
index d021fbaa5e4..12549e7d865 100644
--- a/lib/galaxy/webapps/galaxy/api/__init__.py
+++ b/lib/galaxy/webapps/galaxy/api/__init__.py
@@ -124,13 +124,16 @@ def get_user(galaxy_session: Optional[model.GalaxySession] = Depends(get_session
     return api_user
 
 
-class QualifiedUrlBuilder:
+class UrlBuilder:
 
     def __init__(self, request: Request):
         self.request = request
 
-    def __call__(self, path):
-        return self.request.url_for(path)
+    def __call__(self, name: str, **path_params):
+        qualified = path_params.pop("qualified", False)
+        if qualified:
+            return self.request.url_for(name, **path_params)
+        return self.request.app.url_path_for(name, **path_params)
 
 
 DependsOnUser = Depends(get_user)
@@ -139,8 +142,8 @@ DependsOnUser = Depends(get_user)
 def get_trans(request: Request, app: StructuredApp = DependsOnApp, user: Optional[User] = Depends(get_user),
               galaxy_session: Optional[model.GalaxySession] = Depends(get_session),
               ) -> SessionRequestContext:
-    qualified_url_builder = QualifiedUrlBuilder(request)
-    return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session, qualified_url_builder=qualified_url_builder, host=request.client.host)
+    url_builder = UrlBuilder(request)
+    return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session, url_builder=url_builder, host=request.client.host)
 
 
 DependsOnTrans = Depends(get_trans)
diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py
index a53f4af6a19..d3c0592f5b9 100644
--- a/lib/galaxy/work/context.py
+++ b/lib/galaxy/work/context.py
@@ -19,12 +19,12 @@ class WorkRequestContext(ProvidesHistoryContext):
     objects.
     """
 
-    def __init__(self, app, user=None, history=None, workflow_building_mode=False, qualified_url_builder=None):
+    def __init__(self, app, user=None, history=None, workflow_building_mode=False, url_builder=None):
         self._app = app
         self.__user = user
         self.__user_current_roles = None
         self.__history = history
-        self._qualified_url_builder = qualified_url_builder
+        self._url_builder = url_builder
         self.workflow_building_mode = workflow_building_mode
 
     @property
@@ -32,8 +32,8 @@ class WorkRequestContext(ProvidesHistoryContext):
         return self._app
 
     @property
-    def qualified_url_builder(self):
-        return self._qualified_url_builder
+    def url_builder(self):
+        return self._url_builder
 
     def get_history(self, create=False):
         return self.__history
@@ -60,6 +60,7 @@ class WorkRequestContext(ProvidesHistoryContext):
 
 class SessionRequestContext(WorkRequestContext):
     """Like WorkRequestContext, but provides access to galaxy session and session."""
+
     def __init__(self, **kwargs):
         self.galaxy_session = kwargs.pop('galaxy_session', None)
         self._host = kwargs.pop("host")
@@ -80,4 +81,4 @@ def proxy_work_context_for_history(trans: ProvidesHistoryContext, history: Optio
     history that is different from the user's current history (which also might change during
     the request).
     """
-    return WorkRequestContext(app=trans.app, user=trans.user, history=history or trans.history, qualified_url_builder=trans.qualified_url_builder, workflow_building_mode=workflow_building_mode)
+    return WorkRequestContext(app=trans.app, user=trans.user, history=history or trans.history, url_builder=trans.url_builder, workflow_building_mode=workflow_building_mode)
diff --git a/test/unit/tools/test_execution.py b/test/unit/tools/test_execution.py
index d231a876c50..bc3cd0618bc 100644
--- a/test/unit/tools/test_execution.py
+++ b/test/unit/tools/test_execution.py
@@ -204,7 +204,7 @@ class MockTrans:
         self.workflow_building_mode = False
         self.webapp = Bunch(name="galaxy")
         self.sa_session = self.app.model.context
-        self.qualified_url_builder = None
+        self.url_builder = None
 
     def get_history(self, **kwargs):
         return self.history
diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py
index f8b3b973a0a..2e50ed279f5 100644
--- a/test/unit/unittest_utils/galaxy_mock.py
+++ b/test/unit/unittest_utils/galaxy_mock.py
@@ -241,7 +241,7 @@ class MockTrans:
         self.anonymous = False
         self.debug = True
         self.user_is_admin = True
-        self.qualified_url_builder = None
+        self.url_builder = None
 
         self.galaxy_session = None
         self.__user = user

From 4973f5b9b7682ca1cf4acf719c50ca607e772c17 Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Mon, 30 Aug 2021 09:11:08 -0700
Subject: [PATCH 430/452] fixed test_refresh_preserves_state

---
 lib/galaxy/selenium/navigates_galaxy.py       |  4 ++++
 .../selenium/test_history_panel.py            | 19 ++++++++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 5cde6d1eb17..5e3f175954c 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -316,6 +316,7 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_item_component(self, history_item=None, hid=None, multi_history_panel=False):
         if self.is_beta_history():
+            assert hid
             return self.content_item_by_attributes(hid=hid)
         if history_item is None:
             assert hid
@@ -1478,6 +1479,9 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_item_showing_details(self, hid):
         item_component = self.history_panel_item_component(hid=hid)
+        item_component.wait_for_present()
+        if item_component.details.is_absent:
+            return False
         return item_component.details.is_displayed
 
     def collection_builder_set_name(self, name):
diff --git a/lib/galaxy_test/selenium/test_history_panel.py b/lib/galaxy_test/selenium/test_history_panel.py
index 42a8af26b4f..65d979d1ea6 100644
--- a/lib/galaxy_test/selenium/test_history_panel.py
+++ b/lib/galaxy_test/selenium/test_history_panel.py
@@ -227,15 +227,14 @@ class HistoryPanelTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_refresh_preserves_state(self):
-        if self.is_beta_history():
-            raise pytest.skip("Beta History Panel does not preserve state xref https://github.com/galaxyproject/galaxy/issues/12119")
         self.perform_upload(self.get_filename("1.txt"))
         self.wait_for_history()
 
         # Open the details, verify they are open and do a refresh.
         self.history_panel_ensure_showing_item_details(hid=1)
         self.history_panel_item_body_component(1, wait=True)
-        self.history_panel_refresh_click()
+
+        self._refresh()
         self.wait_for_history()
 
         # After the refresh, verify the details are still open.
@@ -244,9 +243,12 @@ class HistoryPanelTestCase(SeleniumTestCase):
         assert self.history_panel_item_showing_details(hid=1)
 
         # Close the detailed display, refresh, and ensure they are still closed.
-        self.history_panel_click_item_title(hid=1, wait=True)
+        wait = not self.is_beta_history()
+        self.history_panel_click_item_title(hid=1, wait=wait)
         assert not self.history_panel_item_showing_details(hid=1)
-        self.history_panel_refresh_click()
+
+        self._refresh()
+
         self.sleep_for(self.wait_types.UX_TRANSITION)
         self.wait_for_selector_clickable(self.history_panel_item_selector(hid=1))
         assert not self.history_panel_item_showing_details(hid=1)
@@ -255,3 +257,10 @@ class HistoryPanelTestCase(SeleniumTestCase):
     def assert_name_changed(self):
         name = self.history_panel_name()
         self.assertEqual(name, NEW_HISTORY_NAME)
+
+    def _refresh(self):
+        if self.is_beta_history():
+            # beta history has no refresh button
+            self.home()
+        else:
+            self.history_panel_refresh_click()

From 97a191c80c49d5f205a9ca8df149250dcb62d031 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 30 Aug 2021 18:42:49 +0200
Subject: [PATCH 431/452] Adapt `group_roles` API to use new `UrlBuilder`

- Replace `url_for` with `trans.url_builder` that will handle the url generation in both legacy and FastAPI modes.
- Set `group_role` as unique name of the `FastAPIGroupRoles.show` route.
- Change `role_id` path parameter name to `id` to match the legacy route.
---
 lib/galaxy/webapps/galaxy/api/group_roles.py | 10 +++++-----
 lib/galaxy_test/api/test_group_roles.py      |  6 +-----
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py
index 0334d3f8e1d..f9980effb94 100644
--- a/lib/galaxy/webapps/galaxy/api/group_roles.py
+++ b/lib/galaxy/webapps/galaxy/api/group_roles.py
@@ -13,7 +13,6 @@ from galaxy.schema.schema import GroupRoleListModel, GroupRoleModel
 from galaxy.web import (
     expose_api,
     require_admin,
-    url_for,
 )
 from . import (
     BaseGalaxyAPIController,
@@ -42,7 +41,7 @@ RoleIDParam: EncodedDatabaseIdField = Path(
 
 def group_role_to_model(trans, encoded_group_id, role):
     encoded_role_id = trans.security.encode_id(role.id)
-    url = url_for('group_role', group_id=encoded_group_id, id=encoded_role_id)
+    url = trans.url_builder('group_role', group_id=encoded_group_id, id=encoded_role_id)
     return GroupRoleModel(
         id=encoded_role_id,
         name=role.name,
@@ -62,13 +61,14 @@ class FastAPIGroupRoles:
         group_roles = self.manager.index(trans, group_id)
         return GroupRoleListModel(__root__=[group_role_to_model(trans, group_id, gr.role) for gr in group_roles])
 
-    @router.get('/api/groups/{group_id}/roles/{role_id}',
+    @router.get('/api/groups/{group_id}/roles/{id}',
+                name="group_role",
                 require_admin=True,
                 summary='Displays information about a group role.')
     def show(self, trans: ProvidesAppContext = DependsOnTrans,
              group_id: EncodedDatabaseIdField = GroupIDParam,
-             role_id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
-        role = self.manager.show(trans, role_id, group_id)
+             id: EncodedDatabaseIdField = RoleIDParam) -> GroupRoleModel:
+        role = self.manager.show(trans, id, group_id)
         return group_role_to_model(trans, group_id, role)
 
     @router.put('/api/groups/{group_id}/roles/{role_id}',
diff --git a/lib/galaxy_test/api/test_group_roles.py b/lib/galaxy_test/api/test_group_roles.py
index ef2c1b95e2c..67b00527ec3 100644
--- a/lib/galaxy_test/api/test_group_roles.py
+++ b/lib/galaxy_test/api/test_group_roles.py
@@ -1,6 +1,5 @@
 from typing import List
 
-from galaxy.web.framework import DEPRECATED_URL_ATTRIBUTE_MESSAGE
 from galaxy_test.base.populators import DatasetPopulator
 from ._framework import ApiTestCase
 
@@ -68,10 +67,7 @@ class GroupRolesApiTestCase(ApiTestCase):
         self._assert_status_code_is_ok(update_response)
         group_role = update_response.json()
         self._assert_valid_group_role(group_role, assert_id=encoded_role_id)
-        assert (
-            group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}"
-            or group_role["url"] == DEPRECATED_URL_ATTRIBUTE_MESSAGE
-        )
+        assert group_role["url"] == f"/api/groups/{encoded_group_id}/roles/{encoded_role_id}"
 
     def test_update_only_admin(self):
         encoded_group_id = "any-group-id"

From 20fd446fcb2ce125feea378a43b665a53d35f66d Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 30 Aug 2021 15:33:52 -0400
Subject: [PATCH 432/452] Match tabular datatype through datatype mapping, add
 filter to ignore empty rows

---
 .../Elements/HistoryDatasetDisplay.vue        | 63 ++++++++++++-------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
index 147a27f4c78..7d5abb2409b 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
@@ -44,25 +44,31 @@
                             :src="displayUrl"
                         />
                         <div v-else-if="itemContent.item_data">
-                            <div v-if="datasetType == 'tabular'">
-                                <UrlDataProvider
-                                    :url="metaUrl"
-                                    v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
-                                >
-                                    <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
-                                    <div v-else-if="metaError">{{ metaError }}</div>
-                                    <b-table
-                                        v-else
-                                        striped
-                                        hover
-                                        :fields="getFields(metaData)"
-                                        :items="getItems(itemContent.item_data, metaData)"
-                                    />
-                                </UrlDataProvider>
-                            </div>
-                            <pre v-else>
-                                <code class="text-normalwrap">{{ itemContent.item_data }}</code>
-                            </pre>
+                            <UrlDataProvider
+                                :url="datatypesUrl"
+                                v-slot="{ result: datatypesModel, loading: datatypesLoading }"
+                            >
+                                <LoadingSpan v-if="datatypesLoading" message="Loading Datatypes" />
+                                <div v-else-if="isSubType('tabular', datasetType, datatypesModel)">
+                                    <UrlDataProvider
+                                        :url="metaUrl"
+                                        v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
+                                    >
+                                        <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
+                                        <div v-else-if="metaError">{{ metaError }}</div>
+                                        <b-table
+                                            v-else
+                                            striped
+                                            hover
+                                            :fields="getFields(metaData)"
+                                            :items="getItems(itemContent.item_data, metaData)"
+                                        />
+                                    </UrlDataProvider>
+                                </div>
+                                <pre v-else>
+                                    <code class="text-normalwrap">{{ itemContent.item_data }}</code>
+                                </pre>
+                            </UrlDataProvider>
                         </div>
                         <div v-else>No content found.</div>
                         <b-link v-if="itemContent.truncated" :href="itemContent.item_url"> Show More... </b-link>
@@ -77,6 +83,7 @@
 import { getAppRoot } from "onload/loadConfig";
 import LoadingSpan from "components/LoadingSpan";
 import { UrlDataProvider } from "components/providers/UrlDataProvider";
+import { DatatypesMapperModel } from "components/Datatypes/model";
 
 export default {
     components: {
@@ -106,6 +113,9 @@ export default {
             const dataset = this.datasets[this.args.history_dataset_id];
             return dataset && dataset.name;
         },
+        datatypesUrl() {
+            return "api/datatypes/types_and_mapping";
+        },
         downloadUrl() {
             return `${getAppRoot()}dataset/display?dataset_id=${this.args.history_dataset_id}`;
         },
@@ -123,6 +133,10 @@ export default {
         },
     },
     methods: {
+        isSubType(child, parent, datatypesModel) {
+            const datatypesMapper = new DatatypesMapperModel(datatypesModel);
+            return datatypesMapper.isSubType(child, parent);
+        },
         getFields(metaData) {
             const fields = [];
             const columnNames = metaData.metadata_column_names || [];
@@ -145,10 +159,17 @@ export default {
                 if (i >= comments) {
                     const tabs = line.split(delimiter);
                     const rowData = {};
+                    let hasData = false;
                     tabs.forEach((cellData, j) => {
-                        rowData[j] = cellData;
+                        const cellDataTrimmed = cellData.trim();
+                        if (cellDataTrimmed) {
+                            hasData = true;
+                        }
+                        rowData[j] = cellDataTrimmed;
                     });
-                    tableData.push(rowData);
+                    if (hasData) {
+                        tableData.push(rowData);
+                    }
                 }
             });
             return tableData;

From c22aa7d7a27efa4f76d175bf683483015ed064b2 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 30 Aug 2021 15:45:24 -0400
Subject: [PATCH 433/452] Use dataset mapper for html and pdf subtype mapping

---
 .../Elements/HistoryDatasetDisplay.vue        | 76 ++++++++++---------
 1 file changed, 40 insertions(+), 36 deletions(-)

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
index 7d5abb2409b..ed207cd2544 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
@@ -37,41 +37,45 @@
                     <LoadingSpan v-if="loading" message="Loading Dataset" />
                     <div v-else-if="error">{{ error }}</div>
                     <div v-else class="embedded-dataset content-height">
-                        <b-embed
-                            v-if="['pdf', 'html'].includes(datasetType)"
-                            type="iframe"
-                            aspect="16by9"
-                            :src="displayUrl"
-                        />
-                        <div v-else-if="itemContent.item_data">
-                            <UrlDataProvider
-                                :url="datatypesUrl"
-                                v-slot="{ result: datatypesModel, loading: datatypesLoading }"
-                            >
-                                <LoadingSpan v-if="datatypesLoading" message="Loading Datatypes" />
-                                <div v-else-if="isSubType('tabular', datasetType, datatypesModel)">
-                                    <UrlDataProvider
-                                        :url="metaUrl"
-                                        v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
-                                    >
-                                        <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
-                                        <div v-else-if="metaError">{{ metaError }}</div>
-                                        <b-table
-                                            v-else
-                                            striped
-                                            hover
-                                            :fields="getFields(metaData)"
-                                            :items="getItems(itemContent.item_data, metaData)"
-                                        />
-                                    </UrlDataProvider>
+                        <UrlDataProvider
+                            :url="datatypesUrl"
+                            v-slot="{ result: datatypesModel, loading: datatypesLoading }"
+                        >
+                            <LoadingSpan v-if="datatypesLoading" message="Loading Datatypes" />
+                            <div v-else>
+                                <b-embed
+                                    v-if="isSubTypeOfAny(datasetType, ['pdf', 'html'], datatypesModel)"
+                                    type="iframe"
+                                    aspect="16by9"
+                                    :src="displayUrl"
+                                />
+                                <div v-else-if="itemContent.item_data">
+                                    <div v-if="isSubTypeOfAny(datasetType, ['tabular'], datatypesModel)">
+                                        <UrlDataProvider
+                                            :url="metaUrl"
+                                            v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
+                                        >
+                                            <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
+                                            <div v-else-if="metaError">{{ metaError }}</div>
+                                            <b-table
+                                                v-else
+                                                striped
+                                                hover
+                                                :fields="getFields(metaData)"
+                                                :items="getItems(itemContent.item_data, metaData)"
+                                            />
+                                        </UrlDataProvider>
+                                    </div>
+                                    <pre v-else>
+                                            <code class="text-normalwrap">{{ itemContent.item_data }}</code>
+                                        </pre>
                                 </div>
-                                <pre v-else>
-                                    <code class="text-normalwrap">{{ itemContent.item_data }}</code>
-                                </pre>
-                            </UrlDataProvider>
-                        </div>
-                        <div v-else>No content found.</div>
-                        <b-link v-if="itemContent.truncated" :href="itemContent.item_url"> Show More... </b-link>
+                                <div v-else>No content found.</div>
+                                <b-link v-if="itemContent.truncated" :href="itemContent.item_url">
+                                    Show More...
+                                </b-link>
+                            </div>
+                        </UrlDataProvider>
                     </div>
                 </UrlDataProvider>
             </b-card-body>
@@ -133,9 +137,9 @@ export default {
         },
     },
     methods: {
-        isSubType(child, parent, datatypesModel) {
+        isSubTypeOfAny(child, parents, datatypesModel) {
             const datatypesMapper = new DatatypesMapperModel(datatypesModel);
-            return datatypesMapper.isSubType(child, parent);
+            return datatypesMapper.isSubTypeOfAny(child, parents);
         },
         getFields(metaData) {
             const fields = [];

From 06599fb0f7599f5d482e9571ed07a926d834b892 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 30 Aug 2021 16:06:31 -0400
Subject: [PATCH 434/452] Fix indent missed by format client

---
 .../components/Markdown/Elements/HistoryDatasetDisplay.vue    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
index ed207cd2544..c725d313abc 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
@@ -67,8 +67,8 @@
                                         </UrlDataProvider>
                                     </div>
                                     <pre v-else>
-                                            <code class="text-normalwrap">{{ itemContent.item_data }}</code>
-                                        </pre>
+                                        <code class="text-normalwrap">{{ itemContent.item_data }}</code>
+                                    </pre>
                                 </div>
                                 <div v-else>No content found.</div>
                                 <b-link v-if="itemContent.truncated" :href="itemContent.item_url">

From 8dccebbcc98c19be447f60ed1fe880aff975c50e Mon Sep 17 00:00:00 2001
From: Mason Houtz <nerdinacan@gmail.com>
Date: Mon, 30 Aug 2021 13:42:51 -0700
Subject: [PATCH 435/452] fixed test_workflow_run

---
 lib/galaxy/selenium/navigates_galaxy.py       | 27 +++++-----
 lib/galaxy_test/selenium/test_workflow_run.py | 53 ++++++++++++-------
 2 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 5e3f175954c..27acc5320ee 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -399,23 +399,24 @@ class NavigatesGalaxy(HasDriver):
         if self.is_beta_history():
             history_item_selector_state = self.content_item_by_attributes(hid=hid, state=state)
             history_item_selector_state.wait_for_visible()
+            return history_item_selector_state
         else:
             history_item_selector = self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=allowed_force_refreshes, multi_history_panel=multi_history_panel)
             # history_item_selector_state = history_item_selector.with_class(f"state-{state}")
             history_item_selector_state = history_item_selector.with_data("state", state)
-        try:
-            self.history_item_wait_for(history_item_selector_state, allowed_force_refreshes)
-        except self.TimeoutException as e:
-            history_item = self.wait_for_visible(history_item_selector)
-            current_state = "UNKNOWN"
-            classes = history_item.get_attribute("class").split(" ")
-            for clazz in classes:
-                if clazz.startswith("state-"):
-                    current_state = clazz[len("state-"):]
-            template = "Failed waiting on history item %d state to change to [%s] current state [%s]. "
-            message = template % (hid, state, current_state)
-            raise self.prepend_timeout_message(e, message)
-        return history_item_selector_state
+            try:
+                self.history_item_wait_for(history_item_selector_state, allowed_force_refreshes)
+            except self.TimeoutException as e:
+                history_item = self.wait_for_visible(history_item_selector)
+                current_state = "UNKNOWN"
+                classes = history_item.get_attribute("class").split(" ")
+                for clazz in classes:
+                    if clazz.startswith("state-"):
+                        current_state = clazz[len("state-"):]
+                template = "Failed waiting on history item %d state to change to [%s] current state [%s]. "
+                message = template % (hid, state, current_state)
+                raise self.prepend_timeout_message(e, message)
+            return history_item_selector_state
 
     def click_grid_popup_option(self, item_name, option_label):
         item_button = None
diff --git a/lib/galaxy_test/selenium/test_workflow_run.py b/lib/galaxy_test/selenium/test_workflow_run.py
index 890452284d3..a3714872a84 100644
--- a/lib/galaxy_test/selenium/test_workflow_run.py
+++ b/lib/galaxy_test/selenium/test_workflow_run.py
@@ -38,8 +38,13 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.workflow_run_submit()
         self.sleep_for(self.wait_types.UX_TRANSITION)
         self.screenshot("workflow_run_simple_submitted")
-        self._wait_for_hid(2, "ok")
-        self.history_panel_click_item_title(hid=2, wait=True)
+        if self.is_beta_history():
+            item = self.content_item_by_attributes(hid=2, state="ok")
+            item.wait_for_present()
+            item.title.wait_for_and_click()
+        else:
+            self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1)
+            self.history_panel_click_item_title(hid=2, wait=True)
         self.assert_item_summary_includes(2, "2 sequences")
         self.screenshot("workflow_run_simple_complete")
 
@@ -94,7 +99,10 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.screenshot("workflow_run_rename_simple_input")
         self.workflow_run_submit()
         output_hid = 2
-        self._wait_for_hid(output_hid, "ok")
+        if self.is_beta_history():
+            self.content_item_by_attributes(hid=output_hid, state="ok").wait_for_present()
+        else:
+            self.history_panel_wait_for_hid_ok(output_hid, allowed_force_refreshes=1)
         history_id = self.current_history_id()
         details = self.dataset_populator.get_history_dataset_details(history_id, hid=output_hid)
         assert details["name"] == "moocow suffix", details
@@ -126,7 +134,10 @@ steps:
         self.screenshot("workflow_run_step_parameter_input")
         self.workflow_run_submit()
         output_hid = 2
-        self._wait_for_hid(output_hid, "ok")
+        if self.is_beta_history():
+            self.content_item_by_attributes(hid=output_hid, state="ok").wait_for_present()
+        else:
+            self.history_panel_wait_for_hid_ok(output_hid, allowed_force_refreshes=1)
         history_id = self.current_history_id()
         content = self.dataset_populator.get_history_dataset_content(history_id, hid=output_hid)
         assert "12345" in content, content
@@ -143,7 +154,10 @@ steps:
         self.screenshot("workflow_run_rename_subworkflow_input")
         self.workflow_run_submit()
         output_hid = 2
-        self._wait_for_hid(output_hid, "ok")
+        if self.is_beta_history():
+            self.content_item_by_attributes(hid=output_hid, state="ok").wait_for_present()
+        else:
+            self.history_panel_wait_for_hid_ok(output_hid, allowed_force_refreshes=1)
         history_id = self.current_history_id()
         details = self.dataset_populator.get_history_dataset_details(history_id, hid=output_hid)
         assert details["name"] == "moocow suffix", details
@@ -165,8 +179,12 @@ steps:
         self.workflow_run_specify_inputs(inputs)
         self.screenshot("workflow_run_two_inputs")
         self.workflow_run_submit()
-
-        self._wait_for_hid(7, "ok")
+        if self.is_beta_history():
+            timeout = self.wait_length(self.wait_types.JOB_COMPLETION)
+            item = self.content_item_by_attributes(hid=7, state="ok")
+            self.wait_for_present(item, timeout=timeout)
+        else:
+            self.history_panel_wait_for_hid_ok(7, allowed_force_refreshes=1)
         content = self.dataset_populator.get_history_dataset_content(history_id, hid=7)
         self.assertEqual("10.0\n30.0\n20.0\n40.0\n", content)
 
@@ -178,7 +196,12 @@ steps:
         self.workflow_run_specify_inputs(inputs)
         self.screenshot("workflow_run_rules")
         self.workflow_run_submit()
-        self._wait_for_hid(6, "ok")
+        if self.is_beta_history():
+            timeout = self.wait_length(self.wait_types.JOB_COMPLETION)
+            item = self.content_item_by_attributes(hid=6, state="ok")
+            self.wait_for_present(item, timeout=timeout)
+        else:
+            self.history_panel_wait_for_hid_ok(6, allowed_force_refreshes=1)
         output_content = self.dataset_populator.get_history_collection_details(history_id, hid=6)
         rules_test_data.check_example_2(output_content, self.dataset_populator)
 
@@ -235,7 +258,10 @@ steps:
         self.workflow_click_option(".workflow-run")
 
     def _assert_has_3_lines_after_run(self, hid):
-        self._wait_for_hid(hid, "ok")
+        if self.is_beta_history():
+            self.content_item_by_attributes(hid=hid, state="ok").wait_for_present()
+        else:
+            self.history_panel_wait_for_hid_ok(hid, allowed_force_refreshes=1)
         history_id = self.current_history_id()
         content = self.dataset_populator.get_history_dataset_content(history_id, hid=hid)
         assert len([x for x in content.split("\n") if x]) == 3, content
@@ -259,12 +285,3 @@ steps:
         assert initial_value == "", initial_value
         input_element.clear()
         input_element.send_keys(value)
-
-    def _wait_for_hid(self, hid, state="ok"):
-        if self.is_beta_history():
-            # takes a little while for these things to upload and end up in the history
-            timeout = self.wait_length(self.wait_types.JOB_COMPLETION)
-            row_selector = self.content_item_by_attributes(hid=hid, state=state)
-            self.wait_for_visible(row_selector, timeout=timeout)
-        else:
-            self.history_panel_wait_for_hid_ok(7, allowed_force_refreshes=1)

From ed3c4d8b8f82da98b7b534bb3e3c892ace752fea Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Mon, 30 Aug 2021 23:37:11 -0400
Subject: [PATCH 436/452] Remove unused divs in markdown components, slightly
 increase padding for pages to improve scolling and navigation

---
 .../CollectionDisplay.vue                     |  98 ++++++-----
 .../Elements/HistoryDatasetDisplay.vue        | 153 +++++++++---------
 .../Markdown/Elements/JobMetrics.vue          |   8 +-
 .../Markdown/Elements/JobParameters.vue       |   8 +-
 .../components/Markdown/Elements/ToolStd.vue  |  12 +-
 .../Markdown/Elements/Visualization.vue       |  12 +-
 .../Elements/Workflow/WorkflowDisplay.vue     |  84 +++++-----
 client/src/components/Markdown/Markdown.vue   |   2 +-
 client/src/style/scss/markdown.scss           |   7 +
 9 files changed, 186 insertions(+), 198 deletions(-)

diff --git a/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
index 6b69dc66870..077e7b77536 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
@@ -1,54 +1,52 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card body-class="p-0">
-            <b-card-header>
-                <span class="float-right">
-                    <b-button
-                        :href="downloadUrl"
-                        variant="link"
-                        size="sm"
-                        role="button"
-                        title="Download Collection"
-                        type="button"
-                        class="py-0 px-1"
-                        v-b-tooltip.hover
-                    >
-                        <span class="fa fa-download" />
-                    </b-button>
-                    <CurrentUser v-slot="{ user }">
-                        <UserHistories v-if="user" :user="user" v-slot="{ currentHistoryId }">
-                            <b-button
-                                v-if="currentHistoryId"
-                                @click="onCopyCollection(currentHistoryId)"
-                                href="#"
-                                role="button"
-                                variant="link"
-                                title="Import Collection"
-                                type="button"
-                                class="py-0 px-1"
-                                v-b-tooltip.hover
-                            >
-                                <span class="fa fa-file-import" />
-                            </b-button>
-                        </UserHistories>
-                    </CurrentUser>
-                </span>
-                <span>
-                    <span>Dataset Collection:</span>
-                    <span class="font-weight-light">{{ collectionName }}</span>
-                </span>
-            </b-card-header>
-            <b-card-body>
-                <LoadingSpan v-if="loading" message="Loading Collection" />
-                <div v-else class="content-height">
-                    <b-alert v-if="!!messageText" :variant="messageVariant" show>
-                        {{ messageText }}
-                    </b-alert>
-                    <CollectionTree :node="itemContent" :skip-head="true" />
-                </div>
-            </b-card-body>
-        </b-card>
-    </div>
+    <b-card body-class="p-0">
+        <b-card-header>
+            <span class="float-right">
+                <b-button
+                    :href="downloadUrl"
+                    variant="link"
+                    size="sm"
+                    role="button"
+                    title="Download Collection"
+                    type="button"
+                    class="py-0 px-1"
+                    v-b-tooltip.hover
+                >
+                    <span class="fa fa-download" />
+                </b-button>
+                <CurrentUser v-slot="{ user }">
+                    <UserHistories v-if="user" :user="user" v-slot="{ currentHistoryId }">
+                        <b-button
+                            v-if="currentHistoryId"
+                            @click="onCopyCollection(currentHistoryId)"
+                            href="#"
+                            role="button"
+                            variant="link"
+                            title="Import Collection"
+                            type="button"
+                            class="py-0 px-1"
+                            v-b-tooltip.hover
+                        >
+                            <span class="fa fa-file-import" />
+                        </b-button>
+                    </UserHistories>
+                </CurrentUser>
+            </span>
+            <span>
+                <span>Dataset Collection:</span>
+                <span class="font-weight-light">{{ collectionName }}</span>
+            </span>
+        </b-card-header>
+        <b-card-body>
+            <LoadingSpan v-if="loading" message="Loading Collection" />
+            <div v-else class="content-height">
+                <b-alert v-if="!!messageText" :variant="messageVariant" show>
+                    {{ messageText }}
+                </b-alert>
+                <CollectionTree :node="itemContent" :skip-head="true" />
+            </div>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
index c725d313abc..138e9e9a002 100644
--- a/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+++ b/client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
@@ -1,86 +1,79 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card body-class="p-0">
-            <b-card-header v-if="!embedded">
-                <span class="float-right">
-                    <b-button
-                        :href="downloadUrl"
-                        variant="link"
-                        size="sm"
-                        role="button"
-                        title="Download Dataset"
-                        type="button"
-                        class="py-0 px-1"
-                        v-b-tooltip.hover
-                    >
-                        <span class="fa fa-download" />
-                    </b-button>
-                    <b-button
-                        :href="importUrl"
-                        role="button"
-                        variant="link"
-                        title="Import Dataset"
-                        type="button"
-                        class="py-0 px-1"
-                        v-b-tooltip.hover
-                    >
-                        <span class="fa fa-file-import" />
-                    </b-button>
-                </span>
-                <span>
-                    <span>Dataset:</span>
-                    <span class="font-weight-light">{{ datasetName }}</span>
-                </span>
-            </b-card-header>
-            <b-card-body>
-                <UrlDataProvider :url="itemUrl" v-slot="{ result: itemContent, loading, error }">
-                    <LoadingSpan v-if="loading" message="Loading Dataset" />
-                    <div v-else-if="error">{{ error }}</div>
-                    <div v-else class="embedded-dataset content-height">
-                        <UrlDataProvider
-                            :url="datatypesUrl"
-                            v-slot="{ result: datatypesModel, loading: datatypesLoading }"
-                        >
-                            <LoadingSpan v-if="datatypesLoading" message="Loading Datatypes" />
-                            <div v-else>
-                                <b-embed
-                                    v-if="isSubTypeOfAny(datasetType, ['pdf', 'html'], datatypesModel)"
-                                    type="iframe"
-                                    aspect="16by9"
-                                    :src="displayUrl"
-                                />
-                                <div v-else-if="itemContent.item_data">
-                                    <div v-if="isSubTypeOfAny(datasetType, ['tabular'], datatypesModel)">
-                                        <UrlDataProvider
-                                            :url="metaUrl"
-                                            v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
-                                        >
-                                            <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
-                                            <div v-else-if="metaError">{{ metaError }}</div>
-                                            <b-table
-                                                v-else
-                                                striped
-                                                hover
-                                                :fields="getFields(metaData)"
-                                                :items="getItems(itemContent.item_data, metaData)"
-                                            />
-                                        </UrlDataProvider>
-                                    </div>
-                                    <pre v-else>
-                                        <code class="text-normalwrap">{{ itemContent.item_data }}</code>
-                                    </pre>
+    <b-card body-class="p-0">
+        <b-card-header v-if="!embedded">
+            <span class="float-right">
+                <b-button
+                    :href="downloadUrl"
+                    variant="link"
+                    size="sm"
+                    role="button"
+                    title="Download Dataset"
+                    type="button"
+                    class="py-0 px-1"
+                    v-b-tooltip.hover
+                >
+                    <span class="fa fa-download" />
+                </b-button>
+                <b-button
+                    :href="importUrl"
+                    role="button"
+                    variant="link"
+                    title="Import Dataset"
+                    type="button"
+                    class="py-0 px-1"
+                    v-b-tooltip.hover
+                >
+                    <span class="fa fa-file-import" />
+                </b-button>
+            </span>
+            <span>
+                <span>Dataset:</span>
+                <span class="font-weight-light">{{ datasetName }}</span>
+            </span>
+        </b-card-header>
+        <b-card-body>
+            <UrlDataProvider :url="itemUrl" v-slot="{ result: itemContent, loading, error }">
+                <LoadingSpan v-if="loading" message="Loading Dataset" />
+                <div v-else-if="error">{{ error }}</div>
+                <div v-else class="embedded-dataset content-height">
+                    <UrlDataProvider :url="datatypesUrl" v-slot="{ result: datatypesModel, loading: datatypesLoading }">
+                        <LoadingSpan v-if="datatypesLoading" message="Loading Datatypes" />
+                        <div v-else>
+                            <b-embed
+                                v-if="isSubTypeOfAny(datasetType, ['pdf', 'html'], datatypesModel)"
+                                type="iframe"
+                                aspect="16by9"
+                                :src="displayUrl"
+                            />
+                            <div v-else-if="itemContent.item_data">
+                                <div v-if="isSubTypeOfAny(datasetType, ['tabular'], datatypesModel)">
+                                    <UrlDataProvider
+                                        :url="metaUrl"
+                                        v-slot="{ result: metaData, loading: metaLoading, error: metaError }"
+                                    >
+                                        <LoadingSpan v-if="metaLoading" message="Loading Metadata" />
+                                        <div v-else-if="metaError">{{ metaError }}</div>
+                                        <b-table
+                                            v-else
+                                            striped
+                                            hover
+                                            :fields="getFields(metaData)"
+                                            :items="getItems(itemContent.item_data, metaData)"
+                                        />
+                                    </UrlDataProvider>
                                 </div>
-                                <div v-else>No content found.</div>
-                                <b-link v-if="itemContent.truncated" :href="itemContent.item_url">
-                                    Show More...
-                                </b-link>
+                                <pre v-else>
+                                    <code class="text-normalwrap">{{ itemContent.item_data }}</code>
+                                </pre>
                             </div>
-                        </UrlDataProvider>
-                    </div>
-                </UrlDataProvider>
-            </b-card-body>
-        </b-card>
-    </div>
+                            <div v-else>No content found.</div>
+                            <b-link v-if="itemContent.truncated" :href="itemContent.item_url"> Show More... </b-link>
+                        </div>
+                    </UrlDataProvider>
+                </div>
+            </UrlDataProvider>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Elements/JobMetrics.vue b/client/src/components/Markdown/Elements/JobMetrics.vue
index 66d17bc0397..46af304a0a7 100644
--- a/client/src/components/Markdown/Elements/JobMetrics.vue
+++ b/client/src/components/Markdown/Elements/JobMetrics.vue
@@ -1,9 +1,7 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card nobody>
-            <JobMetrics class="job-metrics" :job-id="args.job_id" />
-        </b-card>
-    </div>
+    <b-card nobody>
+        <JobMetrics class="job-metrics" :job-id="args.job_id" />
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Elements/JobParameters.vue b/client/src/components/Markdown/Elements/JobParameters.vue
index f3d48790d11..3560a6e443a 100644
--- a/client/src/components/Markdown/Elements/JobParameters.vue
+++ b/client/src/components/Markdown/Elements/JobParameters.vue
@@ -1,9 +1,7 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card nobody>
-            <JobParameters class="job-parameters" :job-id="args.job_id" :param="args.param" :include-title="false" />
-        </b-card>
-    </div>
+    <b-card nobody>
+        <JobParameters class="job-parameters" :job-id="args.job_id" :param="args.param" :include-title="false" />
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Elements/ToolStd.vue b/client/src/components/Markdown/Elements/ToolStd.vue
index ddb47974030..0a819c29f58 100644
--- a/client/src/components/Markdown/Elements/ToolStd.vue
+++ b/client/src/components/Markdown/Elements/ToolStd.vue
@@ -1,11 +1,9 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card nobody class="content-height">
-            <div :class="name" :job_id="args.job_id">
-                <pre><code class="text-normalwrap">{{ jobContent }}</code></pre>
-            </div>
-        </b-card>
-    </div>
+    <b-card nobody class="content-height">
+        <div :class="name" :job_id="args.job_id">
+            <pre><code class="text-normalwrap">{{ jobContent }}</code></pre>
+        </div>
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Elements/Visualization.vue b/client/src/components/Markdown/Elements/Visualization.vue
index abe938955ef..0f500b9efa2 100644
--- a/client/src/components/Markdown/Elements/Visualization.vue
+++ b/client/src/components/Markdown/Elements/Visualization.vue
@@ -1,11 +1,9 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card body-class="embed-responsive embed-responsive-4by3">
-            <LoadingSpan v-if="loading" class="m-2" message="Loading Visualization" />
-            <div v-else-if="error" class="m-2">{{ error }}</div>
-            <iframe v-else class="embed-responsive-item" :src="visualizationUrl" />
-        </b-card>
-    </div>
+    <b-card body-class="embed-responsive embed-responsive-4by3">
+        <LoadingSpan v-if="loading" class="m-2" message="Loading Visualization" />
+        <div v-else-if="error" class="m-2">{{ error }}</div>
+        <iframe v-else class="embed-responsive-item" :src="visualizationUrl" />
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Elements/Workflow/WorkflowDisplay.vue b/client/src/components/Markdown/Elements/Workflow/WorkflowDisplay.vue
index 73b2692f755..d2ba0a814fd 100644
--- a/client/src/components/Markdown/Elements/Workflow/WorkflowDisplay.vue
+++ b/client/src/components/Markdown/Elements/Workflow/WorkflowDisplay.vue
@@ -1,48 +1,46 @@
 <template>
-    <div class="w-100 p-2">
-        <b-card body-class="p-0">
-            <b-card-header v-if="!embedded">
-                <span class="float-right">
-                    <b-button
-                        :href="downloadUrl"
-                        variant="link"
-                        size="sm"
-                        role="button"
-                        title="Download Workflow"
-                        type="button"
-                        class="py-0 px-1"
-                        v-b-tooltip.hover
-                    >
-                        <span class="fa fa-download" />
-                    </b-button>
-                    <b-button
-                        :href="importUrl"
-                        role="button"
-                        variant="link"
-                        title="Import Workflow"
-                        type="button"
-                        class="py-0 px-1"
-                        v-b-tooltip.hover
-                    >
-                        <span class="fa fa-file-import" />
-                    </b-button>
-                </span>
-                <span>
-                    <span>Workflow:</span>
-                    <span class="font-weight-light">{{ workflowName }}</span>
-                </span>
-            </b-card-header>
-            <b-card-body>
-                <LoadingSpan v-if="loading" message="Loading Workflow" />
-                <div v-else class="content-height">
-                    <div v-for="step in itemContent.steps" :key="step.order_index" class="mb-2">
-                        <div>Step {{ step.order_index + 1 }}: {{ step.label }}</div>
-                        <WorkflowTree :input="step" :skip-head="true" />
-                    </div>
+    <b-card body-class="p-0">
+        <b-card-header v-if="!embedded">
+            <span class="float-right">
+                <b-button
+                    :href="downloadUrl"
+                    variant="link"
+                    size="sm"
+                    role="button"
+                    title="Download Workflow"
+                    type="button"
+                    class="py-0 px-1"
+                    v-b-tooltip.hover
+                >
+                    <span class="fa fa-download" />
+                </b-button>
+                <b-button
+                    :href="importUrl"
+                    role="button"
+                    variant="link"
+                    title="Import Workflow"
+                    type="button"
+                    class="py-0 px-1"
+                    v-b-tooltip.hover
+                >
+                    <span class="fa fa-file-import" />
+                </b-button>
+            </span>
+            <span>
+                <span>Workflow:</span>
+                <span class="font-weight-light">{{ workflowName }}</span>
+            </span>
+        </b-card-header>
+        <b-card-body>
+            <LoadingSpan v-if="loading" message="Loading Workflow" />
+            <div v-else class="content-height">
+                <div v-for="step in itemContent.steps" :key="step.order_index" class="mb-2">
+                    <div>Step {{ step.order_index + 1 }}: {{ step.label }}</div>
+                    <WorkflowTree :input="step" :skip-head="true" />
                 </div>
-            </b-card-body>
-        </b-card>
-    </div>
+            </div>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
diff --git a/client/src/components/Markdown/Markdown.vue b/client/src/components/Markdown/Markdown.vue
index 24b081b19c0..700a77aeaea 100644
--- a/client/src/components/Markdown/Markdown.vue
+++ b/client/src/components/Markdown/Markdown.vue
@@ -44,7 +44,7 @@
                     </div>
                 </b-alert>
             </div>
-            <div v-for="(obj, index) in markdownObjects" :key="index">
+            <div v-for="(obj, index) in markdownObjects" :key="index" class="markdown-components">
                 <p v-if="obj.name == 'default'" v-html="obj.content" class="text-justify m-2" />
                 <div v-else-if="obj.name == 'generate_galaxy_version'" class="galaxy-version">
                     <pre><code>{{ getVersion }}</code></pre>
diff --git a/client/src/style/scss/markdown.scss b/client/src/style/scss/markdown.scss
index 2e39f5bce9f..31de045c3a6 100644
--- a/client/src/style/scss/markdown.scss
+++ b/client/src/style/scss/markdown.scss
@@ -12,3 +12,10 @@
         flex: 1;
     }
 }
+
+.markdown-wrapper {
+    @extend .px-4;
+    .markdown-components {
+        @extend .py-2;
+    }
+}

From c84441099068136da75cb598f3cf3a5918be8d8b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 30 Aug 2021 17:05:08 +0200
Subject: [PATCH 437/452] Keep separate session for ToolShedRepositoryCache

Should fix
```
Traceback (most recent call last):
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/jobs/runners/__init__.py", line 244, in prepare_job
    stderr_file=stderr_file,
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/jobs/runners/__init__.py", line 285, in build_command_line
    stderr_file=stderr_file,
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/jobs/command_factory.py", line 79, in build_command
    __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params)
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/jobs/command_factory.py", line 194, in __handle_dependency_resolution
    if local_dependency_resolution and job_wrapper.dependency_shell_commands:
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/jobs/__init__.py", line 1012, in dependency_shell_commands
    job_directory=self.working_directory
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tools/__init__.py", line 1891, in build_dependency_shell_commands
    tool_instance=self
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tool_util/deps/__init__.py", line 184, in dependency_shell_commands
    requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds)
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tool_util/deps/__init__.py", line 194, in requirements_to_dependencies
    requirement_to_dependency = self._requirements_to_dependencies_dict(requirements, **kwds)
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tool_util/deps/__init__.py", line 283, in _requirements_to_dependencies_dict
    dependency = resolver.resolve(requirement, **kwds)
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tool_util/deps/resolvers/galaxy_packages.py", line 79, in resolve
    return self._find_dep_versioned(name, version, type=type, **kwds)
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tool_util/deps/resolvers/tool_shed_packages.py", line 25, in _find_dep_versioned
    path = self._get_package_installed_dependency_path(installed_tool_dependency, name, version)
  File "/cvmfs/test.galaxyproject.org/galaxy/lib/galaxy/tool_util/deps/resolvers/tool_shed_packages.py", line 43, in _get_package_installed_dependency_path
    tool_shed_repository = installed_tool_dependency.tool_shed_repository
  File "/cvmfs/test.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 480, in __get__
    return self.impl.get(state, dict_)
  File "/cvmfs/test.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 931, in get
    value = self.callable_(state, passive)
  File "/cvmfs/test.galaxyproject.org/venv/lib/python3.6/site-packages/sqlalchemy/orm/strategies.py", line 836, in _load_for_state
    % (orm_util.state_str(state), self.key)
sqlalchemy.orm.exc.DetachedInstanceError: Parent instance <ToolDependency at 0x7fd3e5e82160> is not bound to a Session; lazy load operation of attribute 'tool_shed_repository' cannot proceed (Background on this error at: http://sqlalche.me/e/14/bhk3)
```
Which I think broke in
https://github.com/galaxyproject/galaxy/pull/11737#discussion_r636354063.

Instead of hoping that we preload all attributes it seems more robust to
use a separate session for the cache that is not tied to a thread and
that will not be closed by flushes in other parts of the application.
---
 lib/galaxy/app.py                             |  4 +++
 lib/galaxy/model/orm/engine_factory.py        |  7 ++++-
 lib/galaxy/tools/cache.py                     | 27 +++++++++----------
 test/unit/tools/conftest.py                   | 12 ++++-----
 .../tools/test_tool_shed_repository_cache.py  | 22 ++-------------
 5 files changed, 31 insertions(+), 41 deletions(-)

diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index bb6e982f65b..58b1eea05ea 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -216,6 +216,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
             ("job manager", self._shutdown_job_manager),
             ("application heartbeat", self._shutdown_heartbeat),
             ("repository manager", self._shutdown_repo_manager),
+            ("database connection repository cache", self._shutdown_repo_cache),
             ("database connection", self._shutdown_model),
             ("application stack", self._shutdown_application_stack),
         ]
@@ -391,6 +392,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
     def _shutdown_repo_manager(self):
         self.update_repository_manager.shutdown()
 
+    def _shutdown_repo_cache(self):
+        self.tool_shed_repository_cache.shutdown()
+
     def _shutdown_application_stack(self):
         self.application_stack.shutdown()
 
diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py
index eb1f05d2630..e84c796ba9d 100644
--- a/lib/galaxy/model/orm/engine_factory.py
+++ b/lib/galaxy/model/orm/engine_factory.py
@@ -86,7 +86,12 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac
                 except AttributeError:
                     pass
 
+    # Set check_same_thread to False for sqlite, handled by request-specific session
+    # See https://fastapi.tiangolo.com/tutorial/sql-databases/#note
+    connect_args = {}
+    if 'sqlite://' in url:
+        connect_args['check_same_thread'] = False
     # Create the database engine
-    engine = create_engine(url, **engine_options)
+    engine = create_engine(url, connect_args=connect_args, **engine_options)
     register_after_fork(engine, lambda e: e.dispose())
     return engine
diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py
index 69617e5eda5..25a551d7f4b 100644
--- a/lib/galaxy/tools/cache.py
+++ b/lib/galaxy/tools/cache.py
@@ -13,10 +13,10 @@ from sqlalchemy.orm import (
     defer,
     joinedload,
 )
+from sqlalchemy.orm.session import sessionmaker
 from sqlitedict import SqliteDict
 
 from galaxy.model.tool_shed_install import ToolShedRepository
-from galaxy.structured_app import MinimalManagerApp
 from galaxy.tool_util.toolbox.base import ToolConfRepository
 from galaxy.util import unicodify
 from galaxy.util.hash_util import md5_hash_file
@@ -286,8 +286,8 @@ class ToolShedRepositoryCache:
     repositories: List[ToolShedRepository]
     repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]]
 
-    def __init__(self, app: MinimalManagerApp):
-        self.app = app
+    def __init__(self, session: sessionmaker):
+        self.session = session()
         # Contains ToolConfRepository objects created from shed_tool_conf.xml entries
         self.local_repositories = []
         # Repositories loaded from database
@@ -300,17 +300,13 @@ class ToolShedRepositoryCache:
         self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
 
     def rebuild(self):
-        try:
-            session = self.app.install_model._SessionLocal()
-            self.repositories = session.query(ToolShedRepository).options(
-                defer(ToolShedRepository.metadata), joinedload('tool_dependencies')
-            ).all()
-            repos_by_tuple = defaultdict(list)
-            for repository in self.repositories + self.local_repositories:
-                repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
-            self.repos_by_tuple = repos_by_tuple
-        finally:
-            session.close()
+        self.repositories = self.session.query(ToolShedRepository).options(
+            defer(ToolShedRepository.metadata), joinedload('tool_dependencies')
+        ).all()
+        repos_by_tuple = defaultdict(list)
+        for repository in self.repositories + self.local_repositories:
+            repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
+        self.repos_by_tuple = repos_by_tuple
 
     def get_installed_repository(self, tool_shed=None, name=None, owner=None, installed_changeset_revision=None, changeset_revision=None, repository_id=None):
         if repository_id:
@@ -327,3 +323,6 @@ class ToolShedRepositoryCache:
                 continue
             return repo
         return None
+
+    def shutdown(self) -> None:
+        self.session.close()
diff --git a/test/unit/tools/conftest.py b/test/unit/tools/conftest.py
index 4e6ff369dfb..d150a0ac9a1 100644
--- a/test/unit/tools/conftest.py
+++ b/test/unit/tools/conftest.py
@@ -18,13 +18,13 @@ def mock_app():
 
 @pytest.fixture
 def tool_shed_repository_cache(mock_app):
-    tool_shed_repository_cache = ToolShedRepositoryCache(app=mock_app)
+    tool_shed_repository_cache = ToolShedRepositoryCache(session=mock_app.install_model.context)
     return tool_shed_repository_cache
 
 
 @pytest.fixture
 def repos(mock_app):
-    repositories = [create_repo(mock_app, changeset=i + 1, installed_changeset=i) for i in range(10)]
+    repositories = [create_repo(mock_app.install_model.context, changeset=i + 1, installed_changeset=i) for i in range(10)]
     mock_app.install_model.context.flush()
     return repositories
 
@@ -46,7 +46,7 @@ def tool_conf_repos(tool_shed_repository_cache):
     return tool_shed_repository_cache.local_repositories
 
 
-def create_repo(app, changeset, installed_changeset, config_filename=None):
+def create_repo(session, changeset, installed_changeset, config_filename=None):
     metadata = {
         'tools': [{
             'add_to_tool_panel': False,  # to have repository.includes_tools_for_display_in_tool_panel=False in InstalledRepositoryManager.activate_repository()
@@ -64,8 +64,8 @@ def create_repo(app, changeset, installed_changeset, config_filename=None):
     repository.installed_changeset_revision = str(installed_changeset)
     repository.deleted = False
     repository.uninstalled = False
-    app.install_model.context.add(repository)
-    app.install_model.context.flush()
+    session.add(repository)
+    session.flush()
     tool_dependency = tool_shed_install.ToolDependency(
         name='Name',
         version='100',
@@ -73,5 +73,5 @@ def create_repo(app, changeset, installed_changeset, config_filename=None):
         status='ok',
         tool_shed_repository_id=repository.id,
     )
-    app.install_model.context.add(tool_dependency)
+    session.add(tool_dependency)
     return repository
diff --git a/test/unit/tools/test_tool_shed_repository_cache.py b/test/unit/tools/test_tool_shed_repository_cache.py
index ae99298292e..26966703b07 100644
--- a/test/unit/tools/test_tool_shed_repository_cache.py
+++ b/test/unit/tools/test_tool_shed_repository_cache.py
@@ -1,5 +1,4 @@
 import pytest
-from sqlalchemy.orm.exc import DetachedInstanceError
 
 from .conftest import create_repo
 
@@ -22,8 +21,8 @@ def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_r
     tool_shed_repository_cache.rebuild()
     assert len(tool_shed_repository_cache.repositories) == 10
     assert len(tool_shed_repository_cache.local_repositories) == 10
-    create_repo(tool_shed_repository_cache.app, '21', '20')
-    tool_shed_repository_cache.app.install_model.context.flush()
+    create_repo(tool_shed_repository_cache.session, '21', '20')
+    tool_shed_repository_cache.session.flush()
     tool_shed_repository_cache.rebuild()
     assert len(tool_shed_repository_cache.repositories) == 11
     assert len(tool_shed_repository_cache.local_repositories) == 10
@@ -55,20 +54,3 @@ def test_get_installed_repository(tool_shed_repository_cache, repos, tool_conf_r
         assert repo
     else:
         assert repo is None
-
-
-def test_repo_cache_expunge(tool_shed_repository_cache, repos):
-    tool_shed_repository_cache.rebuild()
-    assert len(tool_shed_repository_cache.repositories) == 10
-    # Modify and commit a repo, will expire in memory attributes of orm objects (unless using a different session)
-    repo = tool_shed_repository_cache.repositories[0]
-    repo.name = 'new name'
-    tool_shed_repository_cache.app.install_model.session.flush()
-    # remove session, should demonstrate the separate session is in use
-    tool_shed_repository_cache.app.install_model.session.remove()
-    assert repo.changeset_revision == "1"
-    assert repo.tool_dependencies[0].name == 'Name'
-    with pytest.raises(DetachedInstanceError):
-        # Make sure this still raises DetachedInstanceError,
-        # keeping this in memory would be expensive
-        repo.metadata

From ade2764e25d408bf5288a53f2b89089a07c7d6cd Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Tue, 31 Aug 2021 11:24:14 -0400
Subject: [PATCH 438/452] Fix padding in legacy dataset display mako

---
 templates/webapps/galaxy/dataset/display.mako | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/webapps/galaxy/dataset/display.mako b/templates/webapps/galaxy/dataset/display.mako
index 0e3ccc90bf6..fb174b6fbd1 100644
--- a/templates/webapps/galaxy/dataset/display.mako
+++ b/templates/webapps/galaxy/dataset/display.mako
@@ -100,7 +100,7 @@
 
     <div class="unified-panel-body">
         <div style="overflow: auto; height: 100%;">
-            <div class="page-body">
+            <div class="page-body p-3">
                 <div style="float: right">
                     ${self.render_item_links( item )}
                 </div>

From c6793dcdde8009ebe665629f4abde2eb5d79d7c2 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Thu, 2 Sep 2021 00:37:49 -0400
Subject: [PATCH 439/452] Hide details header in dataset error page if details
 are unavailable

---
 client/src/components/DatasetInformation/DatasetError.vue | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 5618117ab4d..43b2b846ea5 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -21,10 +21,10 @@
                         <b id="dataset-error-tool-id" class="text-break">{{ jobDetails.tool_id }}</b
                         >.
                     </p>
-                    <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || jobDetails.job_messages">
+                    <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || hasMessages(jobDetails)">
                         <h3>Details</h3>
                     </div>
-                    <div v-if="jobDetails.job_messages" id="dataset-error-job-messages">
+                    <div v-if="hasMessages(jobDetails)" id="dataset-error-job-messages">
                         <p>Execution resulted in the following messages:</p>
                         <div v-for="(job_message, index) in jobDetails.job_messages" :key="index">
                             <pre class="rounded code">{{ job_message["desc"] }}</pre>
@@ -130,6 +130,9 @@ export default {
         };
     },
     methods: {
+        hasMessages(jobDetails) {
+            return jobDetails.job_messages && jobDetails.job_messages.length > 0;
+        },
         onError(err) {
             this.errorMessage = err;
         },

From 96ef23a79ba9b8ec47dc4d8705a602d8950e28cd Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Thu, 2 Sep 2021 16:10:41 -0400
Subject: [PATCH 440/452] Added instructions on how to debug Galaxy backed by a
 Slurm cluster

---
 doc/source/dev/debugging_galaxy_slurm.md | 78 ++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 doc/source/dev/debugging_galaxy_slurm.md

diff --git a/doc/source/dev/debugging_galaxy_slurm.md b/doc/source/dev/debugging_galaxy_slurm.md
new file mode 100644
index 00000000000..e3963cc8e76
--- /dev/null
+++ b/doc/source/dev/debugging_galaxy_slurm.md
@@ -0,0 +1,78 @@
+
+# Debugging Galaxy: Slurm Compute Cluster
+
+This document explains how to debug a Galaxy instance that uses a Slurm compute cluster (not the
+default LocalRunner). If you have a MacOS machine, you need to create a Linux VM using Oracle's 
+VirtualBoxVM and run the instructions in the VM. For reference, I created a 64-bit Linux VM with 
+4096 MB of RAM and 32 GB of disk space (all other parameters were left as default) and managed to 
+successfully run the instructions. If using a Linux VM, you may need to install Git, Python, VSCode, 
+and Docker (if using Irods storage). Finally, These instructions are intended for developers who 
+want to debug their Galaxy instance and are not meant for deployment purposes (We have nice Ansible 
+playbooks for that).
+
+## Debugging Galaxy in VS Code 
+
+1. Install libslurm-dev
+    * `sudo apt install libslurm-dev`
+    * On my VM, the above command puts `slurm/slurm.h` in `/usr/include` and `libslurm.a` in `/usr/lib/x86_64-linux-gnu`
+    * You need these header and library files to compile slurm-drmaa in the next step
+
+2. Install slurm-drmaa
+    * Download slurm-drmaa tar file from `https://github.com/natefoo/slurm-drmaa/releases/download/1.1.2/slurm-drmaa-1.1.2.tar.gz`
+    * Extract the downloaded tar file into home directory: `tar -xzvf slurm-drmaa-1.1.2.tar.gz -C ~`       
+    * To compile and install slurm-drmaa
+        * `cd ~/slurm-drmaa-1.1.2`
+        * `./configure --with-slurm-inc=/usr/include --with-slurm-lib=/usr/lib/x86_64-linux-gpu`
+        * `sudo make install`
+        * On my VM, the above command puts `libdrmaa.so` in `/usr/local/lib`. You need the path to this file in the next step
+
+3. Replace the contents of `~/galaxy/config/job_conf.xml` with the following code snippet (create the file if it does not already exist)
+    ```
+    <?xml version="1.0"?>
+    <job_conf>
+        <plugins workers="3">
+            <plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner">
+                <param id="drmaa_library_path">/usr/local/lib/libdrmaa.so</param>
+            </plugin>
+        </plugins>
+        <destinations default="slurm">
+            <destination id="slurm" runner="slurm"/>
+        </destinations>
+        <resources default="default">
+          <group id="default"></group>
+          <group id="memoryonly">memory</group>
+          <group id="all">processors,memory,time,project</group>
+        </resources>
+    </job_conf>
+    ```
+
+4. Install Slurm (Instructions borrowed from `https://gist.github.com/ckandoth/2acef6310041244a690e4c08d2610423`)
+    * Install the necessary packages: `sudo apt install -y slurm-wlm slurm-wlm-doc`
+    * Load `/usr/share/doc/slurm-wlm/html/configurator.html` in a browser
+        * Set your VM's hostname in `SlurmctldHost` and `NodeName`
+        * Set `StateSaveLocation` to `/var/spool/slurmctld`
+        * Set `SlurmdSpoolDir` to `/var/spool/slurmd`
+        * Set `ProctrackType` to `LinuxProc`
+        * Set `SelectType` to `Cons_res` 
+        * Set `SelectTypeParameters` to `CR_Core_Memory` 
+        * Set `JobAcctGatherType` to `Linux`
+        * Hit `Submit` and save the resulting text into `/etc/slurm-llnl/slurm.conf`
+            * I.e., the configuration file referred to in `/lib/systemd/system/slurmctld.service` and `/lib/systemd/system/slurmd.service`
+    * Create `/var/spool/slurmd` and `/var/spool/slurmctld` and `/var/log/slurm_jobacct.log` file and set ownership appropriately
+        * `sudo mkdir -p /var/spool/slurmd`
+        * `sudo mkdir -p /var/spool/slurmctld`
+        * `sudo touch /var/log/slurm_jobacct.log`
+        * `sudo chown slurm:slurm /var/spool/slurmd /var/spool/slurmctld /var/log/slurm_jobacct.log`
+    * Install mailutils so that Slurm won't complain about `/bin/mail` missing
+        * `sudo apt install -y mailutils`
+    * Make sure munge is installed/running, and a munge.key was created with user-only read-only permissions, owned by munge:munge
+        * `sudo apt install munge`
+        * `sudo service munge start`
+        * `sudo ls -l /etc/munge/munge.key`
+    * Start slurmd and slurmctld services
+        * `sudo service slurmd start`
+        * `sudo service slurmctld start`
+
+5. Follow the instructions [here](./debuging_galaxy.md) to setup VSCode for debugging. 
+
+Enjoy debugging session your Galaxy instance backed by a Slurm cluster!        

From b47374aef7adc848ca4ebea8f8265c60147a0eea Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Thu, 2 Sep 2021 17:10:15 -0400
Subject: [PATCH 441/452] Added instructions for debuuging Galaxy using a Slurm
 cluster

---
 doc/source/dev/debugging_galaxy_slurm.md | 29 +++++++++++++-----------
 doc/source/dev/index.rst                 |  1 +
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/doc/source/dev/debugging_galaxy_slurm.md b/doc/source/dev/debugging_galaxy_slurm.md
index e3963cc8e76..ffd8ec6b742 100644
--- a/doc/source/dev/debugging_galaxy_slurm.md
+++ b/doc/source/dev/debugging_galaxy_slurm.md
@@ -14,19 +14,19 @@ playbooks for that).
 
 1. Install libslurm-dev
     * `sudo apt install libslurm-dev`
-    * On my VM, the above command puts `slurm/slurm.h` in `/usr/include` and `libslurm.a` in `/usr/lib/x86_64-linux-gnu`
+    * On my VM, the above command puts slurm/slurm.h in /usr/include and libslurm.a in /usr/lib/x86_64-linux-gnu
     * You need these header and library files to compile slurm-drmaa in the next step
 
 2. Install slurm-drmaa
-    * Download slurm-drmaa tar file from `https://github.com/natefoo/slurm-drmaa/releases/download/1.1.2/slurm-drmaa-1.1.2.tar.gz`
+    * Download slurm-drmaa tar file from https://github.com/natefoo/slurm-drmaa/releases/download/1.1.2/slurm-drmaa-1.1.2.tar.gz
     * Extract the downloaded tar file into home directory: `tar -xzvf slurm-drmaa-1.1.2.tar.gz -C ~`       
     * To compile and install slurm-drmaa
         * `cd ~/slurm-drmaa-1.1.2`
         * `./configure --with-slurm-inc=/usr/include --with-slurm-lib=/usr/lib/x86_64-linux-gpu`
         * `sudo make install`
-        * On my VM, the above command puts `libdrmaa.so` in `/usr/local/lib`. You need the path to this file in the next step
+        * On my VM, the above command puts libdrmaa.so in /usr/local/lib. You need the path to this file in the next step
 
-3. Replace the contents of `~/galaxy/config/job_conf.xml` with the following code snippet (create the file if it does not already exist)
+3. Replace the contents of ~/galaxy/config/job_conf.xml with the following code snippet (create the file if it does not already exist)
     ```
     <?xml version="1.0"?>
     <job_conf>
@@ -46,24 +46,27 @@ playbooks for that).
     </job_conf>
     ```
 
-4. Install Slurm (Instructions borrowed from `https://gist.github.com/ckandoth/2acef6310041244a690e4c08d2610423`)
+4. Uncomment the following line in ~/galaxy/config/galaxy.yml
+    * `job_config_file: job_conf.xml`
+
+5. Install Slurm (Instructions borrowed from https://gist.github.com/ckandoth/2acef6310041244a690e4c08d2610423)
     * Install the necessary packages: `sudo apt install -y slurm-wlm slurm-wlm-doc`
-    * Load `/usr/share/doc/slurm-wlm/html/configurator.html` in a browser
+    * Load /usr/share/doc/slurm-wlm/html/configurator.html in a browser
         * Set your VM's hostname in `SlurmctldHost` and `NodeName`
         * Set `StateSaveLocation` to `/var/spool/slurmctld`
         * Set `SlurmdSpoolDir` to `/var/spool/slurmd`
         * Set `ProctrackType` to `LinuxProc`
-        * Set `SelectType` to `Cons_res` 
-        * Set `SelectTypeParameters` to `CR_Core_Memory` 
+        * Set `SelectType` to `Cons_res`
+        * Set `SelectTypeParameters` to `CR_Core_Memory`
         * Set `JobAcctGatherType` to `Linux`
-        * Hit `Submit` and save the resulting text into `/etc/slurm-llnl/slurm.conf`
-            * I.e., the configuration file referred to in `/lib/systemd/system/slurmctld.service` and `/lib/systemd/system/slurmd.service`
-    * Create `/var/spool/slurmd` and `/var/spool/slurmctld` and `/var/log/slurm_jobacct.log` file and set ownership appropriately
+        * Hit Submit and save the resulting text into `/etc/slurm-llnl/slurm.conf`
+            * I.e., the configuration file referred to in /lib/systemd/system/slurmctld.service and /lib/systemd/system/slurmd.service
+    * Create /var/spool/slurmd and /var/spool/slurmctld directories and /var/log/slurm_jobacct.log file and set ownership appropriately
         * `sudo mkdir -p /var/spool/slurmd`
         * `sudo mkdir -p /var/spool/slurmctld`
         * `sudo touch /var/log/slurm_jobacct.log`
         * `sudo chown slurm:slurm /var/spool/slurmd /var/spool/slurmctld /var/log/slurm_jobacct.log`
-    * Install mailutils so that Slurm won't complain about `/bin/mail` missing
+    * Install mailutils so that Slurm won't complain about /bin/mail missing
         * `sudo apt install -y mailutils`
     * Make sure munge is installed/running, and a munge.key was created with user-only read-only permissions, owned by munge:munge
         * `sudo apt install munge`
@@ -73,6 +76,6 @@ playbooks for that).
         * `sudo service slurmd start`
         * `sudo service slurmctld start`
 
-5. Follow the instructions [here](./debuging_galaxy.md) to setup VSCode for debugging. 
+6. Follow the instructions [here](./debuging_galaxy.md) to setup VSCode for debugging. 
 
 Enjoy debugging session your Galaxy instance backed by a Slurm cluster!        
diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst
index 74f48f5be70..4f363522ce2 100644
--- a/doc/source/dev/index.rst
+++ b/doc/source/dev/index.rst
@@ -22,5 +22,6 @@ A multi-hour long video playlist covering these slides can be found at
   writing_tests
   debugging_tests
   debugging_galaxy
+  debugging_galaxy_slurm
   starting_galaxy
   file_upload

From 99880294c6b7bdead170a04d5ad6aad7e37d838d Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Fri, 3 Sep 2021 09:27:29 -0400
Subject: [PATCH 442/452] Minor revision

---
 doc/source/dev/debugging_galaxy_slurm.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/dev/debugging_galaxy_slurm.md b/doc/source/dev/debugging_galaxy_slurm.md
index ffd8ec6b742..961bc34dc96 100644
--- a/doc/source/dev/debugging_galaxy_slurm.md
+++ b/doc/source/dev/debugging_galaxy_slurm.md
@@ -46,7 +46,7 @@ playbooks for that).
     </job_conf>
     ```
 
-4. Uncomment the following line in ~/galaxy/config/galaxy.yml
+4. Uncomment the following line in ~/galaxy/config/galaxy.yml (If the file does not exist, create it by copying ~/galaxy/config/galaxy.yml.sample)
     * `job_config_file: job_conf.xml`
 
 5. Install Slurm (Instructions borrowed from https://gist.github.com/ckandoth/2acef6310041244a690e4c08d2610423)

From 41068ee6964eac2dc9ac1fbbacd45048c07c6362 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 3 Sep 2021 11:28:50 -0400
Subject: [PATCH 443/452] revert broken(?) changes to selenium abstraction

---
 lib/galaxy/selenium/navigates_galaxy.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py
index 27acc5320ee..9207abb0c7e 100644
--- a/lib/galaxy/selenium/navigates_galaxy.py
+++ b/lib/galaxy/selenium/navigates_galaxy.py
@@ -1461,17 +1461,15 @@ class NavigatesGalaxy(HasDriver):
 
     def history_panel_click_item_title(self, hid, **kwds):
         item_component = self.history_panel_item_component(hid=hid)
+        details_component = item_component.details
+        details_displayed = details_component.is_displayed
         item_component.title.wait_for_and_click()
+
         if kwds.get("wait", False):
-            if self.is_beta_history():
-                item_component.details.wait_for_present()
+            if details_displayed:
+                details_component.wait_for_absent_or_hidden()
             else:
-                details_component = item_component.details
-                details_displayed = details_component.is_displayed
-                if details_displayed:
-                    details_component.wait_for_absent_or_hidden()
-                else:
-                    details_component.wait_for_visible()
+                details_component.wait_for_visible()
         return item_component
 
     def history_panel_ensure_showing_item_details(self, hid):

From bc4437c1cd64059468a3cd37868c22c35ef22cb0 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 3 Sep 2021 12:37:21 -0400
Subject: [PATCH 444/452] Fix missing dataset hids -- incorrect use of sr-only
 here

---
 client/src/components/History/ContentItem/Dataset/DatasetUI.vue | 2 +-
 .../components/History/ContentItem/DatasetCollection/DscUI.vue  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 375fc1b4a60..0ed7b9805a3 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -49,7 +49,7 @@ either through the props, and make updates through the events -->
 
                 <div class="content-title title p-1 overflow-hidden">
                     <h5 class="text-truncate" v-if="collapsed">
-                        <span class="hid sr-only" data-description="dataset hid">{{ dataset.hid }}</span>
+                        <span class="hid" data-description="dataset hid">{{ dataset.hid }}</span>
                         <span class="name" data-description="dataset name">{{ dataset.title }}</span>
                     </h5>
                 </div>
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index 2ed9276e8c8..5b471b9f0e7 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -57,7 +57,7 @@
 
                 <div class="content-title title flex-grow-1 overflow-hidden" @click.stop="$emit('viewCollection')">
                     <h5 class="text-truncate">
-                        <span class="hid sr-only">{{ dsc.hid }}</span>
+                        <span class="hid">{{ dsc.hid }}</span>
                         <span class="name">{{ dsc.name }}</span>
                         <span class="description">
                             ({{ dsc.collectionType | localize }} {{ dsc.collectionCountDescription | localize }})

From 3065ddf3d5ef6e9002844bd35e49878b6a706ae7 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 3 Sep 2021 12:47:00 -0400
Subject: [PATCH 445/452] Add sub-component to dataset error view

---
 .../DatasetInformation/DatasetError.vue       | 24 +++----
 .../DatasetErrorDetails.vue                   | 62 +++++++++++++++++++
 2 files changed, 69 insertions(+), 17 deletions(-)
 create mode 100644 client/src/components/DatasetInformation/DatasetErrorDetails.vue

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 43b2b846ea5..11e69e96d68 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -21,23 +21,11 @@
                         <b id="dataset-error-tool-id" class="text-break">{{ jobDetails.tool_id }}</b
                         >.
                     </p>
-                    <div v-if="jobDetails.tool_stderr || jobDetails.job_stderr || hasMessages(jobDetails)">
-                        <h3>Details</h3>
-                    </div>
-                    <div v-if="hasMessages(jobDetails)" id="dataset-error-job-messages">
-                        <p>Execution resulted in the following messages:</p>
-                        <div v-for="(job_message, index) in jobDetails.job_messages" :key="index">
-                            <pre class="rounded code">{{ job_message["desc"] }}</pre>
-                        </div>
-                    </div>
-                    <div v-if="jobDetails.tool_stderr">
-                        <p>Tool generated the following standard error:</p>
-                        <pre id="dataset-error-tool-stderr" class="rounded code">{{ jobDetails.tool_stderr }}</pre>
-                    </div>
-                    <div v-if="jobDetails.job_stderr">
-                        <p>Galaxy job runner generated the following standard error:</p>
-                        <pre id="dataset-error-job-stderr" class="rounded code">{{ jobDetails.job_stderr }}</pre>
-                    </div>
+                    <DatasetErrorDetails
+                        :tool-stderr="jobDetails.tool_stderr"
+                        :job-stderr="jobDetails.job_stderr"
+                        :job-messages="jobDetails.job_messages"
+                    />
                     <JobProblemProvider :jobid="dataset.creating_job" v-slot="{ result: jobProblems }" @error="onError">
                         <div v-if="jobProblems && (jobProblems.has_duplicate_inputs || jobProblems.has_empty_inputs)">
                             <h3 class="common_problems mt-3">Detected Common Potential Problems</h3>
@@ -97,6 +85,7 @@
 </template>
 
 <script>
+import DatasetErrorDetails from "./DatasetErrorDetails";
 import FormElement from "components/Form/FormElement";
 import { DatasetProvider } from "components/WorkflowInvocationState/providers";
 import { JobDetailsProvider, JobProblemProvider } from "components/providers/JobProvider";
@@ -110,6 +99,7 @@ library.add(faBug);
 export default {
     components: {
         DatasetProvider,
+        DatasetErrorDetails,
         FontAwesomeIcon,
         FormElement,
         JobDetailsProvider,
diff --git a/client/src/components/DatasetInformation/DatasetErrorDetails.vue b/client/src/components/DatasetInformation/DatasetErrorDetails.vue
new file mode 100644
index 00000000000..63afb97ec7b
--- /dev/null
+++ b/client/src/components/DatasetInformation/DatasetErrorDetails.vue
@@ -0,0 +1,62 @@
+<template>
+    <div>
+        <div v-if="toolStderr || jobStderr || hasMessages">
+            <h3>Details</h3>
+        </div>
+        <div v-if="hasMessages" id="dataset-error-job-messages">
+            <p>Execution resulted in the following messages:</p>
+            <div v-for="(jobMessage, index) in jobMessages" :key="index">
+                <pre class="rounded code">{{ jobMessage["desc"] }}</pre>
+            </div>
+        </div>
+        <div v-if="toolStderr">
+            <p>Tool generated the following standard error:</p>
+            <pre id="dataset-error-tool-stderr" class="rounded code">{{ toolStderr }}</pre>
+        </div>
+        <div v-if="jobStderr">
+            <p>Galaxy job runner generated the following standard error:</p>
+            <pre id="dataset-error-job-stderr" class="rounded code">{{ jobStderr }}</pre>
+        </div>
+    </div>
+</template>
+
+<script>
+import FormElement from "components/Form/FormElement";
+import { DatasetProvider } from "components/WorkflowInvocationState/providers";
+import { JobDetailsProvider, JobProblemProvider } from "components/providers/JobProvider";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faBug } from "@fortawesome/free-solid-svg-icons";
+import { sendErrorReport } from "./services";
+
+library.add(faBug);
+
+export default {
+    components: {
+        DatasetProvider,
+        FontAwesomeIcon,
+        FormElement,
+        JobDetailsProvider,
+        JobProblemProvider,
+    },
+    props: {
+        toolStderr: {
+            type: String,
+            required: true,
+        },
+        jobStderr: {
+            type: String,
+            required: true,
+        },
+        jobMessages: {
+            type: Array,
+            required: true,
+        },
+    },
+    computed: {
+        hasMessages() {
+            return this.jobMessages && this.jobMessages.length > 0;
+        },
+    },
+};
+</script>

From 952d947087d11373e6dfc8bbbb3910a047fae445 Mon Sep 17 00:00:00 2001
From: Kaivan Kamali <kxk302@psu.edu>
Date: Fri, 3 Sep 2021 12:49:36 -0400
Subject: [PATCH 446/452] Removed resources section per John's comment

---
 doc/source/dev/debugging_galaxy_slurm.md | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/doc/source/dev/debugging_galaxy_slurm.md b/doc/source/dev/debugging_galaxy_slurm.md
index 961bc34dc96..fd0965a7da5 100644
--- a/doc/source/dev/debugging_galaxy_slurm.md
+++ b/doc/source/dev/debugging_galaxy_slurm.md
@@ -38,11 +38,6 @@ playbooks for that).
         <destinations default="slurm">
             <destination id="slurm" runner="slurm"/>
         </destinations>
-        <resources default="default">
-          <group id="default"></group>
-          <group id="memoryonly">memory</group>
-          <group id="all">processors,memory,time,project</group>
-        </resources>
     </job_conf>
     ```
 

From cad420112f46835b150e7bd48dda72dc12c79698 Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 3 Sep 2021 13:03:07 -0400
Subject: [PATCH 447/452] Remove unused imports from details sub component

---
 .../DatasetInformation/DatasetErrorDetails.vue  | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetErrorDetails.vue b/client/src/components/DatasetInformation/DatasetErrorDetails.vue
index 63afb97ec7b..5b0d857bb44 100644
--- a/client/src/components/DatasetInformation/DatasetErrorDetails.vue
+++ b/client/src/components/DatasetInformation/DatasetErrorDetails.vue
@@ -21,24 +21,7 @@
 </template>
 
 <script>
-import FormElement from "components/Form/FormElement";
-import { DatasetProvider } from "components/WorkflowInvocationState/providers";
-import { JobDetailsProvider, JobProblemProvider } from "components/providers/JobProvider";
-import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
-import { library } from "@fortawesome/fontawesome-svg-core";
-import { faBug } from "@fortawesome/free-solid-svg-icons";
-import { sendErrorReport } from "./services";
-
-library.add(faBug);
-
 export default {
-    components: {
-        DatasetProvider,
-        FontAwesomeIcon,
-        FormElement,
-        JobDetailsProvider,
-        JobProblemProvider,
-    },
     props: {
         toolStderr: {
             type: String,

From 9a6913fb70d9caa5595754a40914f5350d7c6a8f Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 3 Sep 2021 13:27:10 -0400
Subject: [PATCH 448/452] Add content-top-menu back to dataset and dsc items,
 fixing style

---
 .../src/components/History/ContentItem/Dataset/DatasetUI.vue | 5 ++++-
 .../History/ContentItem/DatasetCollection/DscUI.vue          | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 0ed7b9805a3..562022011b8 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -16,7 +16,10 @@ either through the props, and make updates through the events -->
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
         <!-- name, state buttons, menus -->
-        <nav class="p-1 d-flex align-items-baseline cursor-pointer" @click.stop="$emit('update:expanded', !expanded)">
+        <nav
+            class="content-top-menu p-1 d-flex align-items-baseline cursor-pointer"
+            @click.stop="$emit('update:expanded', !expanded)"
+        >
             <div class="d-flex align-items-baseline flex-grow-1 overflow-hidden">
                 <div class="pl-1" v-if="showSelectionBox">
                     <b-check class="selector" :checked="selected" @change="$emit('update:selected', $event)"></b-check>
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index 5b471b9f0e7..a078d013857 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -10,7 +10,7 @@
         @keydown.arrow-right.self.stop="$emit('viewCollection')"
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
-        <nav class="p-1 d-flex align-items-baseline cursor-pointer">
+        <nav class="content-top-menu p-1 d-flex align-items-baseline cursor-pointer">
             <div class="d-flex align-items-baseline flex-grow-1 overflow-hidden">
                 <div class="pl-1" v-if="showSelection">
                     <b-check

From 0b2ab96b061143d47d2aecd657217894f2d240b5 Mon Sep 17 00:00:00 2001
From: Dannon Baker <dannon.baker@gmail.com>
Date: Fri, 3 Sep 2021 13:47:23 -0400
Subject: [PATCH 449/452] Fix alignment of items by not forcing baseline
 multiple times.

---
 .../components/History/ContentItem/Dataset/DatasetUI.vue   | 7 ++-----
 .../History/ContentItem/DatasetCollection/DscUI.vue        | 4 ++--
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
index 562022011b8..04f41c9ea7f 100644
--- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
+++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue
@@ -16,11 +16,8 @@ either through the props, and make updates through the events -->
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
         <!-- name, state buttons, menus -->
-        <nav
-            class="content-top-menu p-1 d-flex align-items-baseline cursor-pointer"
-            @click.stop="$emit('update:expanded', !expanded)"
-        >
-            <div class="d-flex align-items-baseline flex-grow-1 overflow-hidden">
+        <nav class="content-top-menu p-1 d-flex cursor-pointer" @click.stop="$emit('update:expanded', !expanded)">
+            <div class="d-flex flex-grow-1 overflow-hidden">
                 <div class="pl-1" v-if="showSelectionBox">
                     <b-check class="selector" :checked="selected" @change="$emit('update:selected', $event)"></b-check>
                 </div>
diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
index a078d013857..a8cfa314d6c 100644
--- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
+++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue
@@ -10,8 +10,8 @@
         @keydown.arrow-right.self.stop="$emit('viewCollection')"
         @keydown.space.self.stop.prevent="$emit('update:selected', !selected)"
     >
-        <nav class="content-top-menu p-1 d-flex align-items-baseline cursor-pointer">
-            <div class="d-flex align-items-baseline flex-grow-1 overflow-hidden">
+        <nav class="content-top-menu p-1 d-flex cursor-pointer">
+            <div class="d-flex flex-grow-1 overflow-hidden">
                 <div class="pl-1" v-if="showSelection">
                     <b-check
                         class="selector"

From 5f2feddeb38b830ccd27c3e882c306bcab67535b Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 3 Sep 2021 18:38:44 -0400
Subject: [PATCH 450/452] Remove unused hasmessage function from dataset error
 view

---
 client/src/components/DatasetInformation/DatasetError.vue | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue
index 11e69e96d68..a35b56a565f 100644
--- a/client/src/components/DatasetInformation/DatasetError.vue
+++ b/client/src/components/DatasetInformation/DatasetError.vue
@@ -120,9 +120,6 @@ export default {
         };
     },
     methods: {
-        hasMessages(jobDetails) {
-            return jobDetails.job_messages && jobDetails.job_messages.length > 0;
-        },
         onError(err) {
             this.errorMessage = err;
         },

From 65de2f0d6ea6d0f823b7f4bc7ffeebe59e28c05e Mon Sep 17 00:00:00 2001
From: guerler <aysam.guerler@gmail.com>
Date: Fri, 3 Sep 2021 18:41:31 -0400
Subject: [PATCH 451/452] Place detail check into computed value

---
 .../DatasetInformation/DatasetErrorDetails.vue    | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/client/src/components/DatasetInformation/DatasetErrorDetails.vue b/client/src/components/DatasetInformation/DatasetErrorDetails.vue
index 5b0d857bb44..e2f98519d44 100644
--- a/client/src/components/DatasetInformation/DatasetErrorDetails.vue
+++ b/client/src/components/DatasetInformation/DatasetErrorDetails.vue
@@ -1,8 +1,6 @@
 <template>
-    <div>
-        <div v-if="toolStderr || jobStderr || hasMessages">
-            <h3>Details</h3>
-        </div>
+    <div v-if="hasDetails">
+        <h3>Details</h3>
         <div v-if="hasMessages" id="dataset-error-job-messages">
             <p>Execution resulted in the following messages:</p>
             <div v-for="(jobMessage, index) in jobMessages" :key="index">
@@ -25,18 +23,21 @@ export default {
     props: {
         toolStderr: {
             type: String,
-            required: true,
+            default: null,
         },
         jobStderr: {
             type: String,
-            required: true,
+            default: null,
         },
         jobMessages: {
             type: Array,
-            required: true,
+            default: null,
         },
     },
     computed: {
+        hasDetails() {
+            return this.toolStderr || this.jobStderr || this.hasMessages;
+        },
         hasMessages() {
             return this.jobMessages && this.jobMessages.length > 0;
         },

From 718b70b17bfe30ddfc8078d6ab72456770cf14ec Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Mon, 6 Sep 2021 11:28:47 +0100
Subject: [PATCH 452/452] Downgrade Cheetah3 to 3.2.6.post1

Version 3.2.6.post2 was yanked:

https://pypi.org/project/Cheetah3/3.2.6.post2/

Fix the following warning on startup:

```
2021-09-03T22:30:41.3786134Z WARNING: The candidate selected for download or install is a yanked version: 'cheetah3' candidate (version 3.2.6.post2 at https://files.pythonhosted.org/packages/a4/d4/0325ef3dfd3edf545b5ff0aec15f248d8bf2e5cdda9e56a7b68187181703/Cheetah3-3.2.6.post2-cp36-cp36m-manylinux1_x86_64.whl#sha256=b37e99194c71eea8156c4cd66381257a3a4c45b13c5f7e9fdbcd2772ab0752e8 (from https://pypi.org/simple/cheetah3/) (requires-python:>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*))
2021-09-03T22:30:41.3789180Z Reason for being yanked: Failed
```
---
 lib/galaxy/dependencies/dev-requirements.txt    | 2 +-
 lib/galaxy/dependencies/pinned-requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index d1346637796..df7658ae5ca 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -32,7 +32,7 @@ celery==5.0.5; python_version >= "3.6"
 certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
 cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
 chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
-cheetah3==3.2.6.post2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 circus==0.17.1
 click-didyoumean==0.0.3; python_version >= "3.6"
 click-plugins==1.1.1; python_version >= "3.6"
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index b3db1a5e313..a0a0d68a7ff 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -30,7 +30,7 @@ celery==5.0.5; python_version >= "3.6"
 certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
 chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
-cheetah3==3.2.6.post2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 circus==0.17.1
 click-didyoumean==0.0.3; python_version >= "3.6"
 click-plugins==1.1.1; python_version >= "3.6"