Exploratory Jupyter environment for `galaxy-selenium`.

The new Makefile-based command ``make serve-selenium-notebooks`` will serve notebooks in lib/galaxy_test/selenium/jupyter.

The notebooks will target a Galaxy host running on port 8080 - but are parameterized to be run with papermill (https://papermill.readthedocs.io/) for automate targeting different Galaxy servers, remote Selenium servers, etc..

Screenshots will be embedded right into the notebook.

Example notebooks setup to target just the library code in ``galaxy-selenium`` as well as one appropriate to leverage the Galaxy testing framework are included and documented in depth via modules docstring for ``galaxy_test.selenium.jupyter``.

.gitignore

@@ -96,6 +96,7 @@ config/plugins/**/.cache
 static/welcome.html.*
 static/welcome.html
 static/client_build_hash.txt
+lib/galaxy_test/selenium/jupyter/galaxy_selenium_context.yml
 
 # Tool data.
 tool-data/annotation_profiler_options.xml

Makefile

@@ -182,6 +182,9 @@ client-lint: client-eslint client-format-check ## ES lint and check format of cl
 client-test-watch: client ## Watch and run all client unit tests on changes
 	cd client && yarn run jest-watch
 
+serve-selenium-notebooks: ## Serve testing notebooks for Jupyter
+	cd lib && export PYTHONPATH=`pwd`; jupyter notebook --notebook-dir=galaxy_test/selenium/jupyter
+
 # Release Targets
 release-create-rc: release-ensure-upstream ## Create a release-candidate branch
 	git checkout dev

doc/source/lib/galaxy.selenium.rst

@@ -66,6 +66,14 @@ galaxy.selenium.has\_driver module
    :undoc-members:
    :show-inheritance:
 
+galaxy.selenium.jupyter\_context module
+---------------------------------------
+
+.. automodule:: galaxy.selenium.jupyter_context
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 galaxy.selenium.navigates\_galaxy module
 ----------------------------------------
 

doc/source/lib/galaxy_test.selenium.jupyter.rst

@@ -0,0 +1,7 @@
+galaxy\_test.selenium.jupyter package
+=====================================
+
+.. automodule:: galaxy_test.selenium.jupyter
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/source/lib/galaxy_test.selenium.rst

@@ -6,6 +6,14 @@ galaxy\_test.selenium package
    :undoc-members:
    :show-inheritance:
 
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   galaxy_test.selenium.jupyter
+
 Submodules
 ----------
 
@@ -17,6 +25,14 @@ galaxy\_test.selenium.framework module
    :undoc-members:
    :show-inheritance:
 
+galaxy\_test.selenium.jupyter\_context module
+---------------------------------------------
+
+.. automodule:: galaxy_test.selenium.jupyter_context
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 galaxy\_test.selenium.test\_admin\_app module
 ---------------------------------------------
 
@@ -273,6 +289,14 @@ galaxy\_test.selenium.test\_workflow\_editor module
    :undoc-members:
    :show-inheritance:
 
+galaxy\_test.selenium.test\_workflow\_invocation\_details module
+----------------------------------------------------------------
+
+.. automodule:: galaxy_test.selenium.test_workflow_invocation_details
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 galaxy\_test.selenium.test\_workflow\_management module
 -------------------------------------------------------
 

lib/galaxy/selenium/context.py

@@ -1,6 +1,8 @@
+import os
 from abc import abstractmethod
 from typing import Optional
 
+import yaml
 from six.moves.urllib.parse import urljoin
 
 from .driver_factory import ConfiguredDriver
@@ -46,7 +48,7 @@ class GalaxySeleniumContextImpl(GalaxySeleniumContext):
     """Minimal, simplified GalaxySeleniumContext useful outside the context of test cases.
 
     A variant of this concept that can also populate content via the API
-    to then interact with via the Selenium is :class:galaxy_test.selenium.framework:`GalaxySeleniumContextImpl`.
+    to then interact with via the Selenium is :class:`galaxy_test.selenium.framework.GalaxySeleniumContextImpl`.
     """
 
     def __init__(self, from_dict: Optional[dict] = None) -> None:
@@ -58,3 +60,15 @@ class GalaxySeleniumContextImpl(GalaxySeleniumContext):
 
     def _screenshot_path(self, label, extension=".png"):
         return label + extension
+
+
+def init(config=None, clazz=GalaxySeleniumContextImpl) -> GalaxySeleniumContext:
+    if os.path.exists("galaxy_selenium_context.yml"):
+        with open("galaxy_selenium_context.yml", "r") as f:
+            as_dict = yaml.safe_load(f)
+        context = clazz(as_dict)
+    else:
+        config = config or {}
+        context = clazz(config)
+
+    return context

lib/galaxy/selenium/data.py

@@ -3,7 +3,8 @@ from pkg_resources import resource_string
 
 from .components import Component
 
-new_data_yaml = resource_string(__name__, 'navigation.yml').decode("UTF-8")
-NAVIGATION_RAW = yaml.safe_load(new_data_yaml)
 
-NAVIGATION = Component.from_dict("root", NAVIGATION_RAW)
+def load_root_component():
+    new_data_yaml = resource_string(__name__, 'navigation.yml').decode("UTF-8")
+    navigation_raw = yaml.safe_load(new_data_yaml)
+    return Component.from_dict("root", navigation_raw)

lib/galaxy/selenium/driver_factory.py

@@ -159,6 +159,7 @@ def _which(file):
 
 
 __all__ = (
+    'ConfiguredDriver',
     'get_local_driver',
     'get_remote_driver',
     'is_virtual_display_available',

lib/galaxy/selenium/jupyter_context.py

@@ -0,0 +1,13 @@
+from .context import GalaxySeleniumContextImpl, init as c_init
+
+
+def init(config=None):
+    return c_init(config=config, clazz=JupyterContextImpl)
+
+
+class JupyterContextImpl(GalaxySeleniumContextImpl):
+
+    def screenshot(self, label):
+        path = super().screenshot(label)
+        from IPython.display import Image
+        return Image(filename=path)

lib/galaxy/selenium/navigates_galaxy.py

@@ -14,8 +14,9 @@ import requests
 import yaml
 
 from . import sizzle
+from .components import Component
 from .data import (
-    NAVIGATION,
+    load_root_component,
 )
 from .has_driver import (
     exception_indicates_not_clickable,
@@ -123,17 +124,22 @@ class NavigatesGalaxy(HasDriver):
     class. For instance, the method for clicking an option in the workflow editor is
     workflow_editor_click_option instead of click_workflow_editor_option.
     """
-
     default_password = DEFAULT_PASSWORD
     wait_types = WAIT_TYPES
+    # set to True to reload each invocation (good for interactive test building)
+    _interactive_components: bool = False
+    _root_component: Component = load_root_component()
 
     def get(self, url=""):
         full_url = self.build_url(url)
         return self.driver.get(full_url)
 
     @property
-    def navigation(self):
-        return NAVIGATION
+    def navigation(self) -> Component:
+        if self._interactive_components:
+            return load_root_component()
+        else:
+            return self._root_component
 
     @property
     def components(self):

lib/galaxy_test/selenium/jupyter/__init__.py

@@ -0,0 +1,168 @@
+"""Examples and framework helpers for driving Galaxy with Selenium in Jupyter.
+
+Example Notebooks
+-----------------
+
+There are a couple example notebooks in this directory that demonstrate two different
+helper classes that provide a Python context for interacting with Galaxy.
+
+``notebook_example_library.ipynb`` demonstrates using :class:`galaxy.selenium.jupyter_context.JupyterContext`,
+an interface that only depends on the pip installable library ``galaxy-selenium``.
+``JupyterContext`` inherits :class:`galaxy.selenium.navigates_galaxy.NavigatesGalaxy`
+and :class:`galaxy.selenium.has_driver.HasDriver` and contains numerous functions to
+interact with Galaxy in generic terms using Selenium.
+
+``notebook_example_testing.ipynb`` demonstrates using :class:`galaxy_test.selenium.jupyter_context.JupyterTestContextImpl`,
+an interface that extends ``JupyterContextImpl`` to additionally provide testing framework
+populators (:mod:`galaxy_test.base.populators`) configured to work with the same
+user as is logged in with Selenium and that provide a quick way to populate the target
+Galaxy with testing fixtures.
+
+The first example notebook is good for prototyping stand-alone applications that drive
+Galaxy, the second example notebook is setup to mirror the environment available to Selenium
+test cases that ship with Galaxy (e.g. :class:`galaxy_test.selenium.framework:SeleniumTestCase`)
+and can be used for interactively developing Selenium tests for Galaxy.
+
+These notebooks start with a cell that simply defines a ``config`` variable in the Python
+environment on a cell that is marked with metadata indicated it is "parameters". This
+isn't required at all to use the notebook iteratively, but it does provide a nice hook
+for executing the notebook in a parameterized way using papermill_ after its creation.
+
+The second cell in both of these notebooks setup the Galaxy interaction context - either
+a :class:`galaxy.selenium.jupyter_context.JupyterContextImpl` or
+:class:`galaxy_test.selenium.jupyter_context.JupyterTestContextImpl` depending on
+the notebook. These cells use a function named ``init`` to create the desired
+interaction context (called ``gx_selenium_context`` in both cases).
+
+::
+
+    from galaxy.selenium.jupyter_context import init
+    gx_selenium_context = init(config)
+
+The optional ``config`` dictionary can be used to configure both Selenium and the Galaxy
+that is targeted. If ``config`` is none and the working directory Jupyter is executing
+inside of has a file named ``galaxy_selenium_context.yml`` - ``init`` will read that file
+and use it as the target configuration.
+
+Both contexts can be configured with the following dictionary keys (either from
+``galaxy_selenium_context.yml`` or using papermill_).
+
+``local_galaxy_url``
+    Galaxy URL to target during execution.
+
+``selenium_galaxy_url``
+    Galaxy URL relative to the Selenium being used. This doesn't need to be set unless
+    you're using a remote Selenium server (i.e one not running directly on ``localhost``).
+
+``driver``
+    If set, this should be a dictionary describing variables to send to
+    :class:`galaxy.selenium.driver_factory.ConfiguredDriver` and define how to
+    setup the Selenium client used. Useful keys and their defaults include -
+    ``browser`` (``"auto"``), ``remote`` (``false``), ``remote_host`` (``"127.0.0.1"``),
+    ``remote_port`` (``4444``), ``headless`` (``false``).
+
+Additionally, the testing context also allows additional configuration variables used
+for populating test fixtures (with populators) or by extra testing helpers (e.g.
+``gx_selenium_context.test_login()``).
+
+``admin_api_key``
+    An API key to be used for API interactions that require an admin account.
+
+``login_email``
+    A test account login or email address to be used for tests that require a logged
+    in user.
+
+``login_password``
+    The password to be used for the account referenced by ``login_email``.
+
+.. note:: Using Papermill
+
+    ::
+
+        $ . venv/bin/activate
+        $ pip install papermill
+        $ papermill galaxy_test/selenium/jupyter/notebook_example_library.ipynb  galaxy_test/selenium/jupyter/notebook_example_library_output.ipynb -y "
+        config: {local_galaxy_url: 'http://localhost:8081/'}
+        "
+
+Developing Galaxy Test Cases Interactively with Jupyter
+-------------------------------------------------------
+
+The Setup
+~~~~~~~~~
+
+Writing Selenium tests for Galaxy often involves annotating the Galaxy DOM with
+appropriate ``class`` and ``data-*`` attributes, so it is best to have webpack monitor
+``client/src`` and rebuild as things changes and then disable building the client
+as Galaxy starts. This can be done by executing the follow two commands in separate
+terminal sessions.
+
+::
+
+    $ GALAXY_SKIP_CLIENT_BUILD=1 GALAXY_RUN_WITH_TEST_TOOLS=1 sh run.sh
+    $ make client-watch
+
+Copy ``lib/galaxy_test/selenium/jupyter/galaxy_selenium.yml.sample`` to
+``lib/galaxy_test/selenium/jupyter/galaxy_selenium.yml`` and specify a
+``login_email``, ``login_password``, ``admin_api_key`` to point at values
+suitable for your local test environment.
+
+::
+
+    $ cp lib/galaxy_test/selenium/jupyter/galaxy_selenium.yml.sample lib/galaxy_test/selenium/jupyter/galaxy_selenium.yml
+    $ vi lib/galaxy_test/selenium/jupyter/galaxy_selenium.yml
+
+Start Jupyter for interactive development.
+
+::
+
+    $ make serve-selenium-notebooks
+
+Filling out Tests
+~~~~~~~~~~~~~~~~~~~~
+
+Open (or copy and open) the notebook ``notebook_example_testing.ipynb`` and
+start developing the test in the third notebook cell. This cell will land up
+being the body a test case.
+
+Use ``gx_selenium_context`` to navigate Galaxy, setup test conditions, and
+verify expected behavior. Change ``navigation.yml`` to implement components and
+``client/src`` to annotate the source - these changes will be picked up automatically
+and you can simply re-run the notebook to redo the modified or extended test.
+If Python test code needs to be changed (e.g. helper methods are added to
+:class:`galaxy.selenium.navigates_galaxy.NavigatesGalaxy`), you'll need to
+select "Restart & Run All" from the "Kernel" menu of Jupyter to pickup the new
+changes.
+
+
+Once the test is complete, copy all the cells after the initialization into a
+new test case file (either in ``lib/galaxy_test/selenium`` or
+``test/integration_selenium``). Some examples of pull request commits developed this
+way include Galaxy's initial `workflow invocation detail tests
+<https://github.com/galaxyproject/galaxy/pull/11160/commits/b6d99320d99e93f812adc0df494447525978ebfa>`__
+and initial `history import and export tests <https://github.com/galaxyproject/galaxy/pull/11152/files>`__.
+These commits demonstrate the basics of how to turn operations on a ``gx_selenium_context``
+into a Galaxy test case, how to add UI selectors to ``navigation.yml``, how to add
+functionality to ``navigates_galaxy.py`` to be shared across test cases.
+The `history import and export tests <https://github.com/galaxyproject/galaxy/pull/11152/files>`__
+additionally are examples of both regular Selenium test cases as well as their
+integration-style variant that configures Galaxy before the test runs.
+
+Much more information on structuring Selenium test cases generally, the difference
+between tests that should be place in ``lib/galaxy_test/selenium`` versus ones
+that should be placed in ``test/integration_selenium``, and information on running
+Selenium test cases can all be found in :doc:`../dev/writing_tests`.
+
+.. note::
+
+    Test Case Attributes
+
+    If you'd like to simplify your test by logging into Galaxy as a pre-condition,
+    add ``ensure_registered = True`` on the test case class. Your test case can
+    use an admin key to populate fixtures without any extra annotation but to ensure
+    a user logged in with ``ensure_registered`` is an admin user simply annotate
+    your test case with ``requires_admin = True``.
+
+.. _papermill: https://papermill.readthedocs.io/en/latest/
+
+"""

lib/galaxy_test/selenium/jupyter/galaxy_selenium_context.yml.sample

@@ -0,0 +1,8 @@
+local_galaxy_url: http://localhost:8080
+login_email: test_user@example.com
+login_password: mycoolpassw0rd
+
+# Setup Galaxy's URL relative to remote running Selenium service if needed.
+#selenium_galaxy_url:
+# Setup an admin API key if needed for testing populators & fixture creation.
+#admin_api_key:

lib/galaxy_test/selenium/jupyter_context.py

@@ -0,0 +1,33 @@
+"""Variant of JuypterContextImpl that can also use populators.
+
+This provides an environment separate from test cases that can leaverage
+both Selenium for testing Galaxy with a browser and API populators for filling
+in fixture data rapidly in the target Galaxy.
+"""
+from typing import Optional
+
+from galaxy.selenium.context import init as base_init
+from galaxy.selenium.jupyter_context import JupyterContextImpl
+from galaxy_test.base.api_util import get_admin_api_key
+from .framework import GalaxyTestSeleniumContext
+
+
+class JupyterTestContextImpl(JupyterContextImpl, GalaxyTestSeleniumContext):
+    # Reload components interactively to limit number of Python kernel
+    # restarts needed during test building.
+    _interactive_components = True
+
+    def __init__(self, from_dict: Optional[dict] = None) -> None:
+        from_dict = from_dict or {}
+        super().__init__(from_dict)
+        self.admin_api_key = from_dict.get("admin_api_key", get_admin_api_key())
+        self.login_email = from_dict.get("login_email")
+        self.login_password = from_dict.get("login_password")
+
+    def test_login(self):
+        self.home()
+        self.submit_login(self.login_email, self.login_password)
+
+
+def init(config=None):
+    return base_init(config, clazz=JupyterTestContextImpl)

packages/test_selenium/setup.py

@@ -32,6 +32,7 @@ TEST_DIR = 'tests'
 PACKAGES = [
     'galaxy_test',
     'galaxy_test.selenium',
+    'galaxy_test.selenium.jupyter',
 ]
 ENTRY_POINTS = '''
         [console_scripts]