From ad3025f46a00190674bbc4dbc3bfab16983f9cb4 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 6 Mar 2017 09:28:42 -0500
Subject: [PATCH] Touch up Galaxy testing documentation.

---
 run_tests.sh    | 46 ++++++++++++++++++++++++++++++++++++++--------
 test/TESTING.md |  5 ++---
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 37e2be5bddb..69f9c8d95b8 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -8,7 +8,6 @@ rm -f run_functional_tests.log
 
 show_help() {
 cat <<EOF
-'${0##*/} (test_path)'              for testing all the tools in functional directory
 '${0##*/} -id bbb'                  for testing one tool with id 'bbb' ('bbb' is the tool id)
 '${0##*/} -sid ccc'                 for testing one section with sid 'ccc' ('ccc' is the string after 'section::')
 '${0##*/} -list'                    for listing all the tool ids
@@ -24,14 +23,49 @@ cat <<EOF
 '${0##*/} -qunit testname'          for running single JavaScript test with given name
 '${0##*/} -selenium'                for running all selenium web tests (in test/selenium_tests)
 '${0##*/} -selenium (test_path)'    for running specified selenium web tests (use nosetest path)
-'${0##*/} -casperjs (py_test_path)' for running casperjs JavaScript tests using a Python wrapper for consistency. py_test_path in casperjs_runner.py e.g. 'Test_04_HDAs' or 'Test_04_HDAs.test_00_HDA_states'.
+
+This wrapper script largely serves as a point documentation and convenience -
+most tests shipped with Galaxy can be run with nosetests or qunit directly.
+
+The main test types are as follows:
+
+- API: These tests are located in test/api and test various aspects of the Galaxy
+   API and test general backend aspects of Galaxy using the API.
+- Integration: These tests are located in test/integration and test special
+   configurations of Galaxy. All API tests assume a particular Galaxy configuration
+   defined by test/base/driver_util.py and integration tests can be used to
+   launch and test Galaxy in other configurations.
+- Framework: These tests are all Galaxy tool tests and can be found in
+   test/functional/tools. These are for the most part meant to test and
+   demonstrate features of the tool evaluation environment and of Galaxy tool XML
+   files.
+- Unit: These are Python unit tests either defined as doctests or inside of
+   test/unit. These should generally not require a Galaxy instance and should
+   quickly test just a component or a few components of Galaxy's backend code.
+- QUnit: These are JavaScript unit tests defined in test/qunit.
+- Selenium: These are full stack tests meant to test the Galaxy UI with real
+   browsers and are located in test/selenium_tests.
+- ToolShed: These are web tests that use the older Python web testing
+   framework twill to test ToolShed related functionality. These are
+   located in test/shed_functional.
 
 Nose tests will allow specific tests to be selected per the documentation at
 https://nose.readthedocs.org/en/latest/usage.html#selecting-tests.  These are
 indicated with the optional parameter (test_path).  A few examples are:
 
-Run all TestUserInfo functional tests:
-    ./run_tests.sh test/functional/test_user_info.py:TestUserInfo
+Run all API tests:
+    ./run_tests.sh -api
+
+The same test as above can be run using nosetests directly as follows:
+    nosetests test/api
+
+However when using nosetests directly output options defined in this
+file aren't respected and a new Galaxy instance will be created for each
+TestCase class (this scripts optimizes it so all tests can share a Galaxy
+instance).
+
+Run a full class of API tests:
+    ./run_tests.sh -api test/api/test_tools.py:ToolsTestCase
 
 Run a specific API test:
     ./run_tests.sh -api test/api/test_tools.py:ToolsTestCase.test_map_over_with_output_format_actions
@@ -140,10 +174,6 @@ GALAXY_CONFIG_MASTER_KEY        Master or admin API key to use as part of
                                 testing with GALAXY_TEST_EXTERNAL.
 GALAXY_TEST_USER_API_KEY        User API key to use as part of testing with
                                 GALAXY_TEST_EXTERNAL.
-GALAXY_TEST_HISTORY_ID          Point casperjs tests at specific external
-                                history for testing.
-GALAXY_TEST_WORKFLOW_FILE       Point casperjs tests at specific workflow
-                                file for testing.
 GALAXY_TEST_VERBOSE_ERRORS      Enable more verbose errors during API tests.
 GALAXY_TEST_UPLOAD_ASYNC        Upload tool test inputs asynchronously (may
                                 overwhelm sqlite database).
diff --git a/test/TESTING.md b/test/TESTING.md
index 8bab918de72..e446244cff6 100644
--- a/test/TESTING.md
+++ b/test/TESTING.md
@@ -2,6 +2,5 @@ Galaxy Testing
 ==============
 
 The Galaxy code base is large and contains many kinds of tests. Please
-consult the [Galaxy
-wiki](http://wiki.galaxyproject.org/Admin/Running%20Tests) for more
-information on testing Galaxy.
+run the command ``sh run_tests.sh --help`` from Galaxy's root for a
+description of each test type and location.