Merge pull request #12549 from jmchilton/test_app_1

Establish galaxy-app unit tests.
This commit is contained in:
John Chilton
2021-10-08 10:48:56 -04:00
committed by GitHub
114 changed files with 162 additions and 89 deletions
+4 -4
View File
@@ -17,8 +17,8 @@ variables:
- ~/.cache/yarn
install_tox: &install_tox
run: sudo pip install tox
install_ffprobe: &install_ffprobe
run: sudo apt-get update && sudo apt-get install ffmpeg -y
install_deb_reqs: &install_deb_reqs
run: sudo apt-get update && sudo apt-get install ffmpeg nodejs -y
set_workdir: &set_workdir
working_directory: ~/repo
requires_get_code: &requires_get_code
@@ -100,7 +100,7 @@ jobs:
steps:
- *restore_repo_cache
- *install_tox
- *install_ffprobe
- *install_deb_reqs
- run: tox -e test_galaxy_release
test_galaxy_packages:
docker:
@@ -109,7 +109,7 @@ jobs:
steps:
- *restore_repo_cache
- *install_tox
- *install_ffprobe
- *install_deb_reqs
- run: tox -e test_galaxy_packages
js_lint:
docker:
@@ -25,7 +25,7 @@ from galaxy.util import StructuredExecutionTimer
from galaxy.util.bunch import Bunch
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.web_stack import ApplicationStack
from galaxy_test.base.celery_helper import rebind_container_to_task
from .celery_helper import rebind_container_to_task
# =============================================================================
@@ -11,10 +11,10 @@ from collections import defaultdict
import galaxy.datatypes.registry
import galaxy.model
from galaxy.app_unittest_utils.galaxy_mock import MockApp
from galaxy.tool_util.parser import get_tool_source
from galaxy.tools import create_tool_from_source
from galaxy.util.bunch import Bunch
from .unittest_utils import galaxy_mock
datatypes_registry = galaxy.datatypes.registry.Registry()
@@ -26,7 +26,7 @@ class UsesApp:
def setup_app(self):
self.test_directory = tempfile.mkdtemp()
self.app = galaxy_mock.MockApp()
self.app = MockApp()
self.app.config.new_file_path = os.path.join(self.test_directory, "new_files")
self.app.config.admin_users = "mary@example.com"
self.app.job_search = None
+2 -1
View File
@@ -1527,6 +1527,7 @@ def safe_str_cmp(a, b):
return rv == 0
# Don't use these two directly, prefer method version that "works" with packaged Galaxy.
galaxy_root_path = os.path.join(__path__[0], os.pardir, os.pardir, os.pardir) # type: ignore
galaxy_samples_path = os.path.join(__path__[0], os.pardir, 'config', 'sample') # type: ignore
@@ -1539,7 +1540,7 @@ def galaxy_directory():
def galaxy_samples_directory():
return os.path.abspath(galaxy_samples_path)
return os.path.join(galaxy_directory(), 'lib', 'galaxy', 'config', 'sample')
def config_directories_from_setting(directories_setting, galaxy_root=galaxy_root_path):
+1 -1
View File
@@ -26,6 +26,7 @@ import yaml
from paste import httpserver
from galaxy.app import UniverseApplication as GalaxyUniverseApplication
from galaxy.app_unittest_utils.celery_helper import rebind_container_to_task
from galaxy.config import LOGGING_CONFIG_DEFAULT
from galaxy.model import mapping
from galaxy.model.database_utils import create_database, database_exists
@@ -35,7 +36,6 @@ from galaxy.util import asbool, download_to_file, galaxy_directory
from galaxy.util.properties import load_app_properties
from galaxy.webapps.galaxy import buildapp
from galaxy_test.base.api_util import get_admin_api_key, get_user_api_key
from galaxy_test.base.celery_helper import rebind_container_to_task
from galaxy_test.base.env import (
DEFAULT_WEB_HOST,
target_url_parts,
+1 -1
View File
@@ -1,5 +1,5 @@
include *.rst *.txt LICENSE
include galaxy/*.yml
include galaxy/config/sample/*.sample
include galaxy/config/sample/*.sample*
include galaxy/jobs/runners/util/job_script/*.sh
include galaxy/tools/*tsv
+1
View File
@@ -0,0 +1 @@
../../../lib/galaxy/app_unittest_utils/
+1
View File
@@ -0,0 +1 @@
../../../lib/galaxy/authnz/
+1
View File
@@ -0,0 +1 @@
../../../lib/galaxy/dependencies/
+1
View File
@@ -0,0 +1 @@
../../lib/galaxy_ext/
+2
View File
@@ -6,6 +6,8 @@ galaxy-web-framework
galaxy-web-stack
celery
cloudauthz==0.6.0
cwltool==3.0.20201109103151
kombu
Beaker
pykwalify
+11
View File
@@ -32,7 +32,10 @@ TEST_DIR = 'tests'
PACKAGES = [
'galaxy',
'galaxy.actions',
'galaxy.app_unittest_utils',
'galaxy.authnz',
'galaxy.config',
'galaxy.dependencies',
'galaxy.files',
'galaxy.files.sources',
'galaxy.forms',
@@ -90,6 +93,10 @@ PACKAGES = [
'galaxy.workflow.reports',
'galaxy.workflow.resources',
'galaxy.workflow.schedulers',
'galaxy_ext',
'galaxy_ext.container_monitor',
'galaxy_ext.expressions',
'galaxy_ext.metadata',
]
ENTRY_POINTS = '''
[console_scripts]
@@ -99,6 +106,10 @@ ENTRY_POINTS = '''
PACKAGE_DATA = {
# Be sure to update MANIFEST.in for source dist.
'galaxy': [
'config_schema.yml',
'job_config_schema.yml',
'uwsgi_schema.yml',
'config/sample/*',
],
'tool_shed': [
'scripts/bootstrap_tool_shed/user_info.xml',
+1
View File
@@ -1 +1,2 @@
pytest
testfixtures
+1
View File
@@ -0,0 +1 @@
../../../test/unit/app/
+1 -1
View File
@@ -14,7 +14,7 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)}
virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR"
. "${TEST_ENV_DIR}/bin/activate"
pip install --upgrade pip setuptools wheel
pip install --upgrade pip 'setuptools<58' wheel
pip install -r../lib/galaxy/dependencies/pinned-lint-requirements.txt
# ensure ordered by dependency dag
@@ -13,10 +13,10 @@ from urllib.parse import (
import jwt
import requests
from galaxy.app_unittest_utils.galaxy_mock import MockTrans
from galaxy.authnz import custos_authnz
from galaxy.model import CustosAuthnzToken, User
from galaxy.util import unicodify
from ..unittest_utils.galaxy_mock import MockTrans
class CustosAuthnzTestCase(unittest.TestCase):
+1
View File
@@ -0,0 +1 @@
../../../integration/file_sources_conf.yml
@@ -97,7 +97,19 @@ class BaseJobConfXmlParserTestCase(unittest.TestCase):
def _write_config_from(self, path, template=None):
template = template or {}
contents = open(path).read()
try:
contents = open(path).read()
except FileNotFoundError:
dir_path = os.path.dirname(path)
if os.path.exists(dir_path):
contents = os.listdir(dir_path)
raise Exception(f"Failed to find file {path}, directory {dir_path} exists and contains {contents}")
else:
dir_that_exists = dir_path
while not os.path.exists(dir_that_exists):
dir_that_exists = os.path.dirname(dir_that_exists)
contents = os.listdir(dir_that_exists)
raise Exception(f"Failed to find file {path}, directory {dir_path} does not exist - {dir_that_exists} is the first root that exists and contains {contents}.")
if template:
contents = contents.format(**template)
self._write_config(contents)
@@ -2,6 +2,7 @@ import os
from contextlib import contextmanager
from unittest import TestCase
from galaxy.app_unittest_utils.tools_support import UsesApp
from galaxy.jobs import (
JobWrapper,
TaskWrapper
@@ -13,7 +14,6 @@ from galaxy.model import (
)
from galaxy.tools import evaluation
from galaxy.util.bunch import Bunch
from ..tools_support import UsesApp
TEST_TOOL_ID = "cufftest"
TEST_VERSION_COMMAND = "bwa --version"
@@ -7,12 +7,12 @@ import psutil
from galaxy import job_metrics
from galaxy import model
from galaxy.app_unittest_utils.tools_support import (
UsesApp,
UsesTools,
)
from galaxy.jobs.runners import local
from galaxy.util import bunch
from ..tools_support import (
UsesApp,
UsesTools
)
class TestLocalJobRunner(TestCase, UsesApp, UsesTools):
@@ -6,8 +6,8 @@ import unittest
import sqlalchemy
from galaxy.app_unittest_utils import galaxy_mock
from galaxy.managers.users import UserManager
from ..unittest_utils import galaxy_mock
# =============================================================================
admin_email = 'admin@admin.admin'
@@ -26,9 +26,6 @@ class BaseTestCase(unittest.TestCase):
def tearDownClass(cls):
print('\n', '-' * 20, 'end class', cls)
def __init__(self, *args):
unittest.TestCase.__init__(self, *args)
def setUp(self):
self.log('.' * 20, 'begin test', self)
self.set_up_mocks()
@@ -10,6 +10,7 @@ from galaxy import (
exceptions,
model
)
from galaxy.app_unittest_utils import galaxy_mock
from galaxy.managers import (
base,
hdas
@@ -21,7 +22,6 @@ from galaxy.managers.histories import (
HistorySerializer
)
from .base import BaseTestCase
from ..unittest_utils import galaxy_mock
default_password = '123456'
user2_data = dict(email='user2@user2.user2', username='user2', password=default_password)
@@ -3,9 +3,8 @@ User Manager testing.
Executable directly using: python -m test.unit.managers.test_UserManager
"""
import json
import unittest
from datetime import datetime, timedelta
from datetime import datetime
from sqlalchemy import desc
@@ -13,7 +12,6 @@ from galaxy import exceptions, model
from galaxy.managers import base as base_manager
from galaxy.managers import histories, users
from galaxy.security.passwords import check_password
from galaxy.webapps.galaxy.controllers.user import User
from .base import BaseTestCase
@@ -139,32 +137,6 @@ class UserManagerTestCase(BaseTestCase):
self.assertIsNotNone(user2.id)
self.assertEqual(user2.email, user2_data["email"])
self.assertTrue(check_password(default_password, user2.password))
controller = User(self.app)
response = json.loads(controller.login(self.trans))
self.assertEqual(response["err_msg"], "Please specify a username and password.")
response = json.loads(controller.login(self.trans, payload={"login": user2.email, "password": changed_password}))
self.assertEqual(response["err_msg"], "Invalid password.")
response = json.loads(controller.login(self.trans, payload={"login": user2.username, "password": changed_password}))
self.assertEqual(response["err_msg"], "Invalid password.")
user2.deleted = True
response = json.loads(controller.login(self.trans, payload={"login": user2.username, "password": default_password}))
self.assertEqual(response["err_msg"], "This account has been marked deleted, contact your local Galaxy administrator to restore the account. Contact: admin@email.to.")
user2.deleted = False
user2.external = True
response = json.loads(controller.login(self.trans, payload={"login": user2.username, "password": default_password}))
self.assertEqual(response["err_msg"], "This account was created for use with an external authentication method, contact your local Galaxy administrator to activate it. Contact: admin@email.to.")
user2.external = False
self.trans.app.config.password_expiration_period = timedelta(days=1)
user2.last_password_change = datetime.today() - timedelta(days=1)
response = json.loads(controller.login(self.trans, payload={"login": user2.username, "password": default_password}))
self.assertEqual(response["message"], "Your password has expired. Please reset or change it to access Galaxy.")
self.assertEqual(response["expired_user"], self.trans.security.encode_id(user2.id))
self.trans.app.config.password_expiration_period = timedelta(days=10)
response = json.loads(controller.login(self.trans, payload={"login": user2.username, "password": default_password}))
self.assertEqual(response["message"], "Your password will expire in 11 day(s).")
self.trans.app.config.password_expiration_period = timedelta(days=100)
response = json.loads(controller.login(self.trans, payload={"login": user2.username, "password": default_password}))
self.assertEqual(response["message"], "Success.")
def test_empty_password(self):
self.log("should be able to create a user with no password")
@@ -8,8 +8,8 @@ try:
except ImportError:
psycopg2 = None
from galaxy.app_unittest_utils import galaxy_mock
from galaxy.util import which
from ..unittest_utils import galaxy_mock
def create_base_test(connection, amqp_type, amqp_connection=None):
@@ -2,6 +2,7 @@ import string
import unittest
from galaxy import model
from galaxy.app_unittest_utils import tools_support
from galaxy.exceptions import UserActivationRequiredException
from galaxy.tool_util.parser.output_objects import ToolOutput
from galaxy.tools.actions import (
@@ -10,7 +11,6 @@ from galaxy.tools.actions import (
on_text_for_names
)
from galaxy.util import XML
from .. import tools_support
# I cannot think of a saner way to test if data is being wrapped than use a
@@ -6,9 +6,9 @@ from galaxy import (
model,
util
)
from galaxy.app_unittest_utils import tools_support
from galaxy.tool_util.parser import output_collection_def
from galaxy.tool_util.provided_metadata import LegacyToolProvidedMetadata, NullToolProvidedMetadata
from .. import tools_support
DEFAULT_TOOL_OUTPUT = "out1"
DEFAULT_EXTRA_NAME = "test1"
@@ -2,9 +2,9 @@
test_select_parameters.py.
"""
from galaxy import model
from galaxy.app_unittest_utils.tools_support import datatypes_registry
from galaxy.util import bunch
from .util import BaseParameterTestCase
from ..tools_support import datatypes_registry
class DataColumnParameterTestCase(BaseParameterTestCase):
@@ -1,6 +1,6 @@
from galaxy import model
from galaxy.app_unittest_utils import galaxy_mock
from .util import BaseParameterTestCase
from ..unittest_utils import galaxy_mock
class DataToolParameterTestCase(BaseParameterTestCase):
@@ -1,6 +1,7 @@
from unittest import TestCase
from galaxy import model
from galaxy.app_unittest_utils.tools_support import UsesApp
from galaxy.tools.parameters import (
basic,
dataset_matcher
@@ -10,7 +11,6 @@ from galaxy.util import (
XML,
)
from .test_data_parameters import MockHistoryDatasetAssociation
from ..tools_support import UsesApp
class DatasetMatcherTestCase(TestCase, UsesApp):
@@ -1,6 +1,7 @@
import os
from unittest import TestCase
from galaxy.app_unittest_utils.tools_support import UsesApp
from galaxy.job_execution.datasets import DatasetPath
from galaxy.jobs import SimpleComputeEnvironment
from galaxy.model import (
@@ -28,8 +29,6 @@ from galaxy.tools.parameters.grouping import (
)
from galaxy.util import XML
from galaxy.util.bunch import Bunch
# Test fixtures for Galaxy infrastructure.
from ..tools_support import UsesApp
# To Test:
# - param_file handling.
@@ -6,9 +6,9 @@ from unittest import TestCase
import webob.exc
import galaxy.model
from galaxy.app_unittest_utils import tools_support
from galaxy.tools.parameters import params_to_incoming
from galaxy.util.bunch import Bunch
from .. import tools_support
BASE_REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool">
<command>echo "$param1" #for $r in $repeat# "$r.param2" #end for# &lt; $out1</command>
@@ -5,13 +5,14 @@ import tempfile
from shutil import rmtree
from galaxy import model
from galaxy.app_unittest_utils.galaxy_mock import MockApp
from galaxy.exceptions import MalformedContents
from galaxy.objectstore.unittest_utils import (
Config as TestConfig,
)
from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper, JobImportHistoryArchiveWrapper, unpack_tar_gz_archive
from galaxy.tools.imp_exp.export_history import create_archive
from ..unittest_utils.galaxy_mock import MockApp
from galaxy.util import galaxy_directory
# good enough for the very specific tests we're writing as of now...
@@ -22,6 +23,10 @@ HISTORY_ATTRS = '''{"hid_counter": 2, "update_time": "2016-02-08 18:38:38.705058
JOBS_ATTRS = '''[{"info": null, "tool_id": "upload1", "update_time": "2016-02-08T18:39:23.356482", "stdout": "", "input_mapping": {}, "tool_version": "1.1.4", "traceback": null, "command_line": "python /galaxy/tools/data_source/upload.py /galaxy /scratch/tmppwU9rD /scratch/tmpP4_45Y 1:/scratch/jobs/000/dataset_1_files:/data/000/dataset_1.dat", "exit_code": 0, "output_datasets": [1], "state": "ok", "create_time": "2016-02-08T18:38:39.153873", "params": {"files": [{"to_posix_lines": "Yes", "NAME": "None", "file_data": null, "space_to_tab": null, "url_paste": "/scratch/strio_url_paste_o6nrv8", "__index__": 0, "ftp_files": "", "uuid": "None"}], "paramfile": "/scratch/tmpP4_45Y", "file_type": "auto", "files_metadata": {"file_type": "auto", "__current_case__": 41}, "async_datasets": "None", "dbkey": "?"}, "stderr": ""}]'''
def t_data_path(name):
return os.path.join(galaxy_directory(), "test-data", name)
def _run_jihaw_cleanup(archive_dir, app=None):
app = app or _mock_app()
job = model.Job()
@@ -133,8 +138,8 @@ def test_export_dataset():
sa_session.add(j)
sa_session.flush()
app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True)
app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True)
app.object_store.update_from_file(d1, file_name=t_data_path("1.txt"), create=True)
app.object_store.update_from_file(d2, file_name=t_data_path("2.bed"), create=True)
imported_history = _import_export(app, h)
@@ -201,8 +206,8 @@ def test_export_dataset_with_deleted_and_purged():
assert d1.deleted
app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True)
app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True)
app.object_store.update_from_file(d1, file_name=t_data_path("1.txt"), create=True)
app.object_store.update_from_file(d2, file_name=t_data_path("2.bed"), create=True)
imported_history = _import_export(app, h)
@@ -237,9 +242,9 @@ def test_multi_inputs():
sa_session.add(j)
sa_session.flush()
app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True)
app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True)
app.object_store.update_from_file(d3, file_name="test-data/4.bed", create=True)
app.object_store.update_from_file(d1, file_name=t_data_path("1.txt"), create=True)
app.object_store.update_from_file(d2, file_name=t_data_path("2.bed"), create=True)
app.object_store.update_from_file(d3, file_name=t_data_path("4.bed"), create=True)
imported_history = _import_export(app, h)
@@ -444,8 +449,8 @@ def test_export_collection_with_copied_datasets_and_overlapping_hids():
sa_session.add(dataset_history)
sa_session.flush()
app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True)
app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True)
app.object_store.update_from_file(d1, file_name=t_data_path("1.txt"), create=True)
app.object_store.update_from_file(d2, file_name=t_data_path("2.bed"), create=True)
d1_copy = d1.copy()
d2_copy = d2.copy()
@@ -653,7 +658,7 @@ def _import_export(app, h, dest_export=None):
def test_import_1901_default():
app, new_history = import_archive('test-data/exports/1901_two_datasets.tgz')
app, new_history = import_archive(t_data_path('exports/1901_two_datasets.tgz'))
assert new_history
datasets = new_history.datasets
@@ -3,11 +3,11 @@ import subprocess
import unittest
from galaxy import model
from galaxy.app_unittest_utils import tools_support
from galaxy.job_execution.datasets import DatasetPath
from galaxy.metadata import get_metadata_compute_strategy
from galaxy.objectstore import ObjectStorePopulator
from galaxy.util import safe_makedirs
from .. import tools_support
from galaxy.util import galaxy_directory, safe_makedirs
class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.UsesTools):
@@ -42,7 +42,7 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
self._test_simple_output()
def _test_simple_output(self):
source_file_name = os.path.join(os.getcwd(), "test/functional/tools/for_workflows/cat.xml")
source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/for_workflows/cat.xml")
self._init_tool_for_path(source_file_name)
output_dataset = self._create_output_dataset(
extension="fasta",
@@ -67,7 +67,7 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
self._test_primary_dataset_output_extension()
def _test_primary_dataset_output_extension(self):
source_file_name = os.path.join(os.getcwd(), "test/functional/tools/for_workflows/cat.xml")
source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/for_workflows/cat.xml")
self._init_tool_for_path(source_file_name)
# setting extension to 'auto' here, results in the extension specified in
# galaxy.json (below) being respected.
@@ -100,7 +100,7 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
self._test_primary_dataset_output_metadata_override()
def _test_primary_dataset_output_metadata_override(self):
source_file_name = os.path.join(os.getcwd(), "test/functional/tools/for_workflows/cat.xml")
source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/for_workflows/cat.xml")
self._init_tool_for_path(source_file_name)
output_dataset = self._create_output_dataset(
extension="auto",
@@ -124,7 +124,7 @@ class MetadataTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.U
def test_list_discovery_extended(self):
self.app.config.metadata_strategy = "extended"
source_file_name = os.path.join(os.getcwd(), "test/functional/tools/collection_split_on_column.xml")
source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/collection_split_on_column.xml")
self._init_tool_for_path(source_file_name)
collection = model.DatasetCollection(populated=False)
collection.collection_type = "list"

Some files were not shown because too many files have changed in this diff Show More