mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-24 12:13:33 +08:00
Update tool_test_results dict's attribute names for consistency. Only flag a changeset revision not to be tested if there no valid tests found in that revision.
This commit is contained in:
Dave Bouvier
@@ -130,16 +130,6 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
"architecture": "x86_64",
|
||||
"system": "Darwin 12.2.0"
|
||||
},
|
||||
"test_errors":
|
||||
[
|
||||
{
|
||||
"test_id": "The test ID, generated by twill",
|
||||
"tool_id": "The tool ID that was tested",
|
||||
"tool_version": "The tool version that was tested",
|
||||
"stderr": "The output of the test, or a more detailed description of what was tested and what the error was."
|
||||
"traceback": "The traceback, if any."
|
||||
},
|
||||
]
|
||||
"passed_tests":
|
||||
[
|
||||
{
|
||||
@@ -147,14 +137,24 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
"tool_id": "The tool ID that was tested",
|
||||
"tool_version": "The tool version that was tested",
|
||||
},
|
||||
]
|
||||
"invalid_tests":
|
||||
],
|
||||
"failed_tests":
|
||||
[
|
||||
{
|
||||
"tool_id": "The tool ID that does not have functional tests defined.",
|
||||
"tool_version": "The version of the tool."
|
||||
"tool_guid": "The guid of the tool."
|
||||
"reason_test_is_invalid": "A short explanation of what is invalid.
|
||||
"test_id": "The test ID, generated by twill",
|
||||
"tool_id": "The tool ID that was tested",
|
||||
"tool_version": "The tool version that was tested",
|
||||
"stderr": "The output of the test, or a more detailed description of what was tested and what the error was.",
|
||||
"traceback": "The traceback, if any."
|
||||
},
|
||||
],
|
||||
"missing_test_components":
|
||||
[
|
||||
{
|
||||
"tool_id": "The tool ID that is missing functional test definitions and/or test data.",
|
||||
"tool_version": "The version of the tool.",
|
||||
"tool_guid": "The guid of the tool.",
|
||||
"missing_components": "The components that are missing for this tool to be considered testable."
|
||||
},
|
||||
]
|
||||
}
|
||||
@@ -182,7 +182,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
repository_status = metadata_record.tool_test_results
|
||||
# Clear any old invalid tests for this metadata revision, since this could lead to duplication of invalid test rows,
|
||||
# or tests incorrectly labeled as invalid.
|
||||
repository_status[ 'invalid_tests' ] = []
|
||||
repository_status[ 'missing_test_components' ] = []
|
||||
if 'test_environment' in repository_status:
|
||||
repository_status[ 'test_environment' ] = get_test_environment( repository_status[ 'test_environment' ] )
|
||||
else:
|
||||
@@ -204,6 +204,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
continue
|
||||
else:
|
||||
has_test_data = False
|
||||
testable_revision_found = False
|
||||
# Clone the repository up to the changeset revision we're checking.
|
||||
repo_dir = metadata_record.repository.repo_path( app )
|
||||
repo = hg.repository( get_configured_ui(), repo_dir )
|
||||
@@ -254,12 +255,15 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
failure_reason = ''
|
||||
problem_found = False
|
||||
missing_test_files = []
|
||||
has_test_files = False
|
||||
if tool_has_tests and has_test_data:
|
||||
missing_test_files = check_for_missing_test_files( tool_metadata[ 'tests' ], test_data_path )
|
||||
if missing_test_files:
|
||||
if verbosity >= 2:
|
||||
print "# Tool ID '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \
|
||||
( tool_id, changeset_revision, name, ', '.join( missing_test_files ) )
|
||||
else:
|
||||
has_test_files = True
|
||||
if not has_test_data:
|
||||
failure_reason += 'Repository does not have a test-data directory. '
|
||||
problem_found = True
|
||||
@@ -270,7 +274,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
failure_reason += 'One or more test files are missing for tool %s: %s' % ( tool_id, ', '.join( missing_test_files ) )
|
||||
problem_found = True
|
||||
test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid,
|
||||
reason_test_is_invalid=failure_reason )
|
||||
missing_components=failure_reason )
|
||||
# The repository_metadata.tool_test_results attribute should always have the following structure:
|
||||
# {
|
||||
# "test_environment":
|
||||
@@ -284,7 +288,15 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
# "architecture": "x86_64",
|
||||
# "system": "Darwin 12.2.0"
|
||||
# },
|
||||
# "test_errors":
|
||||
# "passed_tests":
|
||||
# [
|
||||
# {
|
||||
# "test_id": "The test ID, generated by twill",
|
||||
# "tool_id": "The tool ID that was tested",
|
||||
# "tool_version": "The tool version that was tested",
|
||||
# },
|
||||
# ],
|
||||
# "failed_tests":
|
||||
# [
|
||||
# {
|
||||
# "test_id": "The test ID, generated by twill",
|
||||
@@ -293,23 +305,14 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
# "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was."
|
||||
# "traceback": "The captured traceback."
|
||||
# },
|
||||
# ]
|
||||
# "passed_tests":
|
||||
# [
|
||||
# {
|
||||
# "test_id": "The test ID, generated by twill",
|
||||
# "tool_id": "The tool ID that was tested",
|
||||
# "tool_version": "The tool version that was tested",
|
||||
# "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was."
|
||||
# },
|
||||
# ]
|
||||
# "invalid_tests":
|
||||
# ],
|
||||
# "missing_test_components":
|
||||
# [
|
||||
# {
|
||||
# "tool_id": "The ID of the tool that does not have valid tests.",
|
||||
# "tool_version": "The version of the tool."
|
||||
# "tool_guid": "The guid of the tool."
|
||||
# "reason_test_is_invalid": "A short explanation of what is invalid."
|
||||
# "missing_components": "The components that are missing for this tool to be considered testable."
|
||||
# },
|
||||
# ]
|
||||
# }
|
||||
@@ -318,12 +321,14 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
# than the list relevant to what it is testing.
|
||||
# Only append this error dict if it hasn't already been added.
|
||||
if problem_found:
|
||||
if test_errors not in repository_status[ 'invalid_tests' ]:
|
||||
repository_status[ 'invalid_tests' ].append( test_errors )
|
||||
if test_errors not in repository_status[ 'missing_test_components' ]:
|
||||
repository_status[ 'missing_test_components' ].append( test_errors )
|
||||
if tool_has_tests and has_test_files:
|
||||
testable_revision_found = True
|
||||
# Remove the cloned repository path. This has to be done after the check for required test files, for obvious reasons.
|
||||
if os.path.exists( work_dir ):
|
||||
shutil.rmtree( work_dir )
|
||||
if not repository_status[ 'invalid_tests' ]:
|
||||
if not repository_status[ 'missing_test_components' ]:
|
||||
valid_revisions += 1
|
||||
if verbosity >= 1:
|
||||
print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner )
|
||||
@@ -332,22 +337,27 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ):
|
||||
if verbosity >= 1:
|
||||
print '# Some tools have problematic functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner )
|
||||
if verbosity >= 2:
|
||||
for invalid_test in repository_status[ 'invalid_tests' ]:
|
||||
if 'reason_test_is_invalid' in invalid_test:
|
||||
print '# %s' % invalid_test[ 'reason_test_is_invalid' ]
|
||||
for invalid_test in repository_status[ 'missing_test_components' ]:
|
||||
if 'missing_components' in invalid_test:
|
||||
print '# %s' % invalid_test[ 'missing_components' ]
|
||||
if not info_only:
|
||||
# If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform
|
||||
# on which this script was run.
|
||||
if repository_status[ 'invalid_tests' ]:
|
||||
# If functional test definitions or test data are missing, set do_not_test = True if and only if:
|
||||
if repository_status[ 'missing_test_components' ]:
|
||||
# If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been
|
||||
# found in this revision, and:
|
||||
# a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision.
|
||||
# In this case, the revision will never be updated with correct data, and re-testing it would be redundant.
|
||||
# b) There are one or more downloadable revisions, and the revision being tested is the most recent downloadable revision.
|
||||
# In this case, if the repository is updated with test data or functional tests, the downloadable changeset revision
|
||||
# that was tested will be replaced with the new changeset revision, which will be automatically tested.
|
||||
if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ):
|
||||
# In this case, the revision will never be updated with the missing components, and re-testing it would be redundant.
|
||||
# b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable
|
||||
# revision. In this case, if the repository is updated with test data or functional tests, the downloadable
|
||||
# changeset revision that was tested will either be replaced with the new changeset revision, or a new downloadable
|
||||
# changeset revision will be created, either of which will be automatically checked and flagged as appropriate.
|
||||
# In the install and test script, this behavior is slightly different, since we do want to always run functional
|
||||
# tests on the most recent downloadable changeset revision.
|
||||
if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ) and not testable_revision_found:
|
||||
metadata_record.do_not_test = True
|
||||
metadata_record.tools_functionally_correct = False
|
||||
metadata_record.missing_test_components = True
|
||||
metadata_record.tool_test_results = repository_status
|
||||
metadata_record.time_last_tested = datetime.utcnow()
|
||||
app.sa_session.add( metadata_record )
|
||||
|
||||
@@ -174,9 +174,9 @@ class ReportResults( Plugin ):
|
||||
|
||||
def getTestStatus( self, test_identifier ):
|
||||
if test_identifier in self.passed:
|
||||
tests_passed = self.passed[ test_identifier ]
|
||||
passed_tests = self.passed[ test_identifier ]
|
||||
del self.passed[ test_identifier ]
|
||||
return tests_passed
|
||||
return passed_tests
|
||||
return []
|
||||
|
||||
def execute_uninstall_method( repository_dict ):
|
||||
@@ -278,13 +278,13 @@ def json_from_url( url ):
|
||||
url_contents = url_handle.read()
|
||||
return from_json_string( url_contents )
|
||||
|
||||
def register_test_result( url, metadata_id, test_results_dict, tests_passed=False ):
|
||||
def register_test_result( url, metadata_id, test_results_dict, passed_tests=False ):
|
||||
'''
|
||||
This script should never set do_not_test = True, because the repositories should always be re-tested
|
||||
against the most recent code.
|
||||
'''
|
||||
params = {}
|
||||
if tests_passed:
|
||||
if passed_tests:
|
||||
params[ 'tools_functionally_correct' ] = 'true'
|
||||
params[ 'do_not_test' ] = 'false'
|
||||
else:
|
||||
@@ -599,7 +599,15 @@ def main():
|
||||
# "architecture": "x86_64",
|
||||
# "system": "Darwin 12.2.0"
|
||||
# },
|
||||
# "test_errors":
|
||||
# "passed_tests":
|
||||
# [
|
||||
# {
|
||||
# "test_id": "The test ID, generated by twill",
|
||||
# "tool_id": "The tool ID that was tested",
|
||||
# "tool_version": "The tool version that was tested",
|
||||
# },
|
||||
# ]
|
||||
# "failed_tests":
|
||||
# [
|
||||
# {
|
||||
# "test_id": "The test ID, generated by twill",
|
||||
@@ -609,22 +617,13 @@ def main():
|
||||
# "traceback": "The captured traceback."
|
||||
# },
|
||||
# ]
|
||||
# "passed_tests":
|
||||
# [
|
||||
# {
|
||||
# "test_id": "The test ID, generated by twill",
|
||||
# "tool_id": "The tool ID that was tested",
|
||||
# "tool_version": "The tool version that was tested",
|
||||
# "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was."
|
||||
# },
|
||||
# ]
|
||||
# "invalid_tests":
|
||||
# "missing_test_components":
|
||||
# [
|
||||
# {
|
||||
# "tool_id": "The tool ID that does not have functional tests defined.",
|
||||
# "tool_version": "The version of the tool."
|
||||
# "tool_guid": "The guid of the tool."
|
||||
# "reason_test_is_invalid": "A short explanation of what is invalid.
|
||||
# "missing_components": "A short explanation of what is invalid.
|
||||
# },
|
||||
# ]
|
||||
# }
|
||||
@@ -635,17 +634,17 @@ def main():
|
||||
test_environment[ 'galaxy_database_version' ] = get_database_version( app )
|
||||
test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() )
|
||||
repository_status[ 'test_environment' ] = test_environment
|
||||
repository_status[ 'tests_passed' ] = []
|
||||
repository_status[ 'test_errors' ] = []
|
||||
repository_status[ 'invalid_tests' ] = []
|
||||
repository_status[ 'passed_tests' ] = []
|
||||
repository_status[ 'failed_tests' ] = []
|
||||
repository_status[ 'missing_test_components' ] = []
|
||||
if not has_test_data:
|
||||
log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' )
|
||||
# Record the lack of test data.
|
||||
test_errors = dict( tool_id=None, tool_version=None, tool_guid=None,
|
||||
reason_test_is_invalid="Repository %s is missing a test-data directory." % name )
|
||||
repository_status[ 'invalid_tests' ].append( test_errors )
|
||||
failed_tests = dict( tool_id=None, tool_version=None, tool_guid=None,
|
||||
missing_components="Repository %s is missing a test-data directory." % name )
|
||||
repository_status[ 'missing_test_components' ].append( failed_tests )
|
||||
# Record the status of this repository in the tool shed.
|
||||
register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False )
|
||||
register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False )
|
||||
# Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the
|
||||
# repository using Twill.
|
||||
execute_uninstall_method( repository_info_dict )
|
||||
@@ -677,20 +676,20 @@ def main():
|
||||
for plugin in test_plugins:
|
||||
if hasattr( plugin, 'getTestStatus' ):
|
||||
test_identifier = '%s/%s' % ( owner, name )
|
||||
tests_passed = plugin.getTestStatus( test_identifier )
|
||||
passed_tests = plugin.getTestStatus( test_identifier )
|
||||
break
|
||||
repository_status[ 'tests_passed' ] = []
|
||||
for test_id in tests_passed:
|
||||
repository_status[ 'passed_tests' ] = []
|
||||
for test_id in passed_tests:
|
||||
tool_id, tool_version = get_tool_info_from_test_id( test_id )
|
||||
test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version )
|
||||
repository_status[ 'tests_passed' ].append( test_result )
|
||||
repository_status[ 'passed_tests' ].append( test_result )
|
||||
if success:
|
||||
# This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database
|
||||
# to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API
|
||||
# controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and
|
||||
# updates the time_last_tested field to today's date.
|
||||
repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) )
|
||||
register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True )
|
||||
register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=True )
|
||||
log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( changeset_revision, name ) )
|
||||
else:
|
||||
# If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API.
|
||||
@@ -729,13 +728,13 @@ def main():
|
||||
for output_type in [ 'stderr', 'traceback' ]:
|
||||
if output_type in tmp_output:
|
||||
test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] )
|
||||
repository_status[ 'test_errors' ].append( test_status )
|
||||
repository_status[ 'failed_tests' ].append( test_status )
|
||||
# Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome
|
||||
# of the tests, and updates tool_test_results with the relevant log data.
|
||||
# This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested
|
||||
# field to today's date.
|
||||
repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) )
|
||||
register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False )
|
||||
register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False )
|
||||
log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \
|
||||
( changeset_revision, name ) )
|
||||
# Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the
|
||||
|
||||
Reference in New Issue
Block a user