mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge remote-tracking branch 'upstream/release_20.05' into dev
This commit is contained in:
Dannon Baker
@@ -3573,6 +3573,22 @@
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
``metadata_strategy``
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Determines how metadata will be set. Valid values are `directory`,
|
||||
`extended` and `legacy`. In extended mode jobs will decide if a
|
||||
tool run failed, the object stores configuration is serialized and
|
||||
made available to the job and is used for writing output datasets
|
||||
to the object store as part of the job and dynamic output
|
||||
discovery (e.g. discovered datasets <discover_datasets>,
|
||||
unpopulated collections, etc) happens as part of the job.
|
||||
:Default: ``directory``
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``retry_metadata_internally``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
@@ -17,8 +17,8 @@ from galaxy.model import CustosAuthnzToken, User
|
||||
from ..authnz import IdentityProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
STATE_COOKIE_NAME = 'custos-state'
|
||||
NONCE_COOKIE_NAME = 'custos-nonce'
|
||||
STATE_COOKIE_NAME = 'galaxy-oidc-state'
|
||||
NONCE_COOKIE_NAME = 'galaxy-oidc-nonce'
|
||||
KEYCLOAK_BACKENDS = {'custos', 'cilogon', 'keycloak'}
|
||||
|
||||
|
||||
|
||||
@@ -292,7 +292,7 @@ class AuthnzManager(object):
|
||||
raise exceptions.AuthenticationFailed(e.message)
|
||||
except Exception as e:
|
||||
msg = 'The following error occurred when handling callback from `{}` identity provider: ' \
|
||||
'{}'.format(provider, e.message)
|
||||
'{}'.format(provider, str(e))
|
||||
log.exception(msg)
|
||||
return False, msg, (None, None)
|
||||
|
||||
@@ -321,7 +321,7 @@ class AuthnzManager(object):
|
||||
return True, message, backend.logout(trans, post_logout_redirect_url)
|
||||
except Exception as e:
|
||||
msg = 'The following error occurred when logging out from `{}` identity provider: ' \
|
||||
'{}'.format(provider, e.message)
|
||||
'{}'.format(provider, str(e))
|
||||
log.exception(msg)
|
||||
return False, msg, None
|
||||
|
||||
|
||||
@@ -1751,6 +1751,15 @@ galaxy:
|
||||
# database.
|
||||
#enable_job_recovery: true
|
||||
|
||||
# Determines how metadata will be set. Valid values are `directory`,
|
||||
# `extended` and `legacy`. In extended mode jobs will decide if a tool
|
||||
# run failed, the object stores configuration is serialized and made
|
||||
# available to the job and is used for writing output datasets to the
|
||||
# object store as part of the job and dynamic output discovery (e.g.
|
||||
# discovered datasets <discover_datasets>, unpopulated collections,
|
||||
# etc) happens as part of the job.
|
||||
#metadata_strategy: directory
|
||||
|
||||
# Although it is fairly reliable, setting metadata can occasionally
|
||||
# fail. In these instances, you can choose to retry setting it
|
||||
# internally or leave it in a failed state (since retrying internally
|
||||
|
||||
@@ -401,6 +401,17 @@
|
||||
try it early - it will slightly speed up local jobs by
|
||||
embedding metadata calculation in job script itself.
|
||||
-->
|
||||
<param id="metadata_strategy">directory</param>
|
||||
<!-- If set, overwrites metadata_strategy (configurable in galaxy.yml).
|
||||
Valid values are `directory` (default), `extended` and `legacy`.
|
||||
Setting metadata_strategy to `extended` requires that all object stores
|
||||
that can be written to store datasets by uuid. In `extended` mode jobs
|
||||
will decide if a tool run failed, the object stores configuration
|
||||
is serialized and made available to the job and is used for writing output
|
||||
datasets to the object store as part of the job and dynamic output discovery
|
||||
(e.g. discovered datasets <discover_datasets>, unpopulated collections, etc)
|
||||
happens as part of the job.
|
||||
-->
|
||||
<job_metrics />
|
||||
<!-- Above element demonstrates embedded job metrics definition - see
|
||||
job_metrics_conf.xml.sample for full documentation on possible nested
|
||||
|
||||
@@ -906,17 +906,20 @@ class JobWrapper(HasResourceParameters):
|
||||
self.output_paths = None
|
||||
self.output_hdas_and_paths = None
|
||||
self.tool_provided_job_metadata = None
|
||||
# Wrapper holding the info required to restore and clean up from files used for setting metadata externally
|
||||
metadata_strategy_override = None
|
||||
if job.tasks:
|
||||
metadata_strategy_override = "directory"
|
||||
self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override)
|
||||
self.job_runner_mapper = JobRunnerMapper(self, queue.dispatcher.url_to_destination, self.app.job_config)
|
||||
self.params = None
|
||||
if job.params:
|
||||
self.params = loads(job.params)
|
||||
if use_persisted_destination:
|
||||
self.job_runner_mapper.cached_job_destination = JobDestination(from_job=job)
|
||||
# Wrapper holding the info required to restore and clean up from files used for setting metadata externally
|
||||
try:
|
||||
metadata_strategy_override = self.get_destination_configuration('metadata_strategy', None)
|
||||
except JobMappingException:
|
||||
metadata_strategy_override = None
|
||||
if job.tasks:
|
||||
metadata_strategy_override = "directory"
|
||||
self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override)
|
||||
|
||||
self.__commands_in_new_shell = True
|
||||
self.__user_system_pwent = None
|
||||
|
||||
@@ -174,7 +174,7 @@ def set_metadata_portable():
|
||||
|
||||
# Load outputs.
|
||||
import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
|
||||
export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True)
|
||||
export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False)
|
||||
|
||||
for output_name, output_dict in outputs.items():
|
||||
if extended_metadata_collection:
|
||||
|
||||
@@ -526,8 +526,7 @@ class FileParameter(MetadataParameter):
|
||||
return None
|
||||
if isinstance(value, galaxy.model.MetadataFile) or isinstance(value, MetadataTempFile):
|
||||
return value
|
||||
mf = session.query(galaxy.model.MetadataFile).get(value)
|
||||
return mf
|
||||
return session.query(galaxy.model.MetadataFile).get(value)
|
||||
|
||||
def make_copy(self, value, target_context, source_context):
|
||||
value = self.wrap(value, object_session(target_context.parent))
|
||||
@@ -542,7 +541,10 @@ class FileParameter(MetadataParameter):
|
||||
@classmethod
|
||||
def marshal(cls, value):
|
||||
if isinstance(value, galaxy.model.MetadataFile):
|
||||
value = value.id
|
||||
# We want to push value.id to the database, but need to skip this when no session is available,
|
||||
# as in extended_metadata mode, so there we just accept MetadataFile.
|
||||
# We will only serialize MetadataFile in this mode and not push to the database, so this is OK.
|
||||
value = value.id or value
|
||||
return value
|
||||
|
||||
def from_external_value(self, value, parent, path_rewriter=None):
|
||||
|
||||
@@ -2645,6 +2645,18 @@ mapping:
|
||||
it can "recover" them when it starts). This is not safe to use if you are
|
||||
running more than one Galaxy server using the same database.
|
||||
|
||||
metadata_strategy:
|
||||
type: str
|
||||
required: false
|
||||
default: directory
|
||||
desc: |
|
||||
Determines how metadata will be set. Valid values are `directory`, `extended` and `legacy`.
|
||||
In extended mode jobs will decide if a tool run failed, the object stores
|
||||
configuration is serialized and made available to the job and is used for
|
||||
writing output datasets to the object store as part of the job and dynamic
|
||||
output discovery (e.g. discovered datasets <discover_datasets>, unpopulated collections,
|
||||
etc) happens as part of the job.
|
||||
|
||||
retry_metadata_internally:
|
||||
type: bool
|
||||
default: true
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-app"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-auth"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,11 +6,17 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.1.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
---------------------
|
||||
19.9.0 (2019-12-16)
|
||||
---------------------
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.1.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-containers"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,14 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev1'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-data"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-job-execution"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,11 +6,17 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.1.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
---------------------
|
||||
19.9.0 (2019-12-16)
|
||||
---------------------
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.1.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-job-metrics"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,11 +6,17 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.1.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
---------------------
|
||||
19.9.0 (2019-12-16)
|
||||
---------------------
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.1.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-objectstore"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-selenium"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
20.1.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 20.01 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* Initial import from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '20.1.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-test-base"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
20.1.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 20.01 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '20.1.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-test-driver"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,13 +6,19 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
20.5.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
---------------------
|
||||
20.1.0.dev0
|
||||
20.1.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.01 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '20.5.0.dev3'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-tool-util"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-web-framework"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-web-stack"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -6,7 +6,13 @@ History
|
||||
.. to_doc
|
||||
|
||||
---------------------
|
||||
19.9.0.dev0
|
||||
20.5.1.dev0
|
||||
---------------------
|
||||
|
||||
* Initial import from dev branch of Galaxy during 19.09 development cycle.
|
||||
|
||||
|
||||
---------------------
|
||||
20.5.0 (2020-07-04)
|
||||
---------------------
|
||||
|
||||
* First release from the 20.05 branch of Galaxy.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__version__ = '19.9.0.dev0'
|
||||
__version__ = '20.5.1.dev0'
|
||||
|
||||
PROJECT_NAME = "galaxy-webapps"
|
||||
PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
<tool id="metadata_bam" name="metadata BAM" version="1.0.0">
|
||||
<command>echo "${ref_names}" > "${output_of_input_metadata}"</command>
|
||||
<tool id="metadata_bam" name="metadata BAM" version="1.0.0" profile="16.04">
|
||||
<command><![CDATA[
|
||||
#assert $input_bam.metadata.bam_index
|
||||
echo '${ref_names}' > '${output_of_input_metadata}'
|
||||
]]></command>
|
||||
<inputs>
|
||||
<param name="input_bam" type="data" format="bam" label="BAM File"/>
|
||||
<param name="ref_names" type="select" optional="False" label="Select references you would like to restrict bam to" multiple="True">
|
||||
|
||||
@@ -13,6 +13,7 @@ execution:
|
||||
runner: pulsar_embed
|
||||
remote_metadata: true
|
||||
default_file_action: copy
|
||||
metadata_strategy: directory
|
||||
|
||||
tools:
|
||||
- id: upload1
|
||||
|
||||
@@ -16,8 +16,13 @@ class EmbeddedMetadataPulsarIntegrationInstance(integration_util.IntegrationInst
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
|
||||
config['object_store_store_by'] = 'uuid'
|
||||
# We set the global metadata_strategy to `extended, but`metadata_strategy is
|
||||
# being overridden in embedded_pulsar_metadata_job_conf.yml, since extended_metadata does not yet work on pulsar
|
||||
config['metadata_strategy'] = 'extended'
|
||||
config['retry_metadata_internally'] = False
|
||||
|
||||
|
||||
instance = integration_util.integration_module_instance(EmbeddedMetadataPulsarIntegrationInstance)
|
||||
|
||||
test_tools = integration_util.integration_tool_runner(["simple_constructs"])
|
||||
test_tools = integration_util.integration_tool_runner(["simple_constructs", "metadata_bam"])
|
||||
|
||||
Reference in New Issue
Block a user