diff --git a/client/src/components/Panels/Common/ToolSearch.vue b/client/src/components/Panels/Common/ToolSearch.vue index a09dfe7c382..11fe95f8f77 100644 --- a/client/src/components/Panels/Common/ToolSearch.vue +++ b/client/src/components/Panels/Common/ToolSearch.vue @@ -16,9 +16,11 @@ export default { props: { query: { type: String, + default: "", }, placeholder: { type: String, + default: "search tools", }, }, data() { diff --git a/client/src/components/Panels/utilities.js b/client/src/components/Panels/utilities.js index 3d5090edd9c..8c02e3fb7b9 100644 --- a/client/src/components/Panels/utilities.js +++ b/client/src/components/Panels/utilities.js @@ -77,10 +77,9 @@ export function filterTools(layout, results) { function normalize_results(results) { var norm_results = []; results.forEach((result) => { + norm_results.push(result); if (result.includes("/repos/")) { - norm_results.push(result.split("/")[3]); - } else { - norm_results.push(result); + norm_results.push(result.split("/repos/")[1].split("/")[2]); } }); return norm_results; diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 2d5441c6aec..bf9a623d6fc 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -66,6 +66,10 @@ LOGGING_CONFIG_DEFAULT = { 'level': 'WARN', 'qualname': 'paste.httpserver.ThreadPool', }, + 'sqlalchemy_json.track': { + 'level': 'WARN', + 'qualname': 'sqlalchemy_json.track', + }, 'urllib3.connectionpool': { 'level': 'WARN', 'qualname': 'urllib3.connectionpool', diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c9a050792aa..86d4242706d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -2586,6 +2586,10 @@ class SharedComputeEnvironment(SimpleComputeEnvironment): def working_directory(self): return self.job_wrapper.working_directory + def env_config_directory(self): + """Working directory (possibly as environment variable evaluation).""" + return "$_GALAXY_JOB_DIR" + def new_file_path(self): return os.path.abspath(self.app.config.new_file_path) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9d607a22acd..bd3633a52ee 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -382,6 +382,16 @@ class PulsarJobRunner(AsynchronousJobRunner): if os.path.exists(tool_script): log.debug("Registering tool_script for Pulsar transfer [%s]" % tool_script) job_directory_files.append(tool_script) + # Following is job destination environment variables + env = client.env + # extend it with tool defined environment variables + tool_envs = job_wrapper.environment_variables + env.extend(tool_envs) + for tool_env in tool_envs: + job_directory_path = tool_env.get("job_directory_path") + if job_directory_path: + config_files.append(job_directory_path) + client_job_description = ClientJobDescription( command_line=command_line, input_files=input_files, @@ -392,7 +402,7 @@ class PulsarJobRunner(AsynchronousJobRunner): tool=job_wrapper.tool, config_files=config_files, dependencies_description=dependencies_description, - env=client.env, + env=env, rewrite_paths=rewrite_paths, arbitrary_files=path_rewrites_unstructured, touch_outputs=output_names, @@ -1056,6 +1066,9 @@ class PulsarComputeEnvironment(ComputeEnvironment): def working_directory(self): return self._working_directory + def env_config_directory(self): + return self.config_directory() + def config_directory(self): return self._config_directory diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index aabc38d162d..40cab0a1481 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -610,7 +610,9 @@ def summarize_destination_params(trans, job): destination_params = {'Runner': job.job_runner_name, 'Runner Job ID': job.job_runner_external_id, 'Handler': job.handler} - destination_params.update(job.destination_params) + job_destination_params = job.destination_params + if job_destination_params: + destination_params.update(job_destination_params) return destination_params diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0f3e4524277..0261385af72 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1227,9 +1227,9 @@ model.FormDefinition.table = Table( Column("name", TrimmedString(255), nullable=False), Column("desc", TEXT), Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False), - Column("fields", JSONType), + Column("fields", SimpleJSONType), Column("type", TrimmedString(255), index=True), - Column("layout", JSONType)) + Column("layout", SimpleJSONType)) model.FormValues.table = Table( "form_values", metadata, @@ -1237,7 +1237,7 @@ model.FormValues.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now), Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("content", JSONType)) + Column("content", SimpleJSONType)) model.Page.table = Table( "page", metadata, diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index a3f40dd11be..97fe0df49be 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -113,7 +113,8 @@ class MetadataCollection(Mapping): return self.spec[name].wrap(self.spec[name].default, object_session(self.parent)) if name in self.parent._metadata: return self.parent._metadata[name] - raise AttributeError + # Instead of raising an AttributeError for non-existing metadata, we return None + return None def __setattr__(self, name, value): if name == "parent": diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 84ca3b9f8fc..9de9998c145 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -540,13 +540,14 @@ class ToolEvaluator: is_template = False else: is_template = True - with tempfile.NamedTemporaryFile(dir=directory, delete=False) as temp: + with tempfile.NamedTemporaryFile(dir=directory, prefix="tool_env_", delete=False) as temp: config_filename = temp.name self.__write_workdir_file(config_filename, environment_variable_template, param_dict, is_template=is_template, strip=environment_variable_def.get("strip", False)) config_file_basename = os.path.basename(config_filename) # environment setup in job file template happens before `cd $working_directory` - environment_variable["value"] = '`cat "$_GALAXY_JOB_DIR/%s"`' % config_file_basename + environment_variable["value"] = '`cat "%s/%s"`' % (self.compute_environment.env_config_directory(), config_file_basename) environment_variable["raw"] = True + environment_variable["job_directory_path"] = config_filename environment_variables.append(environment_variable) home_dir = self.compute_environment.home_directory() diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 4a62cce87f6..91639a96c33 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1015,7 +1015,7 @@ def roundify(amount, sfs=2): return amount[0:sfs] + '0' * (len(amount) - sfs) -def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False): +def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False, log_exception=True): """ Returns a Unicode string or None. @@ -1041,8 +1041,9 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals if not isinstance(value, str): value = str(value, encoding, error) except Exception as e: - msg = "Value '{}' could not be coerced to Unicode: {}('{}')".format(repr(value), type(e).__name__, e) - log.exception(msg) + if log_exception: + msg = "Value '{}' could not be coerced to Unicode: {}('{}')".format(repr(value), type(e).__name__, e) + log.exception(msg) raise if strip_null: return value.replace('\0', '') diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index 037daf8e880..13cd2951fec 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -77,7 +77,7 @@ def fill_template(template_text, raise first_exception or e t = klass(searchList=[context]) try: - return unicodify(t) + return unicodify(t, log_exception=False) except NotFound as e: if first_exception is None: first_exception = e diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py index 8352e6b1ae4..411daad976e 100644 --- a/test/integration/test_pulsar_embedded.py +++ b/test/integration/test_pulsar_embedded.py @@ -25,4 +25,5 @@ test_tools = integration_util.integration_tool_runner([ "multi_data_param", "output_filter", "vcf_bgzip_test", + "environment_variables", ]) diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 1299a6e48cd..392b6d1cfbc 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -379,6 +379,12 @@ class MappingTests(BaseModelTestCase): assert hist1.name == "History 2b" # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like. + def test_metadata_spec(self): + metadata = dict(chromCol=1, startCol=2, endCol=3) + d = self.model.HistoryDatasetAssociation(extension="interval", metadata=metadata, sa_session=self.model.session) + assert d.metadata.chromCol == 1 + assert d.metadata.anyAttribute is None + def test_jobs(self): model = self.model u = model.User(email="jobtest@foo.bar.baz", password="password")