Merge remote-tracking branch 'upstream/release_21.01' into dev

client/src/components/Panels/Common/ToolSearch.vue

@@ -16,9 +16,11 @@ export default {
     props: {
         query: {
             type: String,
+            default: "",
         },
         placeholder: {
             type: String,
+            default: "search tools",
         },
     },
     data() {

client/src/components/Panels/utilities.js

@@ -77,10 +77,9 @@ export function filterTools(layout, results) {
 function normalize_results(results) {
     var norm_results = [];
     results.forEach((result) => {
+        norm_results.push(result);
         if (result.includes("/repos/")) {
-            norm_results.push(result.split("/")[3]);
-        } else {
-            norm_results.push(result);
+            norm_results.push(result.split("/repos/")[1].split("/")[2]);
         }
     });
     return norm_results;

lib/galaxy/config/__init__.py

@@ -66,6 +66,10 @@ LOGGING_CONFIG_DEFAULT = {
             'level': 'WARN',
             'qualname': 'paste.httpserver.ThreadPool',
         },
+        'sqlalchemy_json.track': {
+            'level': 'WARN',
+            'qualname': 'sqlalchemy_json.track',
+        },
         'urllib3.connectionpool': {
             'level': 'WARN',
             'qualname': 'urllib3.connectionpool',

lib/galaxy/jobs/__init__.py

@@ -2586,6 +2586,10 @@ class SharedComputeEnvironment(SimpleComputeEnvironment):
     def working_directory(self):
         return self.job_wrapper.working_directory
 
+    def env_config_directory(self):
+        """Working directory (possibly as environment variable evaluation)."""
+        return "$_GALAXY_JOB_DIR"
+
     def new_file_path(self):
         return os.path.abspath(self.app.config.new_file_path)
 

lib/galaxy/jobs/runners/pulsar.py

@@ -382,6 +382,16 @@ class PulsarJobRunner(AsynchronousJobRunner):
             if os.path.exists(tool_script):
                 log.debug("Registering tool_script for Pulsar transfer [%s]" % tool_script)
                 job_directory_files.append(tool_script)
+            # Following is job destination environment variables
+            env = client.env
+            # extend it with tool defined environment variables
+            tool_envs = job_wrapper.environment_variables
+            env.extend(tool_envs)
+            for tool_env in tool_envs:
+                job_directory_path = tool_env.get("job_directory_path")
+                if job_directory_path:
+                    config_files.append(job_directory_path)
+
             client_job_description = ClientJobDescription(
                 command_line=command_line,
                 input_files=input_files,
@@ -392,7 +402,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
                 tool=job_wrapper.tool,
                 config_files=config_files,
                 dependencies_description=dependencies_description,
-                env=client.env,
+                env=env,
                 rewrite_paths=rewrite_paths,
                 arbitrary_files=path_rewrites_unstructured,
                 touch_outputs=output_names,
@@ -1056,6 +1066,9 @@ class PulsarComputeEnvironment(ComputeEnvironment):
     def working_directory(self):
         return self._working_directory
 
+    def env_config_directory(self):
+        return self.config_directory()
+
     def config_directory(self):
         return self._config_directory
 

lib/galaxy/managers/jobs.py

@@ -610,7 +610,9 @@ def summarize_destination_params(trans, job):
     destination_params = {'Runner': job.job_runner_name,
                           'Runner Job ID': job.job_runner_external_id,
                           'Handler': job.handler}
-    destination_params.update(job.destination_params)
+    job_destination_params = job.destination_params
+    if job_destination_params:
+        destination_params.update(job_destination_params)
     return destination_params
 
 

lib/galaxy/model/mapping.py

@@ -1227,9 +1227,9 @@ model.FormDefinition.table = Table(
     Column("name", TrimmedString(255), nullable=False),
     Column("desc", TEXT),
     Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
-    Column("fields", JSONType),
+    Column("fields", SimpleJSONType),
     Column("type", TrimmedString(255), index=True),
-    Column("layout", JSONType))
+    Column("layout", SimpleJSONType))
 
 model.FormValues.table = Table(
     "form_values", metadata,
@@ -1237,7 +1237,7 @@ model.FormValues.table = Table(
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, default=now, onupdate=now),
     Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
-    Column("content", JSONType))
+    Column("content", SimpleJSONType))
 
 model.Page.table = Table(
     "page", metadata,

lib/galaxy/model/metadata.py

@@ -113,7 +113,8 @@ class MetadataCollection(Mapping):
             return self.spec[name].wrap(self.spec[name].default, object_session(self.parent))
         if name in self.parent._metadata:
             return self.parent._metadata[name]
-        raise AttributeError
+        # Instead of raising an AttributeError for non-existing metadata, we return None
+        return None
 
     def __setattr__(self, name, value):
         if name == "parent":

lib/galaxy/tools/evaluation.py

@@ -540,13 +540,14 @@ class ToolEvaluator:
                 is_template = False
             else:
                 is_template = True
-            with tempfile.NamedTemporaryFile(dir=directory, delete=False) as temp:
+            with tempfile.NamedTemporaryFile(dir=directory, prefix="tool_env_", delete=False) as temp:
                 config_filename = temp.name
             self.__write_workdir_file(config_filename, environment_variable_template, param_dict, is_template=is_template, strip=environment_variable_def.get("strip", False))
             config_file_basename = os.path.basename(config_filename)
             # environment setup in job file template happens before `cd $working_directory`
-            environment_variable["value"] = '`cat "$_GALAXY_JOB_DIR/%s"`' % config_file_basename
+            environment_variable["value"] = '`cat "%s/%s"`' % (self.compute_environment.env_config_directory(), config_file_basename)
             environment_variable["raw"] = True
+            environment_variable["job_directory_path"] = config_filename
             environment_variables.append(environment_variable)
 
         home_dir = self.compute_environment.home_directory()

lib/galaxy/util/__init__.py

@@ -1015,7 +1015,7 @@ def roundify(amount, sfs=2):
         return amount[0:sfs] + '0' * (len(amount) - sfs)
 
 
-def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False):
+def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False, log_exception=True):
     """
     Returns a Unicode string or None.
 
@@ -1041,8 +1041,9 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals
         if not isinstance(value, str):
             value = str(value, encoding, error)
     except Exception as e:
-        msg = "Value '{}' could not be coerced to Unicode: {}('{}')".format(repr(value), type(e).__name__, e)
-        log.exception(msg)
+        if log_exception:
+            msg = "Value '{}' could not be coerced to Unicode: {}('{}')".format(repr(value), type(e).__name__, e)
+            log.exception(msg)
         raise
     if strip_null:
         return value.replace('\0', '')

lib/galaxy/util/template.py

@@ -77,7 +77,7 @@ def fill_template(template_text,
         raise first_exception or e
     t = klass(searchList=[context])
     try:
-        return unicodify(t)
+        return unicodify(t, log_exception=False)
     except NotFound as e:
         if first_exception is None:
             first_exception = e

test/integration/test_pulsar_embedded.py

@@ -25,4 +25,5 @@ test_tools = integration_util.integration_tool_runner([
     "multi_data_param",
     "output_filter",
     "vcf_bgzip_test",
+    "environment_variables",
 ])

test/unit/data/test_galaxy_mapping.py

@@ -379,6 +379,12 @@ class MappingTests(BaseModelTestCase):
         assert hist1.name == "History 2b"
         # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like.
 
+    def test_metadata_spec(self):
+        metadata = dict(chromCol=1, startCol=2, endCol=3)
+        d = self.model.HistoryDatasetAssociation(extension="interval", metadata=metadata, sa_session=self.model.session)
+        assert d.metadata.chromCol == 1
+        assert d.metadata.anyAttribute is None
+
     def test_jobs(self):
         model = self.model
         u = model.User(email="jobtest@foo.bar.baz", password="password")