Merge branch 'release_19.09' into dev

lib/galaxy/jobs/__init__.py

@@ -2455,6 +2455,10 @@ class ComputeEnvironment(object):
     def tmp_directory(self):
         """Temp directory of target job - none if HOME should not be set."""
 
+    @abstractmethod
+    def galaxy_url(self):
+        """URL to access Galaxy API from for this compute environment."""
+
 
 class SimpleComputeEnvironment(object):
 
@@ -2525,6 +2529,9 @@ class SharedComputeEnvironment(SimpleComputeEnvironment):
     def tmp_directory(self):
         return self.job_wrapper.tmp_directory()
 
+    def galaxy_url(self):
+        return self.job_wrapper.get_destination_configuration("galaxy_infrastructure_url")
+
 
 class NoopQueue(object):
     """

lib/galaxy/jobs/runners/condor.py

@@ -176,7 +176,7 @@ class CondorJobRunner(AsynchronousJobRunner):
             job_id = cjs.job_id
             galaxy_id_tag = cjs.job_wrapper.get_id_tag()
             try:
-                if os.stat(cjs.user_log).st_size == cjs.user_log_size:
+                if cjs.job_wrapper.tool.tool_type != 'interactive' and os.stat(cjs.user_log).st_size == cjs.user_log_size:
                     new_watched.append(cjs)
                     continue
                 s1, s4, s7, s5, s9, log_size = summarize_condor_log(cjs.user_log, job_id)

lib/galaxy/jobs/runners/pulsar.py

@@ -1025,6 +1025,9 @@ class PulsarComputeEnvironment(ComputeEnvironment):
         # meantime.
         return None
 
+    def galaxy_url(self):
+        return self.job_wrapper.get_destination_configuration("galaxy_infrastructure_url")
+
 
 class UnsupportedPulsarException(Exception):
 

lib/galaxy/managers/library_datasets.py

@@ -139,7 +139,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager):
                 val = validation.validate_and_sanitize_basestring(key, val)
                 validated_payload[key] = val
             if key in ('tags'):
-                val = validation.validate_and_sanitize_basestring_list(key, val)
+                val = validation.validate_and_sanitize_basestring_list(key, util.listify(val))
                 validated_payload[key] = val
         return validated_payload
 

lib/galaxy/tool_util/parser/xml.py

@@ -138,9 +138,17 @@ class XmlToolSource(ToolSource):
 
         environment_variables = []
         for environment_variable_el in environment_variables_el.findall("environment_variable"):
+            template = environment_variable_el.text
+            inject = environment_variable_el.get("inject")
+            if inject:
+                assert not template, "Cannot specify inject and environment variable template."
+                assert inject in ["api_key"]
+            if template:
+                assert not inject, "Cannot specify inject and environment variable template."
             definition = {
                 "name": environment_variable_el.get("name"),
-                "template": environment_variable_el.text,
+                "template": template,
+                "inject": inject,
                 "strip": string_as_bool(environment_variable_el.get("strip", False)),
             }
             environment_variables.append(

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -4676,6 +4676,21 @@ variable instead of shell variable.
 </command>
 ```
 
+### inject
+
+The Galaxy user's API key can be injected into an environment variable by setting ``inject``
+attribute to ``api_key`` (e.g. ``inject="api_key"``).
+
+```xml
+<environment_variables>
+    <environment_variable name="GALAXY_API_KEY" inject="api_key" />
+</environment_variables>
+```
+
+The framework allows setting this via environment variable and not via templating variables
+in order to discourage setting the actual values of these keys as command line arguments.
+On shared systems this provides some security by preventing a simple process listing command
+from exposing keys.
 ]]></xs:documentation>
     </xs:annotation>
     <xs:simpleContent>
@@ -4686,15 +4701,27 @@ variable instead of shell variable.
 define.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="inject" type="EnvironmentVariableInject" gxdocs:added="19.09">
+          <xs:annotation>
+            <xs:documentation xml:lang="en">Special variable to inject into the environment variable. Currently 'api_key' is the only option and will cause the user's API key to be injected via this environment variable.</xs:documentation>
+          </xs:annotation>
+        </xs:attribute>
         <xs:attribute name="strip" type="PermissiveBoolean" default="false">
-        <xs:annotation>
-          <xs:documentation xml:lang="en">Whether to strip leading and trailing whitespace from the calculated value before exporting the environment variable.</xs:documentation>
-        </xs:annotation>
+          <xs:annotation>
+            <xs:documentation xml:lang="en">Whether to strip leading and trailing whitespace from the calculated value before exporting the environment variable.</xs:documentation>
+          </xs:annotation>
       </xs:attribute>
       </xs:extension>
     </xs:simpleContent>
   </xs:complexType>
-
+  <xs:simpleType name="EnvironmentVariableInject">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"></xs:documentation>
+    </xs:annotation>
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="api_key" />
+    </xs:restriction>
+  </xs:simpleType>
   <xs:complexType name="ConfigFiles">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[See

lib/galaxy/tools/data/__init__.py

@@ -382,21 +382,21 @@ class TabularToolDataTable(ToolDataTable, Dictifiable):
                 filename = os.path.join(tool_data_path, filename)
             if self.tool_data_path_files.exists(filename):
                 found = True
-            elif self.tool_data_path_files.exists("%s.sample" % filename) and not from_shed_config:
-                log.info("Could not find tool data %s, reading sample" % filename)
-                filename = "%s.sample" % filename
-                found = True
             else:
                 # Since the path attribute can include a hard-coded path to a specific directory
                 # (e.g., <file path="tool-data/cg_crr_files.loc" />) which may not be the same value
                 # as self.tool_data_path, we'll parse the path to get the filename and see if it is
                 # in self.tool_data_path.
                 file_path, file_name = os.path.split(filename)
-                if file_path and file_path != self.tool_data_path:
+                if file_path != self.tool_data_path:
                     corrected_filename = os.path.join(self.tool_data_path, file_name)
                     if self.tool_data_path_files.exists(corrected_filename):
                         filename = corrected_filename
                         found = True
+                    elif not from_shed_config and self.tool_data_path_files.exists("%s.sample" % corrected_filename):
+                        log.info("Could not find tool data %s, reading sample" % corrected_filename)
+                        filename = "%s.sample" % corrected_filename
+                        found = True
 
             errors = []
             if found:

lib/galaxy/tools/evaluation.py

@@ -69,7 +69,7 @@ class ToolEvaluator(object):
         incoming = self.tool.params_from_strings(incoming, self.app)
 
         # Full parameter validation
-        request_context = WorkRequestContext(app=self.app, user=job.history and job.history.user, history=job.history)
+        request_context = WorkRequestContext(app=self.app, user=self._user, history=self._history)
 
         def validate_inputs(input, value, context, **kwargs):
             value = input.from_json(value, request_context, context)
@@ -135,7 +135,9 @@ class ToolEvaluator(object):
 
         param_dict["input"] = input
         param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.join(job_working_directory, 'registry.xml')
-
+        if self._history:
+            param_dict['__history_id__'] = self.app.security.encode_id(self._history.id)
+        param_dict['__galaxy_url__'] = self.compute_environment.galaxy_url()
         param_dict.update(self.tool.template_macro_params)
         # All parameters go into the param_dict
         param_dict.update(incoming)
@@ -536,9 +538,19 @@ class ToolEvaluator(object):
             directory = self.local_working_directory
             environment_variable = environment_variable_def.copy()
             environment_variable_template = environment_variable_def["template"]
+            inject = environment_variable_def.get("inject")
+            if inject == "api_key":
+                if self._user:
+                    from galaxy.managers import api_keys
+                    environment_variable_template = api_keys.ApiKeyManager(self.app).get_or_create_api_key(self._user)
+                else:
+                    environment_variable_template = ""
+                is_template = False
+            else:
+                is_template = True
             fd, config_filename = tempfile.mkstemp(dir=directory)
             os.close(fd)
-            self.__write_workdir_file(config_filename, environment_variable_template, param_dict, strip=environment_variable_def.get("strip", False))
+            self.__write_workdir_file(config_filename, environment_variable_template, param_dict, is_template=is_template, strip=environment_variable_def.get("strip", False))
             config_file_basename = os.path.basename(config_filename)
             # environment setup in job file template happens before `cd $working_directory`
             environment_variable["value"] = '`cat "$_GALAXY_JOB_DIR/%s"`' % config_file_basename
@@ -624,3 +636,12 @@ class ToolEvaluator(object):
         compat.
         """
         return self.compute_environment.sep().join(args)
+
+    @property
+    def _history(self):
+        return self.job.history
+
+    @property
+    def _user(self):
+        history = self._history
+        return history and history.user

lib/galaxy/webapps/galaxy/controllers/user.py

@@ -227,7 +227,7 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin):
             # while sometimes, so we don't want to block on logout.
             send_local_control_task(trans.app,
                                     "recalculate_user_disk_usage",
-                                    {"user_id": trans.security.encode_id(trans.user.id)})
+                                    kwargs={"user_id": trans.security.encode_id(trans.user.id)})
         # Since logging an event requires a session, we'll log prior to ending the session
         trans.log_event("User logged out")
         trans.handle_user_logout(logout_all=logout_all)

test/functional/tools/environment_variables_inject.xml

@@ -0,0 +1,42 @@
+<tool id="environment_variables_inject" name="environment_variables_inject" version="1.0.0">
+    <environment_variables>
+        <!-- Avoid putting actual API keys in the command-line like this in production tools.
+             Every effort should be made to consume these values as enviornment variables. -->
+        <environment_variable name="GX_API" inject="api_key" />
+        <environment_variable name="GX_HISTORY_ID">$__history_id__</environment_variable>
+        <environment_variable name="GX_URL">$__galaxy_url__</environment_variable>
+    </environment_variables>
+    <command>
+        echo "\$GX_API"         > '$out_file_api_key';
+        echo "\$GX_URL"         > '$out_file_galaxy_url';
+        echo "\$GX_HISTORY_ID"  > '$out_file_history_id'
+    </command>
+    <inputs>
+        <param name="inttest" value="1" type="integer" />
+    </inputs>
+    <outputs>
+        <data name="out_file_api_key" format="txt" />
+        <data name="out_file_history_id" format="txt" />
+        <data name="out_file_galaxy_url" format="txt" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="inttest" value="2" />
+            <output name="out_file_api_key">
+                <assert_contents>
+                    <has_line_matching expression="[0-9a-zA-Z]{32}" />
+                </assert_contents>
+            </output>
+            <output name="out_file_history_id">
+                <assert_contents>
+                    <has_line_matching expression="[0-9a-zA-Z]{16}" />
+                </assert_contents>
+            </output>
+            <output name="out_file_galaxy_url">
+                <assert_contents>
+                    <has_line_matching expression="https?.*" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/interactivetool_simple.xml

@@ -1,4 +1,4 @@
-<tool id="interactive_tool_simple" name="interactivetool_simple" tool_type="interactive" version="0.1">
+<tool id="interactivetool_simple" name="interactivetool_simple" tool_type="interactive" version="0.1">
     <requirements>
         <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>

test/functional/tools/interactivetool_two_entry_points.xml

@@ -1,4 +1,4 @@
-<tool id="interactive_tool_two_entry_points" name="interactivetool_two_entry_points" tool_type="interactive" version="0.1">
+<tool id="interactivetool_two_entry_points" name="interactivetool_two_entry_points" tool_type="interactive" version="0.1">
     <requirements>
         <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>

test/functional/tools/samples_tool_conf.xml

@@ -8,6 +8,7 @@
   <tool file="boolean_conditional.xml" />
   <tool file="composite.xml" />
   <tool file="environment_variables.xml" />
+  <tool file="environment_variables_inject.xml" />
   <tool file="code_file.xml" />
   <tool file="disambiguate_cond.xml" />
   <tool file="multi_repeats.xml"/>

test/unit/tool_util/test_parsing.py

@@ -618,6 +618,16 @@ class CollectionOutputYamlTestCase(BaseLoaderTestCase):
         assert len(output_collections) == 1
 
 
+class EnvironmentVariablesTestCase(BaseLoaderTestCase):
+    source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/environment_variables.xml")
+    source_contents = None
+
+    def test_tests(self):
+        tests_dict = self._tool_source.parse_tests_to_dict()
+        tests = tests_dict["tests"]
+        assert len(tests) == 1
+
+
 class ExpectationsTestCase(BaseLoaderTestCase):
     source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/detect_errors.xml")
     source_contents = None

test/unit/tools/test_evaluation.py

@@ -33,6 +33,7 @@ from ..tools_support import UsesApp
 # To Test:
 # - param_file handling.
 TEST_TOOL_DIRECTORY = "/path/to/the/tool"
+TEST_GALAXY_URL = "http://mycool.galaxyproject.org:8456"
 
 
 class ToolEvaluatorTestCase(TestCase, UsesApp):
@@ -42,6 +43,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
         self.tool = MockTool(self.app)
         self.job = Job()
         self.job.history = History()
+        self.job.history.id = 42
         self.job.parameters = [JobParameter(name="thresh", value="4")]
         self.evaluator = ToolEvaluator(self.app, self.tool, self.job, self.test_directory)
 
@@ -65,6 +67,18 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
         command_line, extra_filenames, _ = self.evaluator.build()
         self.assertEqual(command_line, "prog1  4 5")
 
+    def test_eval_galaxy_url(self):
+        self.tool._command_line = "prog1 $__galaxy_url__"
+        self._set_compute_environment()
+        command_line, extra_filenames, _ = self.evaluator.build()
+        self.assertEqual(command_line, "prog1 %s" % TEST_GALAXY_URL)
+
+    def test_eval_history_id(self):
+        self.tool._command_line = "prog1 '$__history_id__'"
+        self._set_compute_environment()
+        command_line, extra_filenames, _ = self.evaluator.build()
+        self.assertEqual(command_line, "prog1 '%s'" % self.app.security.encode_id(42))
+
     def test_conditional_evaluation(self):
         select_xml = XML('''<param name="always_true" type="select"><option value="true">True</option></param>''')
         parameter = SelectToolParameter(self.tool, select_xml)
@@ -264,6 +278,9 @@ class TestComputeEnvironment(SimpleComputeEnvironment):
     def tool_directory(self):
         return TEST_TOOL_DIRECTORY
 
+    def galaxy_url(self):
+        return TEST_GALAXY_URL
+
 
 class MockTool(object):
 

tools/interactive/interactivetool_askomics.xml

@@ -10,15 +10,8 @@
         </entry_point>
     </entry_points>
     <environment_variables>
-        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
-        <environment_variable name="API_KEY" strip="True">
-            #if $__user__:
-                #for $api_key in $__user__.api_keys:
-                    ${api_key.key}
-                    #break
-                #end for
-            #end if
-        </environment_variable>
+        <environment_variable name="GALAXY_URL">$__galaxy_url__</environment_variable>
+        <environment_variable name="API_KEY" inject="api_key" />
     </environment_variables>
     <command><![CDATA[
         #import re

tools/interactive/interactivetool_jupyter_notebook.xml

@@ -9,18 +9,11 @@
         </entry_point>
     </entry_points>
     <environment_variables>
-        <environment_variable name="HISTORY_ID" strip="True">${__app__.security.encode_id($jupyter_notebook.history_id)}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
-        <environment_variable name="REMOTE_HOST">${__app__.config.galaxy_infrastructure_url}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
+        <environment_variable name="HISTORY_ID">$__history_id__</environment_variable>
+        <environment_variable name="REMOTE_HOST">$__galaxy_url__</environment_variable>
         <environment_variable name="GALAXY_WEB_PORT">8080</environment_variable>
-        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
-        <environment_variable name="API_KEY" strip="True">
-            #if $__user__:
-                #for $api_key in $__user__.api_keys:
-                    ${api_key.key}
-                    #break
-                #end for
-            #end if
-        </environment_variable> <!-- FIXME: We should have a better way to get user's API key -->
+        <environment_variable name="GALAXY_URL">$__galaxy_url__</environment_variable>
+        <environment_variable name="API_KEY" inject="api_key" />
     </environment_variables>
     <command detect_errors="aggressive"><![CDATA[
         #import re