opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement request header and body templating

Browse Source
This commit is contained in:
mvdbeek
2024-02-13 14:51:16 +01:00
parent 2f2930015f
commit 5aabab9c93
5 changed files with 200 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_util/xsd/galaxy.xsd
+35 -4
View File
@@ -4039,6 +4039,16 @@ of "type" specified for this expression block.
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="RequestMethodType">
<xs:annotation>
<xs:documentation xml:lang="en">Select a request method, defaults to GET if unspecified</xs:documentation>
</xs:annotation>
<xs:restriction base="xs:string">
<xs:enumeration value="GET" />
<xs:enumeration value="POST" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="ParamSelectOption">
<xs:annotation>
<xs:documentation xml:lang="en"><![CDATA[
@@ -4413,6 +4423,11 @@ used to generate dynamic options.
<xs:documentation xml:lang="en">Determine options from data hosted at specified URL. </xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="request_method" type="RequestMethodType">
<xs:annotation>
<xs:documentation xml:lang="en">Set the request method to use for options provided using from_url. </xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="from_parameter" type="xs:string">
<xs:annotation>
<xs:documentation xml:lang="en">Deprecated.</xs:documentation>
@@ -4446,10 +4461,12 @@ used to generate dynamic options.
</xs:complexType>
<xs:group name="OptionsElement">
<xs:choice>
<xs:element name="filter" type="Filter" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="column" type="Column" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="validator" type="Validator" minOccurs="0" maxOccurs="1"/>
<xs:element name="postprocess_expression" type="Expression" minOccurs="0" maxOccurs="1"/>
<xs:element name="filter" type="Filter" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="column" type="Column" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="validator" type="Validator" minOccurs="0" maxOccurs="1" />
<xs:element name="postprocess_expression" type="Expression" minOccurs="0" maxOccurs="1" />
<xs:element name="request_body" type="RequestBody" minOccurs="0" maxOccurs="1" />
<xs:element name="request_headers" type="RequestHeaders" minOccurs="0" maxOccurs="1" />
<xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded">
<xs:annotation>
<xs:documentation xml:lang="en">Documentation for file</xs:documentation>
@@ -4458,6 +4475,20 @@ used to generate dynamic options.
<xs:element name="option" type="ParamDrillDownOption" minOccurs="0" maxOccurs="unbounded"/>
</xs:choice>
</xs:group>
<xs:complexType name="RequestBody">
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="type" type="xs:string"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:complexType name="RequestHeaders">
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="type" type="xs:string"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:complexType name="Column">
<xs:annotation>
<xs:documentation xml:lang="en"><![CDATA[Optionally contained within an
lib/galaxy/tools/parameters/dynamic_options.py
+78 -15
View File
@@ -4,10 +4,21 @@ on the values of other parameters or other aspects of the current state)
"""
import copy
import json
import logging
import os
import re
from dataclasses import dataclass
from io import StringIO
from typing import (
Any,
cast,
Dict,
get_args,
Optional,
)
from typing_extensions import Literal
from galaxy.model import (
DatasetCollectionElement,
@@ -17,7 +28,10 @@ from galaxy.model import (
User,
)
from galaxy.tools.expressions import do_eval
from galaxy.util import string_as_bool
from galaxy.util import (
Element,
string_as_bool,
)
from galaxy.util.template import fill_template
from . import validation
from .cancelable_request import request
@@ -538,7 +552,7 @@ filter_types = dict(
class DynamicOptions:
"""Handles dynamically generated SelectToolParameter options"""
def __init__(self, elem, tool_param):
def __init__(self, elem: Element, tool_param):
def load_from_parameter(from_parameter, transform_lines=None):
obj = self.tool_param
for field in from_parameter.split("."):
@@ -548,7 +562,7 @@ class DynamicOptions:
return self.parse_file_fields(obj)
self.tool_param = tool_param
self.columns = {}
self.columns: Dict[str, int] = {}
self.filters = []
self.file_fields = None
self.largest_index = 0
@@ -568,11 +582,7 @@ class DynamicOptions:
dataset_file = elem.get("from_dataset", None)
from_parameter = elem.get("from_parameter", None)
self.tool_data_table_name = elem.get("from_data_table", None)
self.from_url = elem.get("from_url")
self.from_url_postprocess = None
from_url_postprocess = elem.find("postprocess_expression")
if from_url_postprocess is not None:
self.from_url_postprocess = from_url_postprocess.text.strip()
self.from_url_options = parse_from_url_options(elem)
# Options are defined from a data table loaded by the app
self._tool_data_table = None
self.elem = elem
@@ -781,25 +791,35 @@ class DynamicOptions:
else:
return [str(values[0]), str(values[1]), bool(values[2])]
if self.from_url:
if from_url_options := self.from_url_options:
context = User.user_template_environment(trans.user)
url = fill_template(self.from_url, context)
url = fill_template(from_url_options.from_url, context)
request_body = template_or_none(from_url_options.request_body, context)
request_headers = template_or_none(from_url_options.request_headers, context)
try:
unset_value = object()
cached_value = trans.get_cache_value(url, unset_value)
cached_value = trans.get_cache_value(
(url, from_url_options.request_method, request_body, request_headers), unset_value
)
if cached_value is unset_value:
data = request(url, timeout=10)
trans.set_cache_value(url, data)
data = request(
url=url,
method=from_url_options.request_method,
data=json.loads(request_body) if request_body else None,
headers=json.loads(request_headers) if request_headers else None,
timeout=10,
)
trans.set_cache_value((url, from_url_options.request_method, request_body, request_headers), data)
else:
data = cached_value
except Exception as e:
log.warning("Fetching from url '%s' failed: %s", url, str(e))
data = None
if self.from_url_postprocess:
if from_url_options.postprocess_expression:
try:
data = do_eval(
self.from_url_postprocess,
from_url_options.postprocess_expression,
data,
)
except Exception as eval_error:
@@ -836,6 +856,49 @@ class DynamicOptions:
return int(column_spec)
REQUEST_METHODS = Literal["GET", "POST"]
@dataclass
class FromUrlOptions:
from_url: str
request_method: REQUEST_METHODS
request_body: Optional[str]
request_headers: Optional[str]
postprocess_expression: Optional[str]
def strip_or_none(maybe_string: Optional[Element]) -> Optional[str]:
if maybe_string is not None:
if maybe_string.text:
return maybe_string.text.strip()
return None
def parse_from_url_options(elem: Element) -> Optional[FromUrlOptions]:
from_url = elem.get("from_url")
if from_url:
request_method = cast(Literal["GET", "POST"], elem.get("request_method", "GET"))
assert request_method in get_args(REQUEST_METHODS)
request_headers = strip_or_none(elem.find("request_headers"))
request_body = strip_or_none(elem.find("request_body"))
postprocess_expression = strip_or_none(elem.find("postprocess_expression"))
return FromUrlOptions(
from_url,
request_method=request_method,
request_headers=request_headers,
request_body=request_body,
postprocess_expression=postprocess_expression,
)
return None
def template_or_none(template: Optional[str], context: Dict[str, Any]) -> Optional[str]:
if template:
return fill_template(template, context=context)
return None
def _get_ref_data(other_values, ref_name):
"""
get the list of data sets from ref_name
lib/galaxy/work/context.py
+6 -5
View File
@@ -4,6 +4,7 @@ from typing import (
Dict,
List,
Optional,
Tuple,
)
from typing_extensions import Literal
@@ -44,15 +45,15 @@ class WorkRequestContext(ProvidesHistoryContext):
self.__user_current_roles: Optional[List[Role]] = None
self.__history = history
self._url_builder = url_builder
self._short_term_cache: Dict[str, Any] = {}
self._short_term_cache: Dict[Tuple[str, ...], Any] = {}
self.workflow_building_mode = workflow_building_mode
self.galaxy_session = galaxy_session
def set_cache_value(self, key: str, value: Any):
self._short_term_cache[key] = value
def set_cache_value(self, args: Tuple[str, ...], value: Any):
self._short_term_cache[args] = value
def get_cache_value(self, key: str, default: Any = None) -> Any:
return self._short_term_cache.get(key, default)
def get_cache_value(self, args: Tuple[str, ...], default: Any = None) -> Any:
return self._short_term_cache.get(args, default)
@property
def app(self):
test/functional/tools/select_from_url.xml
+24 -1
View File
@@ -2,7 +2,8 @@
<command><![CDATA[
echo '$url_param_value' > '$param_value' &&
echo '$url_param_value_postprocessed' > '$param_value_postprocessed' &&
echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postprocessed'
echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postprocessed' &&
echo '$url_param_value_header_and_body' > '$param_value_header_and_body'
]]></command>
<inputs>
<param name="url_param_value" type="select">
@@ -33,17 +34,34 @@ echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postproces
}]]></postprocess_expression>
</options>
</param>
<param name="url_param_value_header_and_body" type="select">
<options from_url="https://postman-echo.com/post" request_method="POST">
<!-- Example for accessing user secrets via extra preferences -->
<request_headers type="json">
{"x-api-key": "${__user__.extra_preferences.fake_api_key if $__user__ else "anon"}"}
</request_headers>
<request_body type="json">
{"name": "value"}
</request_body>
<!-- https://postman-echo.com/post echos values sent to it, so here's we're listing the response headers -->
<postprocess_expression type="ecma5.1"><![CDATA[${
return Object.keys(inputs.headers).map((header) => [header, header])
}]]></postprocess_expression>
</options>
</param>
</inputs>
<outputs>
<data format="txt" label="url param value" name="param_value"></data>
<data format="txt" label="url param value postprocessed" name="param_value_postprocessed"></data>
<data format="txt" label="invalid url param value postprocessed" name="invalid_param_value_postprocessed"></data>
<data format="txt" label="param value for header and body request" name="param_value_header_and_body"></data>
</outputs>
<tests>
<test>
<param name="url_param_value" value="dm6" />
<param name="url_param_value_postprocessed" value="chr2L" />
<param name="invalid_url_param_value_postprocessed" value="default" />
<param name="url_param_value_header_and_body" value="x-api-key" />
<output name="param_value">
<assert_contents>
<has_text text="dm6"></has_text>
@@ -59,6 +77,11 @@ echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postproces
<has_text text="default"></has_text>
</assert_contents>
</output>
<output name="param_value_header_and_body">
<assert_contents>
<has_text text="x-api-key"></has_text>
</assert_contents>
</output>
</test>
</tests>
</tool>
test/unit/app/tools/test_dynamic_options.py
+57
View File
@@ -0,0 +1,57 @@
from galaxy.app_unittest_utils.galaxy_mock import MockApp
from galaxy.tools.parameters.dynamic_options import DynamicOptions
from galaxy.util import XML
from galaxy.util.bunch import Bunch
from galaxy.work.context import WorkRequestContext
def get_from_url_option():
return DynamicOptions(
XML(
"""
<options from_url="https://usegalaxy.org/api/genomes/dm6" request_method="POST">
<request_headers type="json">
{"x-api-key": "${__user__.extra_preferences.resource_api_key if $__user__ else "anon"}"}
</request_headers>
<request_body type="json">
{"some_key": "some_value"}
</request_body>
<postprocess_expression type="ecma5.1"><![CDATA[${
if (inputs) {
return Object.values(inputs.chrom_info).map((v) => [v.chrom, v.len])
} else {
return [["The fallback value", "default"]]
}
}]]></postprocess_expression>
</options>
"""
),
Bunch(),
)
def test_dynamic_option_parsing():
from_url_option = get_from_url_option()
assert from_url_option.from_url_options
assert from_url_option.from_url_options.from_url == "https://usegalaxy.org/api/genomes/dm6"
def test_dynamic_option_cache():
app = MockApp()
trans = WorkRequestContext(app=app)
from_url_option = get_from_url_option()
options = from_url_option.from_url_options
assert options
args = (options.from_url, options.request_method, options.request_body, '{"x-api-key": "anon"}')
trans.set_cache_value(
args,
{
"id": "dm6",
"reference": True,
"chrom_info": [{"chrom": "chr2L", "len": 23513712}],
"prev_chroms": False,
"next_chroms": False,
"start_index": 0,
},
)
assert from_url_option.get_options(trans, {}) == [["chr2L", "23513712", False]]