Infer filename from remote location URL

When no `file` or `value` is given for test inputs or outputs.
This commit is contained in:
davelopez
2023-02-16 14:46:51 +01:00
parent dd37b5ed38
commit 978de0113c
4 changed files with 27 additions and 13 deletions
+5 -1
View File
@@ -1,6 +1,7 @@
import json
import logging
import math
import os
import re
import uuid
from typing import (
@@ -697,7 +698,10 @@ def __parse_test_attributes(output_elem, attrib, parse_elements=False, parse_dis
attributes["sort"] = string_as_bool(attrib.pop("sort", False))
attributes["decompress"] = string_as_bool(attrib.pop("decompress", False))
# `location` may contain an URL to a remote file that will be used to download `file` (if not already present on disk).
attributes["location"] = attrib.get("location")
location = attrib.get("location")
if location and file is None:
file = os.path.basename(location) # If no file specified, try to get filename from URL last component
attributes["location"] = location
try:
attributes["count"] = int(attrib.pop("count"))
except KeyError:
+12 -9
View File
@@ -176,12 +176,11 @@ class RemoteLocationDataResolver(FileDataResolver):
return
if not is_url(location):
raise ValueError(f"Invalid 'location' URL for remote test data provided: {location}")
if filename:
if self._is_direct_url_paste_upload(filename, location):
return # No pre-download required
self._ensure_base_dir_exists()
dest_file_path = self.path(filename)
download_to_file(location, dest_file_path)
if not self._is_valid_filename(filename):
raise ValueError(f"Invalid 'filename' provided: '{filename}'")
self._ensure_base_dir_exists()
dest_file_path = self.path(filename)
download_to_file(location, dest_file_path)
def _ensure_base_dir_exists(self):
if not os.path.exists(self.file_dir):
@@ -201,6 +200,10 @@ class RemoteLocationDataResolver(FileDataResolver):
f"Failed to validate test data '{filename}' with [{hash_function}] - expected [{expected_hash_value}] got [{calculated_hash_value}]"
)
def _is_direct_url_paste_upload(self, filename: str, location: Optional[str]):
"""Checks if the test data file is an URL and will be directly url_pasted to Galaxy."""
return location and filename == location and is_url(location)
def _is_valid_filename(self, filename: str):
"""
Checks that the filename does not contain the following
characters: <, >, :, ", /, \\, |, ?, *, or any control characters.
"""
pattern = r"^[^<>:\"/\\|?*\x00-\x1F]+$"
return bool(re.match(pattern, filename))
+2 -2
View File
@@ -184,8 +184,8 @@ def _process_raw_inputs(
param_extra = raw_input_dict["attributes"]
location = param_extra.get("location")
if param_value is None and location:
# TODO: I bet there is a better way of signaling that we are going to use the location and not the value...
param_value = location
# If no value is given, we try to get the file name directly from the URL
param_value = os.path.basename(location)
if not value.type == "text":
param_value = _split_if_str(param_value)
if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
@@ -10,7 +10,8 @@
</outputs>
<tests>
<test>
<!-- When only the `location` is defined, it will be directly URL pasted to Galaxy -->
<!-- When only the `location` is defined, the name of the input file will be infered from the last component of the URL.
In this example, it will be equivalent to `value="hello.txt"`. -->
<param name="input" location="https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt"/>
<output name="output">
<assert_contents>
@@ -53,5 +54,11 @@
<param name="input" value="not_local_input.txt" location="https://raw.githubusercontent.com/galaxyproject/planemo/master/tests/data/not_hello.txt" checksum="sha1$3436387a8a45b00ef11e621e501ba23b52f06101"/>
<output name="output" file="not_local_output.txt" location="https://raw.githubusercontent.com/galaxyproject/planemo/master/tests/data/not_hello.txt" checksum="sha1$3436387a8a45b00ef11e621e501ba23b52f06101"/>
</test>
<test>
<!-- If the output `file` or `value` is not specified, the last component of the location URL will be used
as filename. In this example, this will be equivalent to `file="not_hello.txt"`. -->
<param name="input" value="not_local_input.txt" location="https://raw.githubusercontent.com/galaxyproject/planemo/master/tests/data/not_hello.txt" checksum="sha1$3436387a8a45b00ef11e621e501ba23b52f06101"/>
<output name="output" location="https://raw.githubusercontent.com/galaxyproject/planemo/master/tests/data/not_hello.txt" checksum="sha1$3436387a8a45b00ef11e621e501ba23b52f06101"/>
</test>
</tests>
</tool>