that support pagination to 10000. Clients can always request more data via offset
requests, but since this is not a streaming API we need to build the
response in memory before we can start sending.
and make sure we include the parent_hda when exporting histories.
Fixes
https://sentry.galaxyproject.org/share/issue/20ee27f675ef476588bbe8ff273eaee7/:
```
KeyError: '31e7840b5aedca43523bfcc8e2e9e56f'
File "galaxy/jobs/__init__.py", line 2039, in finish
task_wrapper = self.tool.exec_after_process(
File "galaxy/tools/__init__.py", line 3100, in exec_after_process
JobImportHistoryArchiveWrapper(self.app, job.id).cleanup_after_job()
File "galaxy/tools/imp_exp/__init__.py", line 81, in cleanup_after_job
model_store.perform_import(new_history, job=job, new_history=True)
File "galaxy/model/store/__init__.py", line 416, in perform_import
self._import_implicit_dataset_conversions(object_import_tracker)
File "galaxy/model/store/__init__.py", line 1280, in _import_implicit_dataset_conversions
idc.parent_hda = object_import_tracker.hdas_by_key[idc_attrs["parent_hda"]]
```
stores
The problem is that `object_store.is_private` would check in which
object store the dataset is stored, and that fails if we haven't written
to the object store yet (which is why extended metadata is a
workaround).
Fixes https://github.com/galaxyproject/galaxy/issues/17208
Fixes a bug reported by @jennaj where using an input dataset from a history
import (which contains implicitly converted datasets in discarded state)
in an input that would require that exact same implicit conversion
would fail with `Job output deleted by user before job completed`.
This will make the whole implicit coversion system much more robust,
since we didn't even exclude datasets in error state, so as soon as one
conversion failed ... you'd be stuck.
Fixes
https://sentry.galaxyproject.org/share/issue/c1c4c9b0ab324b4ea4f98c3b8c402d21/:
```
Message
Uncaught Exception
Stack Trace
Newest
FileNotFoundError: [Errno 2] No such file or directory: ''
File "galaxy/web/framework/middleware/error.py", line 167, in __call__
app_iter = self.application(environ, sr_checker)
File "galaxy/web/framework/middleware/statsd.py", line 29, in __call__
req = self.application(environ, start_response)
File "/cvmfs/main.galaxyproject.org/venv/lib/python3.11/site-packages/paste/httpexceptions.py", line 635, in __call__
return self.application(environ, start_response)
File "galaxy/web/framework/base.py", line 176, in __call__
return self.handle_request(request_id, path_info, environ, start_response)
File "galaxy/web/framework/base.py", line 271, in handle_request
body = method(trans, **kwargs)
File "galaxy/webapps/galaxy/controllers/dataset.py", line 155, in display
display_data, headers = data.datatype.display_data(
File "galaxy/datatypes/tabular.py", line 226, in display_data
chunk=self.get_chunk(trans, dataset, 0),
File "galaxy/datatypes/tabular.py", line 149, in get_chunk
ck_data, last_read = self._read_chunk(trans, dataset, offset, ck_size)
File "galaxy/datatypes/tabular.py", line 159, in _read_chunk
with compression_utils.get_fileobj(dataset.get_file_name()) as f:
File "galaxy/util/compression_utils.py", line 79, in get_fileobj
return get_fileobj_raw(filename, mode, compressed_formats)[1]
File "galaxy/util/compression_utils.py", line 139, in get_fileobj_raw
return compressed_format, open(filename, mode, encoding="utf-8")
```
That happens if the user stops a job before it's queued (or if we have a
job rule preventing queuing).
Fixes
https://sentry.galaxyproject.org/share/issue/8b462ee0d50a4de8bb9ca2c50f049b42/:
```
Message
Could not recover job metrics
Stack Trace
Newest
JobMappingException: This tool is currently being beta-tested and your account has not been given access.
File "galaxy/jobs/mapper.py", line 214, in __handle_rule
job_destination = self.__invoke_expand_function(
File "galaxy/jobs/mapper.py", line 125, in __invoke_expand_function
return expand_function(**actual_args)
File "tpv/rules/gateway.py", line 51, in map_tool_to_destination
return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, job_wrapper, resource_params,
File "tpv/core/mapper.py", line 168, in map_to_destination
evaluated_entity = self.match_combine_evaluate_entities(context, tool, user)
File "tpv/core/mapper.py", line 138, in match_combine_evaluate_entities
evaluated_entity = combined_entity.evaluate_rules(context)
File "tpv/core/entities.py", line 489, in evaluate_rules
rule = rule.evaluate(context)
File "tpv/core/entities.py", line 750, in evaluate
raise JobMappingException(
JobMappingException: This tool is currently being beta-tested and your account has not been given access.
File "galaxy/jobs/handler.py", line 719, in __verify_job_ready
job_destination = job_wrapper.job_destination
File "galaxy/jobs/__init__.py", line 2576, in job_destination
return self.job_runner_mapper.get_job_destination(self.params)
File "galaxy/jobs/mapper.py", line 276, in get_job_destination
return self.__cache_job_destination(params)
File "galaxy/jobs/mapper.py", line 257, in __cache_job_destination
self.cached_job_destination = self.__determine_job_destination(
File "galaxy/jobs/mapper.py", line 241, in __determine_job_destination
job_destination = self.__handle_dynamic_job_destination(raw_job_destination)
File "galaxy/jobs/mapper.py", line 204, in __handle_dynamic_job_destination
return self.__handle_rule(expand_function, destination)
File "galaxy/jobs/mapper.py", line 219, in __handle_rule
raise e
File "galaxy/jobs/mapper.py", line 208, in __handle_rule
job_destination = self.__invoke_expand_function(rule_function, destination)
File "galaxy/jobs/mapper.py", line 125, in __invoke_expand_function
return expand_function(**actual_args)
File "tpv/rules/gateway.py", line 51, in map_tool_to_destination
return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, job_wrapper, resource_params,
File "tpv/core/mapper.py", line 168, in map_to_destination
evaluated_entity = self.match_combine_evaluate_entities(context, tool, user)
File "tpv/core/mapper.py", line 138, in match_combine_evaluate_entities
evaluated_entity = combined_entity.evaluate_rules(context)
File "tpv/core/entities.py", line 489, in evaluate_rules
rule = rule.evaluate(context)
File "tpv/core/entities.py", line 750, in evaluate
raise JobMappingException(
ObjectNotFound: objectstore, _call_method failed: _get_filename on <galaxy.model.Job(60599388) at 0x7fcaec5b6450>, kwargs: {'base_dir': 'job_work', 'dir_only': True, 'obj_dir': True}
File "galaxy/jobs/__init__.py", line 2163, in _collect_metrics
job_metrics_directory = self.working_directory
File "galaxy/jobs/__init__.py", line 1317, in working_directory
self.__working_directory = self.app.object_store.get_filename(
File "galaxy/objectstore/__init__.py", line 475, in get_filename
return self._invoke("get_filename", obj, **kwargs)
File "galaxy/objectstore/__init__.py", line 451, in _invoke
return self.__getattribute__(f"_{delegate}")(obj=obj, **kwargs)
File "galaxy/objectstore/__init__.py", line 1009, in _get_filename
return self._call_method("_get_filename", obj, ObjectNotFound, True, **kwargs)
File "galaxy/objectstore/__init__.py", line 1281, in _call_method
raise default(
```
Minor follow up to https://github.com/galaxyproject/galaxy/pull/18809
Fixes
https://sentry.galaxyproject.org/share/issue/5b8d495edd6545278d4d93d0ed69ac3a/:
```
Message
Uncaught exception in exposed API method:
Stack Trace
Newest
gaierror: [Errno -2] Name or service not known
File "urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "socket.py", line 962, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f8e35406450>: Failed to establish a new connection: [Errno -2] Name or service not known
File "urllib3/connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
File "urllib3/connectionpool.py", line 416, in _make_request
conn.request(method, url, **httplib_request_kw)
File "urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "http/client.py", line 1286, in request
self._send_request(method, url, body, headers, encode_chunked)
File "http/client.py", line 1332, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "http/client.py", line 1281, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "http/client.py", line 1041, in _send_output
self.send(msg)
File "http/client.py", line 979, in send
self.connect()
File "urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
MaxRetryError: HTTPConnectionPool(host='https', port=80): Max retries exceeded with url: //training.galaxyproject.org/training-material//api/ga4gh/trs/v2/tools/variant-analysis-microbial-variants/versions/microbial_variant_calling/GALAXY/descriptor (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f8e35406450>: Failed to establish a new connection: [Errno -2] Name or service not known'))
File "requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "urllib3/connectionpool.py", line 799, in urlopen
retries = retries.increment(
File "urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
ConnectionError: HTTPConnectionPool(host='https', port=80): Max retries exceeded with url: //training.galaxyproject.org/training-material//api/ga4gh/trs/v2/tools/variant-analysis-microbial-variants/versions/microbial_variant_calling/GALAXY/descriptor (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f8e35406450>: Failed to establish a new connection: [Errno -2] Name or service not known'))
File "galaxy/web/framework/decorators.py", line 346, in decorator
rval = func(self, trans, *args, **kwargs)
File "galaxy/webapps/galaxy/api/workflows.py", line 261, in create
archive_data = server.get_version_descriptor(trs_tool_id, trs_version_id)
File "galaxy/workflow/trs_proxy.py", line 115, in get_version_descriptor
return self._get(trs_api_url)["content"]
File "galaxy/workflow/trs_proxy.py", line 126, in _get
response = requests.get(url, params=params, timeout=DEFAULT_SOCKET_TIMEOUT)
File "galaxy/util/requests.py", line 29, in wrapper
rval = f(*args, **kwargs)
File "requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "requests/adapters.py", line 519, in send
raise ConnectionError(e, request=request)
```
which happened for https://usegalaxy.org/workflows/trs_import?run_form=true&trs_url=http%3A%2F%2Flocalhost%3A4000%2F%2Ftraining-material%2F%2Fapi%2Fga4gh%2Ftrs%2Fv2%2Ftools%2Fvariant-analysis-microbial-variants%2Fversions%2Fmicrobial_variant_calling"
mvdbeek