diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index cd7de5fc94d..a2f7a10a195 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -22,7 +22,10 @@ from galaxy.datatypes import sniff from galaxy.datatypes.registry import Registry as DatatypesRegistry from galaxy.jobs import MinimalJobWrapper from galaxy.managers.collections import DatasetCollectionManager -from galaxy.managers.datasets import DatasetAssociationManager +from galaxy.managers.datasets import ( + DatasetAssociationManager, + DatasetManager, +) from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager from galaxy.managers.markdown_util import generate_branded_pdf @@ -30,6 +33,7 @@ from galaxy.managers.model_stores import ModelStoreManager from galaxy.metadata.set_metadata import set_metadata_portable from galaxy.model.scoped_session import galaxy_scoped_session from galaxy.schema.tasks import ( + ComputeDatasetHashTaskRequest, GenerateHistoryContentDownload, GenerateHistoryDownload, GenerateInvocationDownload, @@ -344,6 +348,14 @@ def import_model_store( model_store_manager.import_model_store(request) +@galaxy_task(action="compute dataset hash and store in database") +def compute_dataset_hash( + dataset_manager: DatasetManager, + request: ComputeDatasetHashTaskRequest, +): + dataset_manager.compute_hash(request) + + @galaxy_task(action="pruning history audit table") def prune_history_audit_table(sa_session: galaxy_scoped_session): """Prune ever growing history_audit table.""" diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index bda25a092f3..e013f61040e 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -206,14 +206,6 @@ class Data(metaclass=DataMeta): return cls.allow_datatype_change return cls.composite_type is None - def get_raw_data(self, dataset): - """Returns the full data. To stream it open the file_name and read/write as needed""" - try: - return open(dataset.file_name, "rb").read(-1) - except OSError: - log.exception("%s reading a file that does not exist %s", self.__class__.__name__, dataset.file_name) - return "" - def dataset_content_needs_grooming(self, file_name): """This function is called on an output dataset file after the content is initially generated.""" return False diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 7c7e84c6fc9..7f4036a76e0 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -25,7 +25,9 @@ from galaxy.managers import ( secured, users, ) +from galaxy.schema.tasks import ComputeDatasetHashTaskRequest from galaxy.structured_app import MinimalManagerApp +from galaxy.util.hash_util import memory_bound_hexdigest log = logging.getLogger(__name__) @@ -110,6 +112,48 @@ class DatasetManager(base.ModelManager[model.Dataset], secured.AccessibleManager roles = user.all_roles_exploiting_cache() if user else [] return self.app.security_agent.can_access_dataset(roles, dataset) + def compute_hash(self, request: ComputeDatasetHashTaskRequest): + # For files in extra_files_path + dataset = self.by_id(request.dataset_id) + extra_files_path = request.extra_files_path + if extra_files_path: + extra_dir = dataset.extra_files_path_name + file_path = self.app.object_store.get_filename(dataset, extra_dir=extra_dir, alt_name=extra_files_path) + else: + file_path = dataset.file_name + hash_function = request.hash_function + calculated_hash_value = memory_bound_hexdigest(hash_func_name=hash_function, path=file_path) + extra_files_path = request.extra_files_path + dataset_hash = model.DatasetHash( + hash_function=hash_function.value, + hash_value=calculated_hash_value, + extra_files_path=extra_files_path, + ) + dataset_hash.dataset = dataset + # TODO: replace/update if the combination of dataset_id/hash_function has already + # been stored. + sa_session = self.session() + hash = ( + sa_session.query(model.DatasetHash) + .filter( + model.DatasetHash.dataset_id == dataset.id, + model.DatasetHash.hash_function == hash_function, + model.DatasetHash.extra_files_path == extra_files_path, + ) + .one_or_none() + ) + if hash is None: + sa_session.add(dataset_hash) + sa_session.flush() + else: + old_hash_value = hash.hash_value + if old_hash_value != calculated_hash_value: + log.warning( + f"Re-calculated dataset hash for dataset [{dataset.id}] and new hash value [{calculated_hash_value}] does not equal previous hash value [{old_hash_value}]." + ) + else: + log.debug("Duplicated dataset hash request, no update to the database.") + # TODO: implement above for groups # TODO: datatypes? # .... data, object_store diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index c5783115e1b..f40a85bbd94 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -40,7 +40,7 @@ from galaxy.structured_app import ( BasicSharedApp, MinimalManagerApp, ) -from galaxy.util.hash_util import new_secure_hash +from galaxy.util.hash_util import new_secure_hash_v2 from galaxy.web import url_for log = logging.getLogger(__name__) @@ -197,8 +197,8 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): # to identify if it is needed for some reason. # # Deleting multiple times will re-hash the username/email - email_hash = new_secure_hash(user.email + pseudorandom_value) - uname_hash = new_secure_hash(user.username + pseudorandom_value) + email_hash = new_secure_hash_v2(user.email + pseudorandom_value) + uname_hash = new_secure_hash_v2(user.username + pseudorandom_value) # We must also redact username for role in user.all_roles(): if self.app.config.redact_username_during_deletion: @@ -219,15 +219,15 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): .all() ) for addr in user_addresses: - addr.desc = new_secure_hash(addr.desc + pseudorandom_value) - addr.name = new_secure_hash(addr.name + pseudorandom_value) - addr.institution = new_secure_hash(addr.institution + pseudorandom_value) - addr.address = new_secure_hash(addr.address + pseudorandom_value) - addr.city = new_secure_hash(addr.city + pseudorandom_value) - addr.state = new_secure_hash(addr.state + pseudorandom_value) - addr.postal_code = new_secure_hash(addr.postal_code + pseudorandom_value) - addr.country = new_secure_hash(addr.country + pseudorandom_value) - addr.phone = new_secure_hash(addr.phone + pseudorandom_value) + addr.desc = new_secure_hash_v2(addr.desc + pseudorandom_value) + addr.name = new_secure_hash_v2(addr.name + pseudorandom_value) + addr.institution = new_secure_hash_v2(addr.institution + pseudorandom_value) + addr.address = new_secure_hash_v2(addr.address + pseudorandom_value) + addr.city = new_secure_hash_v2(addr.city + pseudorandom_value) + addr.state = new_secure_hash_v2(addr.state + pseudorandom_value) + addr.postal_code = new_secure_hash_v2(addr.postal_code + pseudorandom_value) + addr.country = new_secure_hash_v2(addr.country + pseudorandom_value) + addr.phone = new_secure_hash_v2(addr.phone + pseudorandom_value) self.session().add(addr) # Purge the user super().purge(user, flush=flush) @@ -563,7 +563,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): user = trans.sa_session.query(self.app.model.User).filter(self.app.model.User.table.c.email == email).first() activation_token = user.activation_token if activation_token is None: - activation_token = util.hash_util.new_secure_hash(str(random.getrandbits(256))) + activation_token = util.hash_util.new_secure_hash_v2(str(random.getrandbits(256))) user.activation_token = activation_token trans.sa_session.add(user) trans.sa_session.flush() diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e1c3edcc4ad..3b39e27169e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -146,7 +146,7 @@ from galaxy.util.form_builder import ( WorkflowField, WorkflowMappingField, ) -from galaxy.util.hash_util import new_secure_hash +from galaxy.util.hash_util import new_insecure_hash from galaxy.util.json import safe_loads from galaxy.util.sanitize_html import sanitize_html @@ -638,7 +638,7 @@ class User(Base, Dictifiable, RepresentById): if User.use_pbkdf2: self.password = galaxy.security.passwords.hash_password(cleartext) else: - self.password = new_secure_hash(text_type=cleartext) + self.password = new_insecure_hash(text_type=cleartext) self.last_password_change = now() def set_random_password(self, length=16): @@ -4063,10 +4063,6 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): def hashes(self): return self.dataset.hashes - def get_raw_data(self): - """Returns the full data. To stream it open the file_name and read/write as needed""" - return self.datatype.get_raw_data(self) - def get_mime(self): """Returns the mime type of the data""" try: diff --git a/lib/galaxy/schema/tasks.py b/lib/galaxy/schema/tasks.py index 9e6116806e9..95211aa652b 100644 --- a/lib/galaxy/schema/tasks.py +++ b/lib/galaxy/schema/tasks.py @@ -5,6 +5,7 @@ from pydantic import ( Field, ) +from galaxy.util.hash_util import HashFunctionNameEnum from .schema import ( DatasetSourceType, HistoryContentType, @@ -101,3 +102,10 @@ class MaterializeDatasetInstanceTaskRequest(BaseModel): "- The encoded id of the the HDA\n" ), ) + + +class ComputeDatasetHashTaskRequest(BaseModel): + dataset_id: int + extra_files_path: Optional[str] + hash_function: HashFunctionNameEnum + user: RequestUser diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index c321ecfb220..abddec1ffb2 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -431,7 +431,7 @@ class CachedDependencyManager(DependencyManager): (dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies ] hash_str = json.dumps(sorted(resolved_dependencies)) - return hash_util.new_secure_hash(hash_str)[:8] # short hash + return hash_util.new_insecure_hash(hash_str)[:8] # short hash def get_hashed_dependencies_path(self, resolved_dependencies): """ diff --git a/lib/galaxy/util/hash_util.py b/lib/galaxy/util/hash_util.py index 427a8eccf81..53fe69a8aa0 100644 --- a/lib/galaxy/util/hash_util.py +++ b/lib/galaxy/util/hash_util.py @@ -6,6 +6,16 @@ introduced hashlib which replaced sha in Python 2.4 and previous versions. import hashlib import hmac import logging +import os +from enum import Enum +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Union, +) from . import smart_str @@ -13,22 +23,39 @@ log = logging.getLogger(__name__) BLOCK_SIZE = 1024 * 1024 +HashFunctionT = Callable[[], "hashlib._Hash"] + sha1 = hashlib.sha1 sha256 = hashlib.sha256 sha512 = hashlib.sha512 sha = sha1 md5 = hashlib.md5 -HASH_NAME_MAP = { - "MD5": md5, - "SHA-1": sha1, - "SHA-256": sha256, - "SHA-512": sha512, + +class HashFunctionNameEnum(str, Enum): + """Particular pieces of information that can be requested for a dataset.""" + + md5 = "MD5" + sha1 = "SHA-1" + sha256 = "SHA-256" + sha512 = "SHA-512" + + +HASH_NAME_MAP: Dict[HashFunctionNameEnum, HashFunctionT] = { + HashFunctionNameEnum.md5: md5, + HashFunctionNameEnum.sha1: sha1, + HashFunctionNameEnum.sha256: sha256, + HashFunctionNameEnum.sha512: sha512, } -HASH_NAMES = list(HASH_NAME_MAP.keys()) +HASH_NAMES: List[HashFunctionNameEnum] = list(HASH_NAME_MAP.keys()) -def memory_bound_hexdigest(hash_func=None, hash_func_name=None, path=None, file=None): +def memory_bound_hexdigest( + hash_func: Optional[HashFunctionT] = None, + hash_func_name: Optional[HashFunctionNameEnum] = None, + path: Optional[str] = None, + file=None, +): if hash_func is None: assert hash_func_name is not None hash_func = HASH_NAME_MAP[hash_func_name] @@ -48,7 +75,7 @@ def memory_bound_hexdigest(hash_func=None, hash_func_name=None, path=None, file= file.close() -def md5_hash_file(path): +def md5_hash_file(path: Union[str, os.PathLike]) -> Optional[str]: """ Return a md5 hashdigest for a file or None if path could not be read. """ @@ -63,19 +90,35 @@ def md5_hash_file(path): return None -def new_secure_hash(text_type): +def new_secure_hash_v2(text_type: Union[bytes, str]) -> str: + """More modern version of new_secure_hash. + + Certain passwords are set via new_insecure_hash (previously new_secure_hash), + so that needs to remain for legacy purposes. """ - Returns the hexdigest of the sha1 hash of the argument `text_type`. + assert text_type is not None + return sha512(smart_str(text_type)).hexdigest() + + +def new_insecure_hash(text_type: Union[bytes, str]) -> str: + """Returns the hexdigest of the sha1 hash of the argument `text_type`. + + Previously called new_secure_hash, but this should not be considered + secure - SHA1 is no longer considered a secure hash and has been broken + since the early 2000s. + + use_pbkdf2 should be set by default and galaxy.security.passwords should + be the default used for passwords in Galaxy. """ assert text_type is not None return sha1(smart_str(text_type)).hexdigest() -def hmac_new(key, value): +def hmac_new(key: Union[bytes, str], value: Union[bytes, str]) -> str: return hmac.new(smart_str(key), smart_str(value), sha).hexdigest() -def is_hashable(value): +def is_hashable(value: Any) -> bool: try: hash(value) except Exception: @@ -83,4 +126,4 @@ def is_hashable(value): return True -__all__ = ("md5", "hashlib", "sha1", "sha", "new_secure_hash", "hmac_new", "is_hashable") +__all__ = ("md5", "hashlib", "sha1", "sha", "new_insecure_hash", "new_secure_hash_v2", "hmac_new", "is_hashable") diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index e402c6096eb..47b1a49ada5 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -34,6 +34,7 @@ from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import ( AnyHDA, AnyHistoryContentItem, + AsyncTaskResultSummary, DatasetAssociationRoles, DatasetSourceType, UpdateDatasetPermissionsPayload, @@ -51,6 +52,7 @@ from galaxy.webapps.galaxy.api.common import ( query_serialization_params, ) from galaxy.webapps.galaxy.services.datasets import ( + ComputeDatasetHashPayload, ConvertedDatasetsMap, DatasetInheritanceChain, DatasetsService, @@ -338,3 +340,16 @@ class FastAPIDatasets: no other checks or restrictions are made. """ return self.service.delete_batch(trans, payload) + + @router.put( + "/api/datasets/{dataset_id}/hash", + summary="Compute dataset hash for dataset and update model", + ) + def compute_hash( + self, + trans=DependsOnTrans, + dataset_id: DecodedDatabaseIdField = DatasetIDPathParam, + hda_ldda: DatasetSourceType = DatasetSourceQueryParam, + payload: ComputeDatasetHashPayload = Body(...), + ) -> AsyncTaskResultSummary: + return self.service.compute_hash(trans, dataset_id, payload, hda_ldda=hda_ldda) diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 4fdb5fd0541..28276b8ac80 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -20,6 +20,7 @@ from galaxy import ( util, web, ) +from galaxy.celery.tasks import compute_dataset_hash from galaxy.datatypes import dataproviders from galaxy.managers.base import ModelSerializer from galaxy.managers.context import ProvidesHistoryContext @@ -43,14 +44,17 @@ from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import ( AnyHDA, AnyHistoryContentItem, + AsyncTaskResultSummary, DatasetAssociationRoles, DatasetSourceId, DatasetSourceType, Model, UpdateDatasetPermissionsPayload, ) +from galaxy.schema.tasks import ComputeDatasetHashTaskRequest from galaxy.schema.types import RelativeUrl from galaxy.security.idencoding import IdEncodingHelper +from galaxy.util.hash_util import HashFunctionNameEnum from galaxy.util.path import safe_walk from galaxy.visualization.data_providers.genome import ( BamDataProvider, @@ -59,7 +63,10 @@ from galaxy.visualization.data_providers.genome import ( ) from galaxy.visualization.data_providers.registry import DataProviderRegistry from galaxy.webapps.base.controller import UsesVisualizationMixin -from galaxy.webapps.galaxy.services.base import ServiceBase +from galaxy.webapps.galaxy.services.base import ( + async_task_summary, + ServiceBase, +) log = logging.getLogger(__name__) @@ -185,6 +192,16 @@ class DeleteDatasetBatchPayload(Model): ) +class ComputeDatasetHashPayload(Model): + hash_function: Optional[HashFunctionNameEnum] = Field( + default=HashFunctionNameEnum.md5, description="Hash function name to use to compute dataset hashes." + ) + extra_files_path: Optional[str] = Field(default=None, description="If set, extra files path to compute a hash for.") + + class Config: + use_enum_values = True # When using .dict() + + class DatasetErrorMessage(Model): dataset: DatasetSourceId = Field( description="The encoded ID of the dataset and its source.", @@ -374,6 +391,23 @@ class DatasetsService(ServiceBase, UsesVisualizationMixin): return DatasetInheritanceChain(__root__=result) + def compute_hash( + self, + trans: ProvidesHistoryContext, + dataset_id: DecodedDatabaseIdField, + payload: ComputeDatasetHashPayload, + hda_ldda: DatasetSourceType = DatasetSourceType.hda, + ) -> AsyncTaskResultSummary: + dataset_instance = self.dataset_manager_by_type[hda_ldda].get_accessible(dataset_id, trans.user) + request = ComputeDatasetHashTaskRequest( + dataset_id=dataset_instance.dataset.id, + extra_files_path=payload.extra_files_path, + hash_function=payload.hash_function, + user=trans.async_request_user, + ) + result = compute_dataset_hash.delay(request=request) + return async_task_summary(result) + def update_permissions( self, trans: ProvidesHistoryContext, diff --git a/lib/galaxy_test/api/test_datasets.py b/lib/galaxy_test/api/test_datasets.py index b4ea953f42d..bb45495e1c2 100644 --- a/lib/galaxy_test/api/test_datasets.py +++ b/lib/galaxy_test/api/test_datasets.py @@ -21,6 +21,18 @@ from galaxy_test.base.populators import ( ) from ._framework import ApiTestCase +COMPOSITE_DATA_FETCH_REQUEST_1 = { + "src": "composite", + "ext": "velvet", + "composite": { + "items": [ + {"src": "pasted", "paste_content": "sequences content"}, + {"src": "pasted", "paste_content": "roadmaps content"}, + {"src": "pasted", "paste_content": "log content"}, + ] + }, +} + class DatasetsApiTestCase(ApiTestCase): history_id: str @@ -481,25 +493,96 @@ class DatasetsApiTestCase(ApiTestCase): @skip_without_datatype("velvet") def test_composite_datatype_download(self): - item = { - "src": "composite", - "ext": "velvet", - "composite": { - "items": [ - {"src": "pasted", "paste_content": "sequences content"}, - {"src": "pasted", "paste_content": "roadmaps content"}, - {"src": "pasted", "paste_content": "log content"}, - ] - }, - } - output = self.dataset_populator.fetch_hda(self.history_id, item, wait=True) - print(output) + output = self.dataset_populator.fetch_hda(self.history_id, COMPOSITE_DATA_FETCH_REQUEST_1, wait=True) response = self._get(f"histories/{self.history_id}/contents/{output['id']}/display?to_ext=zip") self._assert_status_code_is(response, 200) archive = zipfile.ZipFile(BytesIO(response.content)) namelist = archive.namelist() assert len(namelist) == 4, f"Expected 3 elements in [{namelist}]" + def test_compute_md5_on_primary_dataset(self): + hda = self.dataset_populator.new_dataset(self.history_id, wait=True) + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=hda) + assert "hashes" in hda_details, str(hda_details.keys()) + hashes = hda_details["hashes"] + assert len(hashes) == 0 + + self.dataset_populator.compute_hash(hda["id"]) + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=hda) + self.assert_hash_value(hda_details, "940cbe15c94d7e339dc15550f6bdcf4d", "MD5") + + def test_compute_sha1_on_composite_dataset(self): + output = self.dataset_populator.fetch_hda(self.history_id, COMPOSITE_DATA_FETCH_REQUEST_1, wait=True) + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=output) + assert "hashes" in hda_details, str(hda_details.keys()) + hashes = hda_details["hashes"] + assert len(hashes) == 0 + + self.dataset_populator.compute_hash(hda_details["id"], hash_function="SHA-256", extra_files_path="Roadmaps") + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=output) + self.assert_hash_value( + hda_details, + "3cbd311889963528954fe03b28b68a09685ea7a75660bd2268d5b44cafbe0d22", + "SHA-256", + extra_files_path="Roadmaps", + ) + + def test_duplicated_hash_requests_on_primary(self): + hda = self.dataset_populator.new_dataset(self.history_id, wait=True) + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=hda) + assert "hashes" in hda_details, str(hda_details.keys()) + hashes = hda_details["hashes"] + assert len(hashes) == 0 + + self.dataset_populator.compute_hash(hda["id"]) + self.dataset_populator.compute_hash(hda["id"]) + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=hda) + self.assert_hash_value(hda_details, "940cbe15c94d7e339dc15550f6bdcf4d", "MD5") + + def test_duplicated_hash_requests_on_extra_files(self): + output = self.dataset_populator.fetch_hda(self.history_id, COMPOSITE_DATA_FETCH_REQUEST_1, wait=True) + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=output) + assert "hashes" in hda_details, str(hda_details.keys()) + hashes = hda_details["hashes"] + assert len(hashes) == 0 + + # 4 unique requests, but make them twice... + for _ in range(2): + self.dataset_populator.compute_hash(hda_details["id"], hash_function="SHA-256", extra_files_path="Roadmaps") + self.dataset_populator.compute_hash(hda_details["id"], hash_function="SHA-1", extra_files_path="Roadmaps") + self.dataset_populator.compute_hash(hda_details["id"], hash_function="MD5", extra_files_path="Roadmaps") + self.dataset_populator.compute_hash( + hda_details["id"], hash_function="SHA-256", extra_files_path="Sequences" + ) + + hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=output) + self.assert_hash_value(hda_details, "ce0c0ef1073317ff96c896c249b002dc", "MD5", extra_files_path="Roadmaps") + self.assert_hash_value( + hda_details, "fe2e06cdd03922a1ddf3fe6c7e0d299c8044fc8e", "SHA-1", extra_files_path="Roadmaps" + ) + self.assert_hash_value( + hda_details, + "3cbd311889963528954fe03b28b68a09685ea7a75660bd2268d5b44cafbe0d22", + "SHA-256", + extra_files_path="Roadmaps", + ) + self.assert_hash_value( + hda_details, + "4688dca47fe3214516c35acd284a79d97bd6df2bc1c55981b556d995495b91b6", + "SHA-256", + extra_files_path="Sequences", + ) + + def assert_hash_value(self, dataset_details, expected_hash_value, hash_function, extra_files_path=None): + assert "hashes" in dataset_details, str(dataset_details.keys()) + hashes = dataset_details["hashes"] + matching_hashes = [ + h for h in hashes if h["extra_files_path"] == extra_files_path and h["hash_function"] == hash_function + ] + assert len(matching_hashes) == 1 + hash_value = matching_hashes[0]["hash_value"] + assert expected_hash_value == hash_value + def test_storage_show(self): hda = self.dataset_populator.new_dataset(self.history_id, wait=True) hda_details = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=hda) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 586f3c0d1cb..558fe6cc88a 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -640,6 +640,24 @@ class BaseDatasetPopulator(BasePopulator): def get_job_details(self, job_id: str, full: bool = False) -> Response: return self._get(f"jobs/{job_id}", {"full": full}) + def compute_hash( + self, + dataset_id: str, + hash_function: Optional[str] = "MD5", + extra_files_path: Optional[str] = None, + wait: bool = True, + ) -> Response: + data: Dict[str, Any] = dict() + if hash_function: + data["hash_function"] = hash_function + if extra_files_path: + data["extra_files_path"] = extra_files_path + put_response = self._put(f"datasets/{dataset_id}/hash", data, json=True) + api_asserts.assert_status_code_is_ok(put_response) + if wait: + self.wait_on_task(put_response) + return put_response + def cancel_history_jobs(self, history_id: str, wait=True) -> None: active_jobs = self.active_history_jobs(history_id) for active_job in active_jobs: diff --git a/lib/tool_shed/util/admin_util.py b/lib/tool_shed/util/admin_util.py index 3e8eca90341..184b6bc393e 100644 --- a/lib/tool_shed/util/admin_util.py +++ b/lib/tool_shed/util/admin_util.py @@ -13,7 +13,7 @@ from galaxy import ( ) from galaxy.security.validate_user_input import validate_password from galaxy.util import inflector -from galaxy.util.hash_util import new_secure_hash +from galaxy.util.hash_util import new_secure_hash_v2 from galaxy.web.form_builder import CheckboxField from galaxy.web.legacy_framework.grids import ( Grid, @@ -773,8 +773,8 @@ class Admin: compliance_log.info(f"delete-user-event: {user_id}") # See lib/galaxy/webapps/tool_shed/controllers/admin.py pseudorandom_value = str(int(time.time())) - email_hash = new_secure_hash(user.email + pseudorandom_value) - uname_hash = new_secure_hash(user.username + pseudorandom_value) + email_hash = new_secure_hash_v2(user.email + pseudorandom_value) + uname_hash = new_secure_hash_v2(user.username + pseudorandom_value) for role in user.all_roles(): print( role, self.app.config.redact_username_during_deletion, self.app.config.redact_email_during_deletion diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index e0989af283c..810665b404b 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -48,7 +48,7 @@ from galaxy.security.validate_user_input import validate_password_str from galaxy.util import unique_id from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable -from galaxy.util.hash_util import new_secure_hash +from galaxy.util.hash_util import new_insecure_hash from tool_shed.dependencies.repository import relation_builder from tool_shed.util import ( hg_util, @@ -152,7 +152,7 @@ class User(Base, Dictifiable, _HasTable): def check_password(self, cleartext): """Check if 'cleartext' matches 'self.password' when hashed.""" - return self.password == new_secure_hash(text_type=cleartext) + return self.password == new_insecure_hash(text_type=cleartext) def get_disk_usage(self, nice_size=False): return 0 @@ -171,7 +171,7 @@ class User(Base, Dictifiable, _HasTable): if message: raise Exception(f"Invalid password: {message}") # Set 'self.password' to the digest of 'cleartext'. - self.password = new_secure_hash(text_type=cleartext) + self.password = new_insecure_hash(text_type=cleartext) class PasswordResetToken(Base, _HasTable):