From 400922a5a46a69e2a9f828a91b70749b82caca0f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 28 Feb 2022 11:33:37 +0100 Subject: [PATCH] Drop pkg_resources --- lib/galaxy/eggs/__init__.py | 8 -------- lib/galaxy/exceptions/error_codes.py | 6 ++---- .../jobs/runners/util/job_script/__init__.py | 9 ++++----- lib/galaxy/managers/licenses.py | 4 ++-- lib/galaxy/managers/markdown_util.py | 4 ++-- lib/galaxy/selenium/data.py | 4 ++-- lib/galaxy/tools/__init__.py | 4 ++-- lib/galaxy/util/resources.py | 14 ++++++++++++++ lib/galaxy/util/rules_dsl.py | 5 +++-- lib/galaxy/webapps/__init__.py | 3 ++- lib/galaxy_test/base/populators.py | 9 ++++----- packages/util/requirements.txt | 1 + scripts/get_platforms.py | 12 ------------ 13 files changed, 38 insertions(+), 45 deletions(-) delete mode 100644 lib/galaxy/eggs/__init__.py create mode 100644 lib/galaxy/util/resources.py delete mode 100755 scripts/get_platforms.py diff --git a/lib/galaxy/eggs/__init__.py b/lib/galaxy/eggs/__init__.py deleted file mode 100644 index 95e6a1707da..00000000000 --- a/lib/galaxy/eggs/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -For backwards compatibility -""" - -import pkg_resources - - -require = pkg_resources.require diff --git a/lib/galaxy/exceptions/error_codes.py b/lib/galaxy/exceptions/error_codes.py index f4415686411..3dfd43cf79d 100644 --- a/lib/galaxy/exceptions/error_codes.py +++ b/lib/galaxy/exceptions/error_codes.py @@ -5,9 +5,7 @@ See the file error_codes.json for actual error code descriptions. from json import loads from typing import Dict -from pkg_resources import resource_string - -from galaxy.util import unicodify +from galaxy.util.resources import resource_string # Error codes are provided as a convience to Galaxy API clients, but at this @@ -45,7 +43,7 @@ def _from_dict(entry): return (name, ErrorCode(code, message)) -error_codes_json = unicodify(resource_string(__name__, 'error_codes.json')) +error_codes_json = resource_string(__name__.rsplit('.', 1)[0], 'error_codes.json') error_codes_by_name: Dict[str, ErrorCode] = {} for entry in loads(error_codes_json): diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 61f13eb7b49..90d34b69e1f 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -5,26 +5,25 @@ import time from string import Template from typing import Any, Dict -from pkg_resources import resource_string - from galaxy.job_execution.setup import JobIO from galaxy.util import ( RWXR_XR_X, unicodify, ) +from galaxy.util.resources import resource_string log = logging.getLogger(__name__) DEFAULT_SHELL = '/bin/bash' DEFAULT_JOB_FILE_TEMPLATE = Template( - unicodify(resource_string(__name__, 'DEFAULT_JOB_FILE_TEMPLATE.sh')) + resource_string(__name__, 'DEFAULT_JOB_FILE_TEMPLATE.sh') ) SLOTS_STATEMENT_CLUSTER_DEFAULT = \ - unicodify(resource_string(__name__, 'CLUSTER_SLOTS_STATEMENT.sh')) + resource_string(__name__, 'CLUSTER_SLOTS_STATEMENT.sh') MEMORY_STATEMENT_DEFAULT = \ - unicodify(resource_string(__name__, 'MEMORY_STATEMENT.sh')) + resource_string(__name__, 'MEMORY_STATEMENT.sh') SLOTS_STATEMENT_SINGLE = """ GALAXY_SLOTS="1" diff --git a/lib/galaxy/managers/licenses.py b/lib/galaxy/managers/licenses.py index 8af0d02e39d..589c893e139 100644 --- a/lib/galaxy/managers/licenses.py +++ b/lib/galaxy/managers/licenses.py @@ -2,7 +2,6 @@ import json import logging from typing import List -from pkg_resources import resource_string from pydantic import ( BaseModel, Field, @@ -10,6 +9,7 @@ from pydantic import ( ) from galaxy import exceptions +from galaxy.util.resources import resource_string log = logging.getLogger(__name__) @@ -85,7 +85,7 @@ RECOMMENDED_LICENSES = [ "MPL-2.0", "PDDL-1.0", ] -SPDX_LICENSES_STRING = resource_string(__name__, 'licenses.json').decode("UTF-8") +SPDX_LICENSES_STRING = resource_string(__name__.rsplit('.', 1)[0], 'licenses.json') SPDX_LICENSES = json.loads(SPDX_LICENSES_STRING) for license in SPDX_LICENSES["licenses"]: license["recommended"] = license["licenseId"] in RECOMMENDED_LICENSES diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 26787fc452d..11f9b451a49 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -27,7 +27,6 @@ from typing import ( ) import markdown -import pkg_resources try: import weasyprint except Exception: @@ -47,6 +46,7 @@ from galaxy.managers.jobs import ( from galaxy.model.item_attrs import get_item_annotation_str from galaxy.model.orm.now import now from galaxy.schema import PdfDocumentType +from galaxy.util.resources import resource_string from galaxy.util.sanitize_html import sanitize_html from .markdown_parse import GALAXY_MARKDOWN_FUNCTION_CALL_LINE, validate_galaxy_markdown @@ -596,7 +596,7 @@ def to_pdf_raw(basic_markdown: str, css_paths: Optional[List[str]] = None) -> by output_file.write(as_html) output_file.close() html = weasyprint.HTML(filename=index) - stylesheets = [weasyprint.CSS(string=pkg_resources.resource_string(__name__, 'markdown_export_base.css'))] + stylesheets = [weasyprint.CSS(string=resource_string(resource_string, 'markdown_export_base.css'))] for css_path in css_paths: with open(css_path) as f: css_content = f.read() diff --git a/lib/galaxy/selenium/data.py b/lib/galaxy/selenium/data.py index b59d6836a6c..ad6b3727ea8 100644 --- a/lib/galaxy/selenium/data.py +++ b/lib/galaxy/selenium/data.py @@ -1,10 +1,10 @@ import yaml -from pkg_resources import resource_string +from galaxy.util.resources import resource_string from .components import Component def load_root_component() -> Component: - new_data_yaml = resource_string(__name__, 'navigation.yml').decode("UTF-8") + new_data_yaml = resource_string(__name__.rsplit('.', 1)[0], 'navigation.yml') navigation_raw = yaml.safe_load(new_data_yaml) return Component.from_dict("root", navigation_raw) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 39a80968b1d..67db1b9b404 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -30,7 +30,6 @@ import packaging.version import webob.exc from lxml import etree from mako.template import Template -from pkg_resources import resource_string from webob.compat import cgi_FieldStorage from galaxy import ( @@ -112,6 +111,7 @@ from galaxy.util.dictifiable import Dictifiable from galaxy.util.expressions import ExpressionContext from galaxy.util.form_builder import SelectField from galaxy.util.json import safe_loads +from galaxy.util.resources import resource_string from galaxy.util.rules_dsl import RuleSet from galaxy.util.template import ( fill_template, @@ -210,7 +210,7 @@ GALAXY_LIB_TOOLS_VERSIONED = { "winSplitter": packaging.version.parse("1.0.1"), } -BIOTOOLS_MAPPING_CONTENT = resource_string(__name__, 'biotools_mappings.tsv').decode("UTF-8") +BIOTOOLS_MAPPING_CONTENT = resource_string(__name__, 'biotools_mappings.tsv') BIOTOOLS_MAPPING: Dict[str, str] = dict([cast(Tuple[str, str], tuple(x.split("\t"))) for x in BIOTOOLS_MAPPING_CONTENT.splitlines() if not x.startswith("#")]) REQUIRE_FULL_DIRECTORY = { diff --git a/lib/galaxy/util/resources.py b/lib/galaxy/util/resources.py new file mode 100644 index 00000000000..bcd7e2565db --- /dev/null +++ b/lib/galaxy/util/resources.py @@ -0,0 +1,14 @@ +try: + from importlib.resources import files # type: ignore[attr-defined] +except ImportError: + # Python < 3.9 + from importlib_resources import files # type: ignore[no-redef] + + +def resource_string(package_or_requirement, resource_name): + """ + Return specified resource as a string. + + Replacement function for pkg_resources.resource_string, but returns unicode string instead of bytestring. + """ + return files(package_or_requirement).joinpath(resource_name).read_text() diff --git a/lib/galaxy/util/rules_dsl.py b/lib/galaxy/util/rules_dsl.py index 47a94a759da..cbe95652f9c 100644 --- a/lib/galaxy/util/rules_dsl.py +++ b/lib/galaxy/util/rules_dsl.py @@ -4,11 +4,12 @@ import re from typing import List, Type import yaml -from pkg_resources import resource_stream + +from galaxy.util.resources import resource_string def get_rules_specification(): - return yaml.safe_load(resource_stream(__name__, 'rules_dsl_spec.yml')) + return yaml.safe_load(resource_string(__name__.rsplit('.', 1)[0], 'rules_dsl_spec.yml')) def _ensure_rule_contains_keys(rule, keys): diff --git a/lib/galaxy/webapps/__init__.py b/lib/galaxy/webapps/__init__.py index 988de1719ee..d0e1dc365cc 100644 --- a/lib/galaxy/webapps/__init__.py +++ b/lib/galaxy/webapps/__init__.py @@ -1,3 +1,4 @@ """Galaxy webapps root package -- this is a namespace package.""" -__import__("pkg_resources").declare_namespace(__name__) +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) # type: ignore[has-type] diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 9183483c2c5..81f3ab2ef05 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -69,7 +69,6 @@ from gxformat2 import ( ImporterGalaxyInterface, ) from gxformat2._yaml import ordered_load -from pkg_resources import resource_string from requests.models import Response from galaxy.tool_util.client.staging import InteractorStaging @@ -83,8 +82,8 @@ from galaxy.tool_util.verify.wait import ( from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, galaxy_root_path, - unicodify, ) +from galaxy.util.resources import resource_string from . import api_asserts from .api import ApiTestInteractor @@ -92,10 +91,10 @@ from .api import ApiTestInteractor CWL_TOOL_DIRECTORY = os.path.join(galaxy_root_path, "test", "functional", "tools", "cwl_tools") # Simple workflow that takes an input and call cat wrapper on it. -workflow_str = unicodify(resource_string(__name__, "data/test_workflow_1.ga")) +workflow_str = resource_string(__name__.rsplit('.', 1)[0], "data/test_workflow_1.ga") # Simple workflow that takes an input and filters with random lines twice in a # row - first grabbing 8 lines at random and then 6. -workflow_random_x2_str = unicodify(resource_string(__name__, "data/test_workflow_2.ga")) +workflow_random_x2_str = resource_string(__name__.rsplit('.', 1)[0], "data/test_workflow_2.ga") DEFAULT_TIMEOUT = 60 # Secs to wait for state to turn ok @@ -1124,7 +1123,7 @@ class BaseWorkflowPopulator(BasePopulator): def load_workflow_from_resource(self, name: str, filename: Optional[str] = None) -> dict: if filename is None: filename = f"data/{name}.ga" - content = unicodify(resource_string(__name__, filename)) + content = resource_string(__name__.rsplit('.', 1)[0], filename) return self.load_workflow(name, content=content) def simple_workflow(self, name: str, **create_kwds) -> str: diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index ffffd7b7ee3..bdd17557364 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -1,6 +1,7 @@ bleach boltons docutils +importlib_resources markupsafe packaging pycryptodome diff --git a/scripts/get_platforms.py b/scripts/get_platforms.py deleted file mode 100755 index a4d3733798f..00000000000 --- a/scripts/get_platforms.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python - -import os -import sys - -assert sys.version_info[:2] >= (2, 6) - -lib = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")) -sys.path.insert(1, lib) - -import pkg_resources -print(pkg_resources.get_platform())