From a23cff23b29ccb742667459e4e1aa9e884b5e7e9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 4 Jan 2023 10:48:23 +0000 Subject: [PATCH] Check GitHub API rate limit in ``get_alternative_versions``, skip test if hit Fix the following mulled unit test random failure: ``` @external_dependency_management def test_get_alternative_versions(): > versions = get_alternative_versions("recipes/bioblend", "meta.yaml") test/unit/tool_util/mulled/test_get_tests.py:79: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ lib/galaxy/tool_util/deps/mulled/get_tests.py:178: in get_alternative_versions r.raise_for_status() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = f"{self.status_code} Server Error: {reason} for url: {self.url}" ) if http_error_msg: > raise HTTPError(http_error_msg, response=self) E requests.exceptions.HTTPError: 403 Client Error: rate limit exceeded for url: https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/bioblend .tox/mulled/lib/python3.7/site-packages/requests/models.py:1021: HTTPError ------------------------------ Captured log call ------------------------------- DEBUG urllib3.connectionpool:connectionpool.py:1007 Starting new HTTPS connection (1): api.github.com:443 DEBUG urllib3.connectionpool:connectionpool.py:465 https://api.github.com:443 "GET /repos/bioconda/bioconda-recipes/contents/recipes/bioblend HTTP/1.1" 403 277 ``` --- lib/galaxy/tool_util/deps/mulled/get_tests.py | 2 ++ .../tool_util/deps/mulled/mulled_search.py | 24 +++++-------------- lib/galaxy/util/__init__.py | 16 ++++++++++++- test/unit/tool_util/mulled/test_get_tests.py | 9 ++++++- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/get_tests.py b/lib/galaxy/tool_util/deps/mulled/get_tests.py index 3780739b6ce..e7ac94e9f90 100644 --- a/lib/galaxy/tool_util/deps/mulled/get_tests.py +++ b/lib/galaxy/tool_util/deps/mulled/get_tests.py @@ -20,6 +20,7 @@ except ImportError: Template = None # type: ignore[assignment,misc] UndefinedError = Exception # type: ignore[assignment,misc] +from galaxy.util import check_github_api_response_rate_limit from .util import ( get_file_from_recipe_url, MULLED_SOCKET_TIMEOUT, @@ -175,6 +176,7 @@ def get_alternative_versions(filepath, filename, recipes_path=None, github_repo= # else use the GitHub API: versions = [] r = requests.get(f"https://api.github.com/repos/{github_repo}/contents/{filepath}", timeout=MULLED_SOCKET_TIMEOUT) + check_github_api_response_rate_limit(r) r.raise_for_status() for subfile in json.loads(r.text): if subfile["type"] == "dir": diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_search.py b/lib/galaxy/tool_util/deps/mulled/mulled_search.py index 20700fbf29e..7555a36456a 100755 --- a/lib/galaxy/tool_util/deps/mulled/mulled_search.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_search.py @@ -5,15 +5,14 @@ import json import logging import sys import tempfile -from datetime import ( - datetime, - timezone, -) import requests from galaxy.tool_util.deps.conda_util import CondaContext -from galaxy.util import which +from galaxy.util import ( + check_github_api_response_rate_limit, + which, +) from .mulled_list import get_singularity_containers from .util import ( build_target, @@ -147,17 +146,6 @@ class GitHubSearch: Tool to search the GitHub bioconda-recipes repo """ - @staticmethod - def _check_response_rate_limit(response): - if response.status_code == 403 and "API rate limit exceeded" in response.json()["message"]: - # It can take tens of minutes before the rate limit window resets - message = "GitHub API rate limit exceeded." - rate_limit_reset_UTC_timestamp = response.headers.get("X-RateLimit-Reset") - if rate_limit_reset_UTC_timestamp: - rate_limit_reset_datetime = datetime.fromtimestamp(int(rate_limit_reset_UTC_timestamp), tz=timezone.utc) - message += f" The rate limit window will reset at {rate_limit_reset_datetime.isoformat()}." - raise Exception(message) - def get_json(self, search_string): """ Takes search_string variable and return results from the bioconda-recipes github repository in JSON format @@ -169,7 +157,7 @@ class GitHubSearch: f"https://api.github.com/search/code?q={search_string}+in:path+repo:bioconda/bioconda-recipes+path:recipes", timeout=MULLED_SOCKET_TIMEOUT, ) - self._check_response_rate_limit(response) + check_github_api_response_rate_limit(response) response.raise_for_status() return response.json() @@ -188,7 +176,7 @@ class GitHubSearch: f"https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/{search_string}", timeout=MULLED_SOCKET_TIMEOUT, ) - self._check_response_rate_limit(response) + check_github_api_response_rate_limit(response) return response.status_code == 200 diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index d72e90d924f..59e1ce1d22b 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -25,7 +25,10 @@ import threading import time import unicodedata import xml.dom.minidom -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from hashlib import md5 @@ -1806,6 +1809,17 @@ def is_url(uri, allow_list=None): return any(uri.startswith(scheme) for scheme in allow_list) +def check_github_api_response_rate_limit(response): + if response.status_code == 403 and "API rate limit exceeded" in response.json()["message"]: + # It can take tens of minutes before the rate limit window resets + message = "GitHub API rate limit exceeded." + rate_limit_reset_UTC_timestamp = response.headers.get("X-RateLimit-Reset") + if rate_limit_reset_UTC_timestamp: + rate_limit_reset_datetime = datetime.fromtimestamp(int(rate_limit_reset_UTC_timestamp), tz=timezone.utc) + message += f" The rate limit window will reset at {rate_limit_reset_datetime.isoformat()}." + raise Exception(message) + + def download_to_file(url, dest_file_path, timeout=30, chunk_size=2**20): """Download a URL to a file in chunks.""" with requests.get(url, timeout=timeout, stream=True) as r, open(dest_file_path, "wb") as f: diff --git a/test/unit/tool_util/mulled/test_get_tests.py b/test/unit/tool_util/mulled/test_get_tests.py index 36d82d7788a..582d481056b 100644 --- a/test/unit/tool_util/mulled/test_get_tests.py +++ b/test/unit/tool_util/mulled/test_get_tests.py @@ -1,3 +1,5 @@ +from unittest import SkipTest + from galaxy.tool_util.deps.mulled.get_tests import ( deep_test_search, find_anaconda_versions, @@ -76,7 +78,12 @@ def test_open_recipe_file(): @external_dependency_management def test_get_alternative_versions(): - versions = get_alternative_versions("recipes/bioblend", "meta.yaml") + try: + versions = get_alternative_versions("recipes/bioblend", "meta.yaml") + except Exception as e: + if "API rate limit" in str(e): + raise SkipTest("Hitting GitHub API rate limit") + raise assert versions == ["recipes/bioblend/0.7.0/meta.yaml"]