mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Check GitHub API rate limit in `get_alternative_versions`, skip test if hit
Fix the following mulled unit test random failure:
```
@external_dependency_management
def test_get_alternative_versions():
> versions = get_alternative_versions("recipes/bioblend", "meta.yaml")
test/unit/tool_util/mulled/test_get_tests.py:79:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
lib/galaxy/tool_util/deps/mulled/get_tests.py:178: in get_alternative_versions
r.raise_for_status()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <Response [403]>
f"{self.status_code} Server Error: {reason} for url: {self.url}"
)
if http_error_msg:
> raise HTTPError(http_error_msg, response=self)
E requests.exceptions.HTTPError: 403 Client Error: rate limit exceeded for url: https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/bioblend
.tox/mulled/lib/python3.7/site-packages/requests/models.py:1021: HTTPError
------------------------------ Captured log call -------------------------------
DEBUG urllib3.connectionpool:connectionpool.py:1007 Starting new HTTPS connection (1): api.github.com:443
DEBUG urllib3.connectionpool:connectionpool.py:465 https://api.github.com:443 "GET /repos/bioconda/bioconda-recipes/contents/recipes/bioblend HTTP/1.1" 403 277
```
This commit is contained in:
@@ -20,6 +20,7 @@ except ImportError:
|
||||
Template = None # type: ignore[assignment,misc]
|
||||
UndefinedError = Exception # type: ignore[assignment,misc]
|
||||
|
||||
from galaxy.util import check_github_api_response_rate_limit
|
||||
from .util import (
|
||||
get_file_from_recipe_url,
|
||||
MULLED_SOCKET_TIMEOUT,
|
||||
@@ -175,6 +176,7 @@ def get_alternative_versions(filepath, filename, recipes_path=None, github_repo=
|
||||
# else use the GitHub API:
|
||||
versions = []
|
||||
r = requests.get(f"https://api.github.com/repos/{github_repo}/contents/{filepath}", timeout=MULLED_SOCKET_TIMEOUT)
|
||||
check_github_api_response_rate_limit(r)
|
||||
r.raise_for_status()
|
||||
for subfile in json.loads(r.text):
|
||||
if subfile["type"] == "dir":
|
||||
|
||||
@@ -5,15 +5,14 @@ import json
|
||||
import logging
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import (
|
||||
datetime,
|
||||
timezone,
|
||||
)
|
||||
|
||||
import requests
|
||||
|
||||
from galaxy.tool_util.deps.conda_util import CondaContext
|
||||
from galaxy.util import which
|
||||
from galaxy.util import (
|
||||
check_github_api_response_rate_limit,
|
||||
which,
|
||||
)
|
||||
from .mulled_list import get_singularity_containers
|
||||
from .util import (
|
||||
build_target,
|
||||
@@ -147,17 +146,6 @@ class GitHubSearch:
|
||||
Tool to search the GitHub bioconda-recipes repo
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _check_response_rate_limit(response):
|
||||
if response.status_code == 403 and "API rate limit exceeded" in response.json()["message"]:
|
||||
# It can take tens of minutes before the rate limit window resets
|
||||
message = "GitHub API rate limit exceeded."
|
||||
rate_limit_reset_UTC_timestamp = response.headers.get("X-RateLimit-Reset")
|
||||
if rate_limit_reset_UTC_timestamp:
|
||||
rate_limit_reset_datetime = datetime.fromtimestamp(int(rate_limit_reset_UTC_timestamp), tz=timezone.utc)
|
||||
message += f" The rate limit window will reset at {rate_limit_reset_datetime.isoformat()}."
|
||||
raise Exception(message)
|
||||
|
||||
def get_json(self, search_string):
|
||||
"""
|
||||
Takes search_string variable and return results from the bioconda-recipes github repository in JSON format
|
||||
@@ -169,7 +157,7 @@ class GitHubSearch:
|
||||
f"https://api.github.com/search/code?q={search_string}+in:path+repo:bioconda/bioconda-recipes+path:recipes",
|
||||
timeout=MULLED_SOCKET_TIMEOUT,
|
||||
)
|
||||
self._check_response_rate_limit(response)
|
||||
check_github_api_response_rate_limit(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
@@ -188,7 +176,7 @@ class GitHubSearch:
|
||||
f"https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/{search_string}",
|
||||
timeout=MULLED_SOCKET_TIMEOUT,
|
||||
)
|
||||
self._check_response_rate_limit(response)
|
||||
check_github_api_response_rate_limit(response)
|
||||
return response.status_code == 200
|
||||
|
||||
|
||||
|
||||
@@ -25,7 +25,10 @@ import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import xml.dom.minidom
|
||||
from datetime import datetime
|
||||
from datetime import (
|
||||
datetime,
|
||||
timezone,
|
||||
)
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from hashlib import md5
|
||||
@@ -1806,6 +1809,17 @@ def is_url(uri, allow_list=None):
|
||||
return any(uri.startswith(scheme) for scheme in allow_list)
|
||||
|
||||
|
||||
def check_github_api_response_rate_limit(response):
|
||||
if response.status_code == 403 and "API rate limit exceeded" in response.json()["message"]:
|
||||
# It can take tens of minutes before the rate limit window resets
|
||||
message = "GitHub API rate limit exceeded."
|
||||
rate_limit_reset_UTC_timestamp = response.headers.get("X-RateLimit-Reset")
|
||||
if rate_limit_reset_UTC_timestamp:
|
||||
rate_limit_reset_datetime = datetime.fromtimestamp(int(rate_limit_reset_UTC_timestamp), tz=timezone.utc)
|
||||
message += f" The rate limit window will reset at {rate_limit_reset_datetime.isoformat()}."
|
||||
raise Exception(message)
|
||||
|
||||
|
||||
def download_to_file(url, dest_file_path, timeout=30, chunk_size=2**20):
|
||||
"""Download a URL to a file in chunks."""
|
||||
with requests.get(url, timeout=timeout, stream=True) as r, open(dest_file_path, "wb") as f:
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from unittest import SkipTest
|
||||
|
||||
from galaxy.tool_util.deps.mulled.get_tests import (
|
||||
deep_test_search,
|
||||
find_anaconda_versions,
|
||||
@@ -76,7 +78,12 @@ def test_open_recipe_file():
|
||||
|
||||
@external_dependency_management
|
||||
def test_get_alternative_versions():
|
||||
versions = get_alternative_versions("recipes/bioblend", "meta.yaml")
|
||||
try:
|
||||
versions = get_alternative_versions("recipes/bioblend", "meta.yaml")
|
||||
except Exception as e:
|
||||
if "API rate limit" in str(e):
|
||||
raise SkipTest("Hitting GitHub API rate limit")
|
||||
raise
|
||||
assert versions == ["recipes/bioblend/0.7.0/meta.yaml"]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user