Merge pull request #13664 from nsoranzo/release_21.05_backport_dbf7ac364a20835acf16ccdffea3018a58023e2d

[21.05] Partially backport #13532 to fix `test_github_search` mulled unit test

lib/galaxy/tool_util/deps/mulled/mulled_search.py

@@ -5,6 +5,10 @@ import json
 import logging
 import sys
 import tempfile
+from datetime import (
+    datetime,
+    timezone,
+)
 
 import requests
 
@@ -140,13 +144,31 @@ class GitHubSearch():
     Tool to search the GitHub bioconda-recipes repo
     """
 
+    @staticmethod
+    def _check_response_rate_limit(response):
+        if response.status_code == 403 and "API rate limit exceeded" in response.json()["message"]:
+            # It can take tens of minutes before the rate limit window resets
+            message = "GitHub API rate limit exceeded."
+            rate_limit_reset_UTC_timestamp = response.headers.get("X-RateLimit-Reset")
+            if rate_limit_reset_UTC_timestamp:
+                rate_limit_reset_datetime = datetime.fromtimestamp(int(rate_limit_reset_UTC_timestamp), tz=timezone.utc)
+                message += f" The rate limit window will reset at {rate_limit_reset_datetime.isoformat()}."
+            raise Exception(message)
+
     def get_json(self, search_string):
         """
         Takes search_string variable and return results from the bioconda-recipes github repository in JSON format
+
+        DEPRECATED: this method is currently unreliable because the API query
+        sometimes succeeds but returns no items.
         """
         response = requests.get(
-            "https://api.github.com/search/code?q=%s+in:path+repo:bioconda/bioconda-recipes+path:recipes" % search_string, timeout=MULLED_SOCKET_TIMEOUT).json()
-        return response
+            f"https://api.github.com/search/code?q={search_string}+in:path+repo:bioconda/bioconda-recipes+path:recipes",
+            timeout=MULLED_SOCKET_TIMEOUT,
+        )
+        self._check_response_rate_limit(response)
+        response.raise_for_status()
+        return response.json()
 
     def process_json(self, json, search_string):
         """
@@ -164,10 +186,12 @@ class GitHubSearch():
         """
         Check if a recipe exists in bioconda-recipes which matches search_string exactly
         """
-        if requests.get("https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/%s" % search_string, timeout=MULLED_SOCKET_TIMEOUT).status_code == 200:
-            return True
-        else:
-            return False
+        response = requests.get(
+            f"https://api.github.com/repos/bioconda/bioconda-recipes/contents/recipes/{search_string}",
+            timeout=MULLED_SOCKET_TIMEOUT,
+        )
+        self._check_response_rate_limit(response)
+        return response.status_code == 200
 
 
 def get_package_hash(packages, versions):
@@ -338,10 +362,11 @@ def main(argv=None):
         github = GitHubSearch()
 
         for item in args.search:
-            github_json = github.get_json(item)
-            github_results[item] = github.process_json(github_json, item)
             if github.recipe_present(item):
                 github_recipe_present.append(item)
+            else:
+                github_json = github.get_json(item)
+                github_results[item] = github.process_json(github_json, item)
 
         json_results['github'] = github_results
         json_results['github_recipe_present'] = {

test/unit/tool_util/mulled/test_mulled_search.py

@@ -27,12 +27,21 @@ def test_conda_search():
 
 
 @external_dependency_management
-def test_github_search():
+def test_github_recipe_present():
     t = GitHubSearch()
-    search1 = t.process_json(t.get_json("adsfasdf"), "adsfasdf")
-    search2 = t.process_json(t.get_json("bioconductor-gosemsim"), "bioconductor-gosemsim")
-    assert search1 == []
-    assert {'path': 'recipes/bioconductor-gosemsim/meta.yaml', 'name': 'meta.yaml'} in search2
+
+    search_string2expected = {
+        "adsfasdf": False,
+        "bioconductor-gosemsim": True,
+    }
+    for search_string, expected in search_string2expected.items():
+        try:
+            is_recipe_present = t.recipe_present(search_string)
+        except Exception as e:
+            if "API rate limit" in str(e):
+                pytest.skip("Hitting GitHub API rate limit")
+            raise
+        assert is_recipe_present is expected
 
 
 @external_dependency_management