opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/galaxyproject/galaxy into realtimetools

Browse Source
This commit is contained in:
Daniel Blankenberg
2019-06-06 17:39:45 -04:00
parent 7b4c1d7675 169e78df7c
commit ccc4fc482f
90 changed files with 2739 additions and 995 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bootstrap_history.py
+84 -27
View File
@@ -38,7 +38,7 @@ DEVTEAM = [
"davebx", "martenson", "jmchilton",
"tnabtaf", "natefoo", "jgoecks",
"guerler", "jennaj", "nekrut", "jxtx",
"VJalili"
"VJalili", "WilliamHolden", "Nerdinacan"
]
TEMPLATE = string.Template("""
@@ -143,15 +143,21 @@ Highlights
Feature description.
New Visualisations
New Visualizations
===========================================================
.. visualizations
New Datatypes
===========================================================
.. datatypes
Builtin Tool Updates
===========================================================
.. tools
Release Notes
===========================================================
@@ -174,6 +180,9 @@ Schedule
* Planned Release Date: ${release_date}
""")
PRS_TEMPLATE = """
.. github_links
"""
RELEASE_ISSUE_TEMPLATE = string.Template("""
@@ -337,7 +346,7 @@ def do_release(argv):
release=release_name
)
announce_file = _release_file(release_name + "_announce.rst")
open(announce_file, "w").write(announce_info.encode("utf-8"))
_write_file(announce_file, announce_info)
announce_user_info = ANNOUNCE_USER_TEMPLATE.substitute(
month_name=month_name,
@@ -345,7 +354,10 @@ def do_release(argv):
release=release_name
)
announce_user_file = _release_file(release_name + "_announce_user.rst")
open(announce_user_file, "w").write(announce_user_info.encode("utf-8"))
_write_file(announce_user_file, announce_user_info)
prs_file = _release_file(release_name + "_prs.rst")
_write_file(prs_file, PRS_TEMPLATE)
next_version_params = _next_version_params(release_name)
next_version = next_version_params["version"]
@@ -354,10 +366,9 @@ def do_release(argv):
next_announce = NEXT_TEMPLATE.substitute(**next_version_params)
open(next_release_file, "w").write(next_announce.encode("utf-8"))
releases_index = _release_file("index.rst")
releases_index_contents = open(releases_index, "r").read()
releases_index_contents = _read_file(releases_index)
releases_index_contents = releases_index_contents.replace(".. announcements\n", ".. announcements\n " + next_version + "_announce\n")
with open(releases_index, "w") as f:
f.write(releases_index_contents)
_write_file(releases_index, releases_index_contents)
for pr in _get_prs(release_name):
# 2015-06-29 18:32:13 2015-04-22 19:11:53 2015-08-12 21:15:45
@@ -365,15 +376,16 @@ def do_release(argv):
"title": pr.title,
"number": pr.number,
"head": pr.head,
"labels": _pr_to_labels(pr),
}
main([argv[0], "--release_file", "%s_prs.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)])
main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)])
def check_release(argv):
github = _github_client()
release_name = argv[2]
for pr in _get_prs(release_name):
_text_target(github, pr)
_text_target(github, pr, labels=_pr_to_labels(pr))
def check_blocking_prs(argv):
@@ -451,8 +463,16 @@ def _get_prs(release_name, state="closed"):
user=PROJECT_OWNER,
repo=PROJECT_NAME,
)
reached_old_prs = False
for page in pull_requests:
if reached_old_prs:
break
for pr in page:
if pr.created_at < datetime.datetime(2016, 11, 1, 0, 0):
reached_old_prs = True
pass
merged_at = pr.merged_at
milestone = pr.milestone
proper_state = state != "closed" or merged_at
@@ -504,10 +524,15 @@ def main(argv):
if newest_release is None:
newest_release = sorted(os.listdir(RELEASES_PATH))[-1]
history_path = os.path.join(RELEASES_PATH, newest_release)
history = open(history_path, "r").read().decode("utf-8")
user_announce_path = history_path[0:-len(".rst")] + "_announce_user.rst"
prs_path = history_path[0:-len(".rst")] + "_prs.rst"
def extend(from_str, line, source=history):
from_str += "\n"
history = _read_file(history_path)
user_announce = _read_file(user_announce_path)
prs_content = _read_file(prs_path)
def extend_target(target, line, source=history):
from_str = ".. %s\n" % target
return source.replace(from_str, from_str + line + "\n")
ident = argv[1]
@@ -548,48 +573,72 @@ def main(argv):
if owner in DEVTEAM:
owner = None
text = ".. _Pull Request {0}: {1}/pull/{0}".format(pull_request, PROJECT_URL)
history = extend(".. github_links", text)
prs_content = extend_target("github_links", text, prs_content)
if owner:
to_doc += "\n(thanks to `@%s <https://github.com/%s>`__)." % (
owner, owner,
)
to_doc += "\n`Pull Request {0}`_".format(pull_request)
if github:
text_target = _text_target(github, pull_request)
labels = None
if req and 'labels' in req:
labels = req['labels']
text_target = _text_target(github, pull_request, labels=labels)
elif ident.startswith("issue"):
issue = ident[len("issue"):]
text = ".. _Issue {0}: {1}/issues/{0}".format(issue, PROJECT_URL)
history = extend(".. github_links", text)
prs_content = extend_target("github_links", text, prs_content)
to_doc += "`Issue {0}`_".format(issue)
else:
short_rev = ident[:7]
text = ".. _{0}: {1}/commit/{0}".format(short_rev, PROJECT_URL)
history = extend(".. github_links", text)
prs_content = extend_target("github_links", text, prs_content)
to_doc += "{0}_".format(short_rev)
to_doc = wrap(to_doc)
history = extend(".. %s\n" % text_target, to_doc, history)
open(history_path, "w").write(history.encode("utf-8"))
history = extend_target(text_target, to_doc, history)
if req and 'labels' in req:
labels = req['labels']
if 'area/datatypes' in labels:
user_announce = extend_target("datatypes", to_doc, user_announce)
if 'area/visualizations' in labels:
user_announce = extend_target("visualizations", to_doc, user_announce)
if 'area/tools' in labels:
user_announce = extend_target("tools", to_doc, user_announce)
_write_file(history_path, history)
_write_file(prs_path, prs_content)
_write_file(user_announce_path, user_announce)
def _text_target(github, pull_request):
labels = []
def _read_file(path):
with open(path, "r") as f:
return f.read().decode("utf-8")
def _write_file(path, contents):
with open(path, "w") as f:
f.write(contents.encode("utf-8"))
def _text_target(github, pull_request, labels=None):
pr_number = None
if isinstance(pull_request, string_types):
pr_number = pull_request
else:
pr_number = pull_request.number
try:
labels = github.issues.labels.list_by_issue(int(pr_number), user=PROJECT_OWNER, repo=PROJECT_NAME)
except Exception as e:
print(e)
if labels is None:
labels = []
try:
labels = github.issues.labels.list_by_issue(int(pr_number), user=PROJECT_OWNER, repo=PROJECT_NAME)
labels = [l.name.lower() for l in labels]
except Exception as e:
print(e)
is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False
if len(labels) == 0:
print('No labels found for %s' % pr_number)
return None
for label in labels:
label_name = label.name.lower()
for label_name in labels:
if label_name == "minor":
is_minor = True
elif label_name == "major":
@@ -604,11 +653,14 @@ def _text_target(github, pull_request):
is_enhancement = True
elif label_name in ["kind/testing", "kind/refactoring"]:
is_small_enhancement = True
elif label_name == "procedures":
# Treat procedures as an implicit enhancement.
is_enhancement = True
is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement
if not(is_bug or is_some_kind_of_enhancement or is_minor or is_merge):
print("No kind/ or minor or merge label found for %s" % _pr_to_str(pull_request))
print("No 'kind/*' or 'minor' or 'merge' or 'procedures' label found for %s" % _pr_to_str(pull_request))
text_target = None
if is_minor or is_merge:
@@ -632,6 +684,11 @@ def _text_target(github, pull_request):
return text_target
def _pr_to_labels(pr):
labels = [l["name"].lower() for l in pr.labels]
return labels
def _previous_release(to):
previous_release = None
for release in _releases():
scripts/tool_shed/build_ts_whoosh_index.py
+31 -32
View File
@@ -17,8 +17,8 @@ import os
import sys
from optparse import OptionParser
from mercurial import hg, ui
from six.moves import configparser
from whoosh.fields import Schema, STORED, TEXT
from whoosh.filedb.filestore import FileStorage
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
@@ -31,38 +31,17 @@ from galaxy.util import (
unicodify
)
from galaxy.webapps.tool_shed import config, model
from galaxy.webapps.tool_shed.search.repo_search import schema as repo_schema
from galaxy.webapps.tool_shed.search.tool_search import schema as tool_schema
from galaxy.webapps.tool_shed.util.hgweb_config import HgWebConfigManager
if sys.version_info > (3,):
long = int
logging.basicConfig(level='DEBUG')
repo_schema = Schema(
id=STORED,
name=TEXT(stored=True),
description=TEXT(stored=True),
long_description=TEXT(stored=True),
homepage_url=TEXT(stored=True),
remote_repository_url=TEXT(stored=True),
repo_owner_username=TEXT(stored=True),
times_downloaded=STORED,
approved=STORED,
last_updated=STORED,
full_last_updated=STORED)
tool_schema = Schema(
name=TEXT(stored=True),
description=TEXT(stored=True),
owner=TEXT(stored=True),
id=TEXT(stored=True),
help=TEXT(stored=True),
version=TEXT(stored=True),
repo_name=TEXT(stored=True),
repo_owner_username=TEXT(stored=True),
repo_id=STORED)
def build_index(sa_session, whoosh_index_dir, path_to_repositories):
def build_index(sa_session, whoosh_index_dir, path_to_repositories, hgweb_config_dir):
"""
Build the search indexes. One for repositories and another for tools within.
"""
@@ -85,7 +64,7 @@ def build_index(sa_session, whoosh_index_dir, path_to_repositories):
repos_indexed = 0
tools_indexed = 0
for repo in get_repos(sa_session, path_to_repositories):
for repo in get_repos(sa_session, path_to_repositories, hgweb_config_dir):
repo_index_writer.add_document(id=repo.get('id'),
name=unicodify(repo.get('name')),
@@ -94,10 +73,12 @@ def build_index(sa_session, whoosh_index_dir, path_to_repositories):
homepage_url=unicodify(repo.get('homepage_url')),
remote_repository_url=unicodify(repo.get('remote_repository_url')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
categories=unicodify(repo.get('categories')),
times_downloaded=repo.get('times_downloaded'),
approved=repo.get('approved'),
last_updated=repo.get('last_updated'),
full_last_updated=repo.get('full_last_updated'))
full_last_updated=repo.get('full_last_updated'),
repo_lineage=unicodify(repo.get('repo_lineage')))
# Tools get their own index
for tool in repo.get('tools_list'):
tool_index_writer.add_document(id=unicodify(tool.get('id')),
@@ -121,13 +102,19 @@ def build_index(sa_session, whoosh_index_dir, path_to_repositories):
print("TOTAL tools indexed: ", tools_indexed)
def get_repos(sa_session, path_to_repositories):
def get_repos(sa_session, path_to_repositories, hgweb_config_dir):
"""
Load repos from DB and included tools from .xml configs.
"""
hgwcm = HgWebConfigManager()
hgwcm.hgweb_config_dir = hgweb_config_dir
results = []
for repo in sa_session.query(model.Repository).filter_by(deleted=False).filter_by(deprecated=False).filter(model.Repository.type != 'tool_dependency_definition'):
category_names = []
for rca in sa_session.query(model.RepositoryCategoryAssociation).filter(model.RepositoryCategoryAssociation.repository_id == repo.id):
for category in sa_session.query(model.Category).filter(model.Category.id == rca.category.id):
category_names.append(category.name)
categories = (",").join(category_names)
repo_id = repo.id
name = repo.name
description = repo.description
@@ -154,6 +141,14 @@ def get_repos(sa_session, path_to_repositories):
last_updated = pretty_print_time_interval(repo.update_time)
full_last_updated = repo.update_time.strftime("%Y-%m-%d %I:%M %p")
# load all changesets of the repo
repo_path = hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name))
hg_repo = hg.repository(ui.ui(), repo_path)
lineage = []
for changeset in hg_repo.changelog:
lineage.append(str(changeset) + ":" + str(hg_repo.changectx(changeset)))
repo_lineage = str(lineage)
# Parse all the tools within repo for separate index.
tools_list = []
path = os.path.join(path_to_repositories, *directory_hash_id(repo.id))
@@ -178,7 +173,9 @@ def get_repos(sa_session, path_to_repositories):
approved=approved,
last_updated=last_updated,
full_last_updated=full_last_updated,
tools_list=tools_list))
tools_list=tools_list,
repo_lineage=repo_lineage,
categories=categories))
return results
@@ -218,9 +215,11 @@ def get_sa_session_and_needed_config_settings(path_to_tool_shed_config):
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("-c", "--config", dest="path_to_tool_shed_config", default="config/tool_shed.ini", help="specify tool_shed.ini location")
parser.add_option("-r", "--hgweb", dest="hgweb_config_dir", default=".", help="specify hgweb.config location")
(options, args) = parser.parse_args()
path_to_tool_shed_config = options.path_to_tool_shed_config
hgweb_config_dir = options.hgweb_config_dir
sa_session, config_settings = get_sa_session_and_needed_config_settings(path_to_tool_shed_config)
whoosh_index_dir = config_settings.get('whoosh_index_dir', None)
path_to_repositories = config_settings.get('file_path', 'database/community_files')
build_index(sa_session, whoosh_index_dir, path_to_repositories)
build_index(sa_session, whoosh_index_dir, path_to_repositories, hgweb_config_dir)