From 651221510a03b6704d3e218b1d1c4c0fb2d920bd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 13 Jul 2022 18:11:03 +0300 Subject: [PATCH 1/5] Attempt to use epoch of client build time for cache busting Store webpack build hash in meta.json file, and use that to setup the cache busting. This should prevent clients from unnecessarily reloading client bundles when the server restarts and/or when individual workers finish booting at staggered intervals. This became more important with usegalaxy.org not forking off workers from a preloaded master (so all workers would have the same starttime). --- .gitignore | 1 + client/package.json | 1 + client/webpack.config.js | 9 +++++++++ client/yarn.lock | 7 +++++++ lib/galaxy/app.py | 3 ++- lib/galaxy/web/framework/base.py | 8 ++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f956b65bd5c..ce4ad4211eb 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ lib/galaxy.egg-info lib/galaxy/web/framework/static/maps lib/galaxy/web/framework/static/scripts lib/galaxy/web/framework/static/style +lib/galaxy/web/framework/meta.json # Database stuff /database/ diff --git a/client/package.json b/client/package.json index 42e72995779..d0b5651efcd 100644 --- a/client/package.json +++ b/client/package.json @@ -44,6 +44,7 @@ "date-fns-tz": "^1.3.3", "decode-uri-component": "^0.2.0", "dom-to-image": "^2.6.0", + "dumpmeta-webpack-plugin": "^0.2.0", "elkjs": "^0.7.1", "file-saver": "^2.0.5", "flush-promises": "^1.0.2", diff --git a/client/webpack.config.js b/client/webpack.config.js index fbc9b3e71b3..6293c7b479b 100644 --- a/client/webpack.config.js +++ b/client/webpack.config.js @@ -5,6 +5,7 @@ const VueLoaderPlugin = require("vue-loader/lib/plugin"); const MiniCssExtractPlugin = require("mini-css-extract-plugin"); const CssMinimizerPlugin = require("css-minimizer-webpack-plugin"); const DuplicatePackageCheckerPlugin = require("@cerner/duplicate-package-checker-webpack-plugin"); +const { DumpMetaPlugin } = require("dumpmeta-webpack-plugin"); const scriptsBase = path.join(__dirname, "src"); const testsBase = path.join(__dirname, "tests"); @@ -200,6 +201,14 @@ module.exports = (env = {}, argv = {}) => { filename: "[name].css", }), new DuplicatePackageCheckerPlugin(), + new DumpMetaPlugin({ + filename: path.join(__dirname, "../lib/galaxy/web/framework/meta.json"), + prepare: (stats) => ({ + // add any other information you need to dump + hash: stats.hash, + epoch: Date.now(), + }), + }), ], devServer: { client: { diff --git a/client/yarn.lock b/client/yarn.lock index c6deda3c339..58d7357ddf7 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -3855,6 +3855,13 @@ dot-case@^3.0.4: no-case "^3.0.4" tslib "^2.0.3" +dumpmeta-webpack-plugin@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/dumpmeta-webpack-plugin/-/dumpmeta-webpack-plugin-0.2.0.tgz#c3a9da7d163ad3bb1d0373fb2c0bf5bb9b626b8f" + integrity sha512-9oxLTPvYVtk7HOmOq2rGQlvajCNiXtIKLInIrdNpdgzeoMMELaAsRsRBZfm6xf0JuwyBfytdJxsmtM6luTeqzA== + dependencies: + webpack ">=4.0.0 <6.0.0" + duplexify@^3.6.0: version "3.7.1" resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309" diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 67defe9bf31..ac2c29920ad 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -123,6 +123,7 @@ from galaxy.web import ( legacy_url_for, url_for, ) +from galaxy.web.framework.base import server_starttime from galaxy.web.proxy import ProxyManager from galaxy.web.short_term_storage import ( ShortTermStorageAllocator, @@ -739,7 +740,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.url_for = url_for self.legacy_url_for = legacy_url_for - self.server_starttime = int(time.time()) # used for cachebusting + self.server_starttime = server_starttime # used for cachebusting # Limit lifetime of tool shed repository cache to app startup self.tool_shed_repository_cache = None self.api_spec = None diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index fb2a5d69b13..5cbcb008490 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -2,6 +2,7 @@ A simple WSGI application/framework. """ import io +import json import logging import os import socket @@ -25,11 +26,17 @@ import webob.exc as httpexceptions from paste.response import HeaderDict from galaxy.util import smart_str +from galaxy.util.resources import resource_string log = logging.getLogger(__name__) #: time of the most recent server startup server_starttime = int(time.time()) +try: + meta_json = json.loads(resource_string(__package__, "meta.json")) + server_starttime = meta_json.get("epoch") or server_starttime +except Exception: + meta_json = {} def __resource_with_deleted(self, member_name, collection_name, **kwargs): @@ -567,6 +574,7 @@ __all__ = ( "httpexceptions", "lazy_property", "routes", + "server_starttime", "walk_controller_modules", "WebApplication", ) From 057b877f1c71773d97924ebaac3c898045e8166e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 13 Jul 2022 15:19:53 -0400 Subject: [PATCH 2/5] Use same Date object for internal and external timestamps (was probably close enough, but...) --- client/webpack.config.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/webpack.config.js b/client/webpack.config.js index 6293c7b479b..b44940bfbb7 100644 --- a/client/webpack.config.js +++ b/client/webpack.config.js @@ -25,6 +25,8 @@ const modulesExcludedFromLibs = [ "citeproc", ].join("|"); +const buildDate = new Date(); + module.exports = (env = {}, argv = {}) => { // environment name based on -d, -p, webpack flag const targetEnv = process.env.NODE_ENV == "production" || argv.mode == "production" ? "production" : "development"; @@ -194,7 +196,7 @@ module.exports = (env = {}, argv = {}) => { }), new webpack.DefinePlugin({ __targetEnv__: JSON.stringify(targetEnv), - __buildTimestamp__: JSON.stringify(new Date().toISOString()), + __buildTimestamp__: JSON.stringify(buildDate.toISOString()), }), new VueLoaderPlugin(), new MiniCssExtractPlugin({ @@ -206,7 +208,7 @@ module.exports = (env = {}, argv = {}) => { prepare: (stats) => ({ // add any other information you need to dump hash: stats.hash, - epoch: Date.now(), + epoch: Date.parse(buildDate), }), }), ], From 7ab11efea97433d601221868df61d539feb4c42f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 15 Jul 2022 09:49:54 +0300 Subject: [PATCH 3/5] Upgrade FastAPI to 0.79.0 Fixes `RuntimeError: Response content longer than Content-Length` --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- lib/galaxy/webapps/openapi/utils.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index f69aa8e6be8..fe71e420fed 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -62,7 +62,7 @@ docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or ( ecdsa==0.17.0; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" edam-ontology==1.25.2 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0" -fastapi==0.78.0; python_full_version >= "3.6.1" +fastapi==0.79.0; python_full_version >= "3.6.1" fluent-logger==0.10.0; python_version >= "3.5" fonttools==4.33.3; python_version >= "3.7" fs==2.4.16 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 7236e2878d8..b6aaf1c25cf 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -52,7 +52,7 @@ docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or ( ecdsa==0.17.0; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" edam-ontology==1.25.2 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0" -fastapi==0.78.0; python_full_version >= "3.6.1" +fastapi==0.79.0; python_full_version >= "3.6.1" fs==2.4.16 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") diff --git a/lib/galaxy/webapps/openapi/utils.py b/lib/galaxy/webapps/openapi/utils.py index 6100b87ee31..b2abb4ee4ad 100644 --- a/lib/galaxy/webapps/openapi/utils.py +++ b/lib/galaxy/webapps/openapi/utils.py @@ -32,7 +32,6 @@ from fastapi.encoders import jsonable_encoder from fastapi.openapi.constants import ( METHODS_WITH_BODY, REF_PREFIX, - STATUS_CODES_WITH_NO_BODY, ) from fastapi.openapi.models import OpenAPI from fastapi.params import ( @@ -44,6 +43,7 @@ from fastapi.utils import ( deep_dict_update, generate_operation_id_for_path, get_model_definitions, + is_body_allowed_for_status_code, ) from pydantic import BaseModel from pydantic.fields import ( @@ -266,7 +266,7 @@ def get_openapi_path( operation.setdefault("responses", {}).setdefault(status_code, {})[ "description" ] = route.response_description - if route_response_media_type and route.status_code not in STATUS_CODES_WITH_NO_BODY: + if route_response_media_type and is_body_allowed_for_status_code(route.status_code): response_schema = {"type": "string"} if lenient_issubclass(current_response_class, JSONResponse): if route.response_field: From 0f417def333a11253f21a5a55289e5301b551e18 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 15 Jul 2022 10:26:47 +0300 Subject: [PATCH 4/5] Avoid FileNotFoundError when Dataset is purged, but HDA is not Fixes https://sentry.galaxyproject.org/share/issue/dadaa8d6b70042a29e9a946ffdd2b5af/: ``` FileNotFoundError: [Errno 2] No such file or directory: '' File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/recursive.py", line 85, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/middleware/statsd.py", line 29, in __call__ req = self.application(environ, start_response) File "/cvmfs/main.galaxyproject.org/venv/lib/python3.8/site-packages/paste/httpexceptions.py", line 640, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/base.py", line 166, in __call__ return self.handle_request(environ, start_response) File "galaxy/web/framework/base.py", line 251, in handle_request body = method(trans, **kwargs) File "galaxy/webapps/galaxy/controllers/dataset.py", line 590, in display_by_username_and_slug first_chunk = dataset.datatype.get_chunk(trans, dataset, 0) File "galaxy/datatypes/tabular.py", line 107, in get_chunk with compression_utils.get_fileobj(dataset.file_name) as f: File "galaxy/util/compression_utils.py", line 66, in get_fileobj return get_fileobj_raw(filename, mode, compressed_formats)[1] File "galaxy/util/compression_utils.py", line 126, in get_fileobj_raw return compressed_format, open(filename, mode, encoding="utf-8") ``` --- lib/galaxy/webapps/galaxy/controllers/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 1f3d712b199..bd5e0b93cc7 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -151,7 +151,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE raise web.httpexceptions.HTTPNotFound(f"Invalid reference dataset id: {str(hda_id)}.") if not self._can_access_dataset(trans, data): return trans.show_error_message("You are not allowed to access this dataset") - if data.purged: + if data.purged or data.dataset.purged: return trans.show_error_message("The dataset you are attempting to view has been purged.") elif data.deleted and not (trans.user_is_admin or (data.history and trans.get_user() == data.history.user)): return trans.show_error_message("The dataset you are attempting to view has been deleted.") From edd05bab4b301f30c09d2cbf036f5c5d876bdd18 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 15 Jul 2022 12:43:32 +0100 Subject: [PATCH 5/5] Update yarn.lock file Some changes were missing in commit 651221510a03b6704d3e218b1d1c4c0fb2d920bd . --- client/yarn.lock | 53 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/client/yarn.lock b/client/yarn.lock index 58d7357ddf7..2cdbe924650 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -1661,6 +1661,14 @@ "@types/eslint" "*" "@types/estree" "*" +"@types/eslint-scope@^3.7.3": + version "3.7.4" + resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.4.tgz#37fc1223f0786c39627068a12e94d6e6fc61de16" + integrity sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA== + dependencies: + "@types/eslint" "*" + "@types/estree" "*" + "@types/eslint@*": version "8.4.1" resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-8.4.1.tgz#c48251553e8759db9e656de3efc846954ac32304" @@ -1674,6 +1682,11 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.50.tgz#1e0caa9364d3fccd2931c3ed96fdbeaa5d4cca83" integrity sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw== +"@types/estree@^0.0.51": + version "0.0.51" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40" + integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== + "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18": version "4.17.28" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz#c47def9f34ec81dc6328d0b1b5303d1ec98d86b8" @@ -3968,6 +3981,14 @@ enhanced-resolve@^5.8.3: graceful-fs "^4.2.4" tapable "^2.2.0" +enhanced-resolve@^5.9.3: + version "5.10.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz#0dc579c3bb2a1032e357ac45b8f3a6f3ad4fb1e6" + integrity sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ== + dependencies: + graceful-fs "^4.2.4" + tapable "^2.2.0" + ent@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" @@ -6337,7 +6358,7 @@ json-parse-better-errors@^1.0.2: resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw== -json-parse-even-better-errors@^2.3.0: +json-parse-even-better-errors@^2.3.0, json-parse-even-better-errors@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== @@ -10033,6 +10054,36 @@ webpack-sources@^3.2.3: resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== +"webpack@>=4.0.0 <6.0.0": + version "5.73.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.73.0.tgz#bbd17738f8a53ee5760ea2f59dce7f3431d35d38" + integrity sha512-svjudQRPPa0YiOYa2lM/Gacw0r6PvxptHj4FuEKQ2kX05ZLkjbVc5MnPs6its5j7IZljnIqSVo/OsY2X0IpHGA== + dependencies: + "@types/eslint-scope" "^3.7.3" + "@types/estree" "^0.0.51" + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/wasm-edit" "1.11.1" + "@webassemblyjs/wasm-parser" "1.11.1" + acorn "^8.4.1" + acorn-import-assertions "^1.7.6" + browserslist "^4.14.5" + chrome-trace-event "^1.0.2" + enhanced-resolve "^5.9.3" + es-module-lexer "^0.9.0" + eslint-scope "5.1.1" + events "^3.2.0" + glob-to-regexp "^0.4.1" + graceful-fs "^4.2.9" + json-parse-even-better-errors "^2.3.1" + loader-runner "^4.2.0" + mime-types "^2.1.27" + neo-async "^2.6.2" + schema-utils "^3.1.0" + tapable "^2.1.1" + terser-webpack-plugin "^5.1.3" + watchpack "^2.3.1" + webpack-sources "^3.2.3" + webpack@^5.68.0: version "5.68.0" resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.68.0.tgz#a653a58ed44280062e47257f260117e4be90d560"