From aca9490d899a26a535c395a4cd8003e967567b44 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 10 Dec 2019 23:21:29 +0000 Subject: [PATCH 1/2] Explicitly include PyPI as an ``--extra-index-url`` to pip Backport combining commits 244fc834cd4dd3980cc3edb755d22ca5a9372421 and 09fdac513d59ca6dad440616a53daa97751ff905 . This is now needed because wheels.galaxyproject.org is not a full-fledged pypiserver any more but just an S3 bucket served via a CDN. --- scripts/common_startup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 71bb77ae7c9..033e41d2c32 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -130,17 +130,18 @@ if [ $SET_VENV -eq 1 ]; then fi : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"} +: ${PYPI_INDEX_URL:="https://pypi.python.org/simple"} if [ $REPLACE_PIP -eq 1 ]; then pip install 'pip>=8.1' fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install -r requirements.txt --index-url ${GALAXY_WHEELS_INDEX_URL} + pip install -r requirements.txt --index-url ${GALAXY_WHEELS_INDEX_URL} --extra-index-url "${PYPI_INDEX_URL}" GALAXY_CONDITIONAL_DEPENDENCIES=`PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))"` [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url ${GALAXY_WHEELS_INDEX_URL} fi if [ $FETCH_WHEELS -eq 1 -a $DEV_WHEELS -eq 1 ]; then dev_requirements='./lib/galaxy/dependencies/dev-requirements.txt' - [ -f $dev_requirements ] && pip install -r $dev_requirements --index-url ${GALAXY_WHEELS_INDEX_URL} + [ -f $dev_requirements ] && pip install -r $dev_requirements --index-url ${GALAXY_WHEELS_INDEX_URL} --extra-index-url "${PYPI_INDEX_URL}" fi From 0429c4d515536f9cca6b70b2abeb019de807c955 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 11 Dec 2019 01:12:53 +0000 Subject: [PATCH 2/2] Fix importing a directory from user_library_import_dir Fix the following traceback: ``` Traceback (most recent call last): File "/galaxy/lib/galaxy/web/framework/decorators.py", line 282, in decorator rval = func(self, trans, *args, **kwargs) File "/galaxy/lib/galaxy/webapps/galaxy/api/library_contents.py", line 250, in create status, output = self._upload_library_dataset(trans, library_id, real_folder_id, **payload) File "/galaxy/lib/galaxy/webapps/galaxy/api/library_contents.py", line 337, in _upload_library_dataset **kwd) File "/galaxy/lib/galaxy/actions/library.py", line 91, in _upload_dataset full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir) File "/galaxy/lib/galaxy/actions/library.py", line 48, in validate_server_directory_upload if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, whitelist=trans.app.config.user_library_import_symlink_whitelist, username=username): TypeError: safe_contains() got an unexpected keyword argument 'username' ``` Also add a test case to reproduce the fixed issue, which was introduced in commit f4442ffc807285a96fa049f9a7e961b5e945bbff . Reported by @jhl667 . --- lib/galaxy/actions/library.py | 4 +-- .../test_upload_configuration_options.py | 32 ++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/actions/library.py b/lib/galaxy/actions/library.py index a9c5170819d..6d0723c4d23 100644 --- a/lib/galaxy/actions/library.py +++ b/lib/galaxy/actions/library.py @@ -45,8 +45,8 @@ def validate_server_directory_upload(trans, server_dir): unsafe = None if safe_relpath(server_dir): username = trans.user.username if trans.app.config.user_library_import_check_permissions else None - if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, whitelist=trans.app.config.user_library_import_symlink_whitelist, username=username): - for unsafe in unsafe_walk(full_dir, whitelist=[import_dir] + trans.app.config.user_library_import_symlink_whitelist): + if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, whitelist=trans.app.config.user_library_import_symlink_whitelist): + for unsafe in unsafe_walk(full_dir, whitelist=[import_dir] + trans.app.config.user_library_import_symlink_whitelist, username=username): log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe)) else: log.error('User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s', server_dir, os.path.realpath(full_dir)) diff --git a/test/integration/test_upload_configuration_options.py b/test/integration/test_upload_configuration_options.py index dd076d4e060..88058195693 100644 --- a/test/integration/test_upload_configuration_options.py +++ b/test/integration/test_upload_configuration_options.py @@ -705,7 +705,7 @@ class ServerDirectoryValidUsageTestCase(BaseUploadContentConfigurationTestCase): return cls.temp_config_dir("server") -class ServerDirectoryRestrictedToAdminsUsageTestCase(BaseUploadContentConfigurationTestCase): +class UserServerDirectoryOffByDefaultTestCase(BaseUploadContentConfigurationTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): @@ -719,6 +719,36 @@ class ServerDirectoryRestrictedToAdminsUsageTestCase(BaseUploadContentConfigurat assert response.status_code == 403, response.json() +class UserServerDirectoryValidUsageTestCase(BaseUploadContentConfigurationTestCase): + + @classmethod + def user_server_dir(cls): + return cls.temp_config_dir("user_library_import_dir") + + @classmethod + def handle_galaxy_config_kwds(cls, config): + user_server_dir = cls.user_server_dir() + os.makedirs(user_server_dir) + config["user_library_import_dir"] = os.path.join(user_server_dir) + + def test_valid_user_server_dir_uploads_okay(self): + dir_to_import = 'library' + full_dir_path = os.path.join(self.user_server_dir(), TEST_USER, dir_to_import) + os.makedirs(full_dir_path) + file_content = "hello world\n" + with tempfile.NamedTemporaryFile(mode='w', dir=full_dir_path, delete=False) as fh: + fh.write(file_content) + file_to_import = fh.name + + library_dataset = self.library_populator.new_library_dataset("serverdirupload", upload_option="upload_directory", server_dir=dir_to_import) + # Check the file is still there and was not modified + with open(file_to_import, 'r') as fh: + read_content = fh.read() + assert read_content == file_content + + assert library_dataset["file_size"] == 12, library_dataset + + class FetchByPathTestCase(BaseUploadContentConfigurationTestCase): require_admin_user = True