From 7758fe4b33fa2193e186d0a94f69fc05aec8758f Mon Sep 17 00:00:00 2001 From: youri Date: Wed, 8 Jul 2015 16:39:43 +0200 Subject: [PATCH 1/5] Fix for md5-check urls for download_by_url --- .../tool_dependencies/recipe/step_handler.py | 63 +++++++++---------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 4899327b865..f67463d4de2 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -150,30 +150,12 @@ class CompressedFile( object ): class Download( object ): - def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): - """ - The given download_url can have an extension like #md5# or #sha256#. - This indicates a checksum which will be chekced after download. - If the checksum does not match an exception is thrown. - - https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757 - """ - + def url_download( self, install_dir, downloaded_file_name, download_url, extract=True, checksums={} ): file_path = os.path.join( install_dir, downloaded_file_name ) src = None dst = None - checksum = None - sha256 = False - md5 = False # Set a timer so we don't sit here forever. - - if '#md5#' in download_url: - md5 = True - download_url, checksum = download_url.split('#md5#') - elif '#sha256#' in download_url: - sha256 = True - download_url, checksum = download_url.split('#sha256#') - + start_time = time.time() try: src = urllib2.urlopen( download_url ) @@ -197,18 +179,18 @@ class Download( object ): src.close() if dst: dst.close() - - try: - if sha256: - downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest() - elif md5: - downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() - - if checksum and downloaded_checksum != checksum: - raise Exception( 'Given checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) - except Exception, e: - raise - + + #try: + if checksums.has_key('sha256sum'): + downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest() + if downloaded_checksum != checksums['sha256sum']: + raise Exception( 'Given sha256 checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) + + if checksums.has_key('md5sum'): + downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() + if downloaded_checksum != checksums['md5sum']: + raise Exception( 'Given md5 checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) + if extract: if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): archive = CompressedFile( file_path ) @@ -660,7 +642,9 @@ class DownloadByUrl( Download, RecipeStep ): downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] - dir = self.url_download( work_dir, downloaded_filename, url, extract=True ) + + checksums = dict(filter(lambda i:i[0] in ['md5sum','sha256sum'], action_dict.iteritems())) + dir = self.url_download( work_dir, downloaded_filename, url, extract=True, checksums=checksums ) if is_binary: log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG ) if os.path.exists( log_file ): @@ -678,6 +662,19 @@ class DownloadByUrl( Download, RecipeStep ): # # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 # + # + # + # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 + # + + md5sum = action_elem.get( 'md5sum', None ) + sha256sum = action_elem.get( 'sha256sum', None ) + + if(md5sum): + action_dict['md5sum'] = md5sum + if(sha256sum): + action_dict['sha256sum'] = sha256sum + if is_binary_download: action_dict[ 'is_binary' ] = True if action_elem.text: From edd754d1ae998c7c6605d948d4adae742a66fe6a Mon Sep 17 00:00:00 2001 From: youri Date: Wed, 8 Jul 2015 17:19:12 +0200 Subject: [PATCH 2/5] Fix for md5-check urls for download_by_url --- .../tool_dependencies/recipe/step_handler.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index a2ed9940d63..e88a28d6a3c 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -150,15 +150,7 @@ class CompressedFile( object ): class Download( object ): - def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): - """ - The given download_url can have an extension like #md5# or #sha256#. - This indicates a checksum which will be checked after download. - If the checksum does not match an exception is thrown. - - https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757 - """ - + def url_download( self, install_dir, downloaded_file_name, download_url, extract=True, checksums={} ): file_path = os.path.join( install_dir, downloaded_file_name ) src = None dst = None From b369ce49b80dc5ef861a6934c7a66933fe8c809a Mon Sep 17 00:00:00 2001 From: Eric Rasche Date: Wed, 8 Jul 2015 10:33:21 -0500 Subject: [PATCH 3/5] Update docs, re-instate backwards compatability --- .../tool_dependencies/recipe/step_handler.py | 92 ++++++++++++------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index e88a28d6a3c..04aaba6b56d 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -149,13 +149,25 @@ class CompressedFile( object ): class Download( object ): - + def url_download( self, install_dir, downloaded_file_name, download_url, extract=True, checksums={} ): + """ + The given download_url can have an extension like #md5#, #sha256#, (or #md5= to support pypi defaults). + + https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757 + + Alternatively, to not break HTTP spec, you can specify md5 and + sha256 as keys in the element. + + This indicates a checksum which will be checked after download. + If the checksum does not match an exception is thrown. + """ file_path = os.path.join( install_dir, downloaded_file_name ) src = None dst = None + + # Set a timer so we don't sit here forever. - start_time = time.time() try: src = urllib2.urlopen( download_url ) @@ -179,18 +191,34 @@ class Download( object ): src.close() if dst: dst.close() - - #try: - if checksums.has_key('sha256sum'): + + if checksums.has_key('sha256sum') or '#sha256#' in download_url: downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest() - if downloaded_checksum != checksums['sha256sum']: - raise Exception( 'Given sha256 checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) - - if checksums.has_key('md5sum'): + + # Determine expected value + if checksums.has_key('sha256sum'): + expected = checksums['sha256sum'] + else: + expected = download_url.split('#sha256#')[1] + + if downloaded_checksum != expected: + raise Exception( 'Given sha256 checksum does not match with the one from the downloaded file (%s != %s).' % (downloaded_checksum, expected) ) + + if checksums.has_key('md5sum') or '#md5#' in download_url or '#md5=' in download_url: downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() - if downloaded_checksum != checksums['md5sum']: - raise Exception( 'Given md5 checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) - + + # Determine expected value + if checksums.has_key('md5sum'): + expected = checksums['md5sum'] + else: + if '#md5=' in download_url: + expected = download_url.split('#md5=')[1] + else: + expected = download_url.split('#md5#')[1] + + if downloaded_checksum != expected: + raise Exception( 'Given md5 checksum does not match with the one from the downloaded file (%s != %s).' % (downloaded_checksum, expected) ) + if extract: if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): archive = CompressedFile( file_path ) @@ -620,7 +648,7 @@ class DownloadByUrl( Download, RecipeStep ): def __init__( self, app ): self.app = app self.type = 'download_by_url' - + def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -642,7 +670,7 @@ class DownloadByUrl( Download, RecipeStep ): downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] - + checksums = dict(filter(lambda i:i[0] in ['md5sum','sha256sum'], action_dict.iteritems())) dir = self.url_download( work_dir, downloaded_filename, url, extract=True, checksums=checksums ) if is_binary: @@ -666,15 +694,11 @@ class DownloadByUrl( Download, RecipeStep ): # # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 # - - md5sum = action_elem.get( 'md5sum', None ) - sha256sum = action_elem.get( 'sha256sum', None ) - - if(md5sum): - action_dict['md5sum'] = md5sum - if(sha256sum): - action_dict['sha256sum'] = sha256sum - + + for hash_type in ('md5sum', 'sha256sum'): + if hash_type in action_elem: + action_dict[hash_type] = action_elem[hash_type] + if is_binary_download: action_dict[ 'is_binary' ] = True if action_elem.text: @@ -903,26 +927,26 @@ class SetEnvironment( RecipeStep ): This method works with with a combination of three tool dependency definition tag sets, which are defined in the tool_dependencies.xml file in the order discussed here. The example for this discussion is the tool_dependencies.xml file contained in the osra repository, which is available at: - + https://testtoolshed.g2.bx.psu.edu/view/bgruening/osra - + The first tag set defines a complex repository dependency like this. This tag set ensures that changeset revision XXX of the repository named package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been previously installed. - + ... - + * By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks something like this (we'll reference this file later in this discussion. ---- GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; export GRAPHICSMAGICK_ROOT_DIR ---- - + The second tag set defines a specific package dependency that has been previously installed (guaranteed by the tag set discussed above) and compiled, where the compiled dependency is needed by the tool dependency currently being installed (osra version 2.0.0 in this case) and complied in order for its installation and compilation to @@ -931,21 +955,21 @@ class SetEnvironment( RecipeStep ): compile. When this tag set is handled, one of the effects is that the env.sh file associated with graphicsmagick version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment variables (e.g. PATH, PYTHONPATH, etc). - + - + The third tag set enables discovery of the same required package dependency discussed above for correctly compiling the osra version 2.0.0 package, but in this case the package can be discovered at tool execution time. Using the $ENV[] option as shown in this example, the value of the environment variable named GRAPHICSMAGICK_ROOT_DIR (which was set in the environment using the second tag set described above) will be used to automatically alter the env.sh file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we discussed the env.sh file for version 1.3.18 of the graphicsmagick package above. - + $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ $INSTALL_DIR/potrace/build/lib/ @@ -953,11 +977,11 @@ class SetEnvironment( RecipeStep ): $INSTALL_DIR/share - + The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy that looks something like this. Notice that the path to the gmagick binary is included here since it expands the defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. - + ---- LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH @@ -1700,7 +1724,7 @@ class SetupVirtualEnv( Download, RecipeStep ): # Use raw strings so that python won't automatically unescape the quotes before passing the command # to subprocess.Popen. for site_packages_command in [ r"""%s -c 'import site; site.getsitepackages()[0]'""" % \ - os.path.join( venv_directory, "bin", "python" ), + os.path.join( venv_directory, "bin", "python" ), r"""%s -c 'import os, sys; print os.path.join( sys.prefix, "lib", "python" + sys.version[:3], "site-packages" )'""" % \ os.path.join( venv_directory, "bin", "python" ) ]: output = install_environment.handle_command( tool_dependency=tool_dependency, From 72b7381fc288fe78026493433171e11d6f7fae76 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 13 Jul 2015 11:36:16 -0400 Subject: [PATCH 4/5] Improvements to @erasche's download hash changes suggested by @yhoogstrate. --- .../tool_dependencies/recipe/step_handler.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 04aaba6b56d..49da14737b9 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -9,6 +9,7 @@ import time import urllib2 import zipfile import hashlib +import re from galaxy.util import asbool from galaxy.util.template import fill_template @@ -193,28 +194,25 @@ class Download( object ): dst.close() if checksums.has_key('sha256sum') or '#sha256#' in download_url: - downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest() + downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest().lower() # Determine expected value if checksums.has_key('sha256sum'): - expected = checksums['sha256sum'] + expected = checksums['sha256sum'].lower() else: - expected = download_url.split('#sha256#')[1] + expected = download_url.split('#sha256#')[1].lower() if downloaded_checksum != expected: raise Exception( 'Given sha256 checksum does not match with the one from the downloaded file (%s != %s).' % (downloaded_checksum, expected) ) if checksums.has_key('md5sum') or '#md5#' in download_url or '#md5=' in download_url: - downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() + downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest().lower() # Determine expected value if checksums.has_key('md5sum'): - expected = checksums['md5sum'] + expected = checksums['md5sum'].lower() else: - if '#md5=' in download_url: - expected = download_url.split('#md5=')[1] - else: - expected = download_url.split('#md5#')[1] + expected = re.split('#md5[#=]', download_url)[1].lower() if downloaded_checksum != expected: raise Exception( 'Given md5 checksum does not match with the one from the downloaded file (%s != %s).' % (downloaded_checksum, expected) ) @@ -924,8 +922,8 @@ class SetEnvironment( RecipeStep ): def handle_environment_variables( self, install_environment, tool_dependency, env_var_dict, set_prior_environment_commands ): """ - This method works with with a combination of three tool dependency definition tag sets, which are defined - in the tool_dependencies.xml file in the order discussed here. The example for this discussion is the + This method works with a combination of three tool dependency definition tag sets, which are defined in + the tool_dependencies.xml file in the order discussed here. The example for this discussion is the tool_dependencies.xml file contained in the osra repository, which is available at: https://testtoolshed.g2.bx.psu.edu/view/bgruening/osra From 37b231cb3d0ece73daa4ba45149ec14d04abc1d6 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 13 Jul 2015 14:43:05 -0400 Subject: [PATCH 5/5] Verify MD5 and SHA256 sums on other types of actions. For all types of actions other than download_by_url, these must be specified as attributes on the tag, the old `#hash#` method in the URL is not supported. --- .../tool_dependencies/recipe/step_handler.py | 81 ++++++++++++------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 49da14737b9..3d284626567 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -228,6 +228,16 @@ class Download( object ): return extraction_path + def get_elem_checksums( self, elem ): + rval = {} + for hash_type in ('md5sum', 'sha256sum'): + if hash_type in elem.keys(): + rval[hash_type] = elem.get(hash_type).lower() + return rval + + def get_dict_checksums( self, dct ): + return dict(filter(lambda i:i[0] in ['md5sum','sha256sum'], dct.iteritems())) + class RecipeStep( object ): """Abstract class that defines a standard format for handling recipe steps when installing packages.""" @@ -554,10 +564,10 @@ class DownloadBinary( Download, RecipeStep ): self.app = app self.type = 'download_binary' - def download_binary( self, url, work_dir ): + def download_binary( self, url, work_dir, checksums ): """Download a pre-compiled binary from the specified URL.""" downloaded_filename = os.path.split( url )[ -1 ] - dir = self.url_download( work_dir, downloaded_filename, url, extract=False ) + dir = self.url_download( work_dir, downloaded_filename, url, extract=False, checksums=checksums ) return downloaded_filename def filter_actions_after_binary_installation( self, actions ): @@ -584,7 +594,8 @@ class DownloadBinary( Download, RecipeStep ): log.debug( 'Attempting to download from %s to %s', url, str( target_directory ) ) downloaded_filename = None try: - downloaded_filename = self.download_binary( url, work_dir ) + checksums = self.get_dict_checksums( action_dict ) + downloaded_filename = self.download_binary( url, work_dir, checksums ) if initial_download: # Filter out any actions that are not download_binary, chmod, or set_environment. filtered_actions = self.filter_actions_after_binary_installation( actions[ 1: ] ) @@ -638,6 +649,7 @@ class DownloadBinary( Download, RecipeStep ): url_template_elem = url_template_elems[ 0 ] action_dict[ 'url' ] = Template( url_template_elem.text ).safe_substitute( platform_info_dict ) action_dict[ 'target_directory' ] = action_elem.get( 'target_directory', None ) + action_dict.update( self.get_elem_checksums( action_elem ) ) return action_dict @@ -669,7 +681,7 @@ class DownloadByUrl( Download, RecipeStep ): else: downloaded_filename = os.path.split( url )[ -1 ] - checksums = dict(filter(lambda i:i[0] in ['md5sum','sha256sum'], action_dict.iteritems())) + checksums = self.get_dict_checksums( action_dict ) dir = self.url_download( work_dir, downloaded_filename, url, extract=True, checksums=checksums ) if is_binary: log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG ) @@ -692,11 +704,6 @@ class DownloadByUrl( Download, RecipeStep ): # # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 # - - for hash_type in ('md5sum', 'sha256sum'): - if hash_type in action_elem: - action_dict[hash_type] = action_elem[hash_type] - if is_binary_download: action_dict[ 'is_binary' ] = True if action_elem.text: @@ -704,6 +711,7 @@ class DownloadByUrl( Download, RecipeStep ): target_filename = action_elem.get( 'target_filename', None ) if target_filename: action_dict[ 'target_filename' ] = target_filename + action_dict.update( self.get_elem_checksums( action_elem ) ) return action_dict @@ -735,7 +743,8 @@ class DownloadFile( Download, RecipeStep ): filename = url.split( '/' )[ -1 ] if current_dir is not None: work_dir = current_dir - self.url_download( work_dir, filename, url, extract=action_dict[ 'extract' ] ) + checksums = self.get_dict_checksums( action_dict ) + self.url_download( work_dir, filename, url, extract=action_dict[ 'extract' ], checksums=checksums ) if initial_download: dir = os.path.curdir return tool_dependency, filtered_actions, dir @@ -749,6 +758,7 @@ class DownloadFile( Download, RecipeStep ): if target_filename: action_dict[ 'target_filename' ] = target_filename action_dict[ 'extract' ] = asbool( action_elem.get( 'extract', False ) ) + action_dict.update( self.get_elem_checksums( action_elem ) ) return action_dict @@ -1109,7 +1119,7 @@ class SetupPerlEnvironment( Download, RecipeStep ): # # # XML::Parser - # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz + # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz # dir = None if initial_download: @@ -1129,7 +1139,8 @@ class SetupPerlEnvironment( Download, RecipeStep ): with lcd( current_dir ): with settings( warn_only=True ): perl_packages = action_dict.get( 'perl_packages', [] ) - for perl_package in perl_packages: + for perl_package_dict in perl_packages: + perl_package = perl_package_dict['package'] # If set to a true value then MakeMaker's prompt function will always # return the default without waiting for user input. cmd = '''PERL_MM_USE_DEFAULT=1; export PERL_MM_USE_DEFAULT; ''' @@ -1139,7 +1150,8 @@ class SetupPerlEnvironment( Download, RecipeStep ): # We assume a URL to a gem file. url = perl_package perl_package_name = url.split( '/' )[ -1 ] - dir = self.url_download( work_dir, perl_package_name, url, extract=True ) + checksums = perl_package_dict.get('checksums', {}) + dir = self.url_download( work_dir, perl_package_name, url, extract=True, checksums=checksums ) # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). tmp_work_dir = os.path.join( work_dir, dir ) if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): @@ -1217,7 +1229,8 @@ class SetupPerlEnvironment( Download, RecipeStep ): # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz # Unfortunately CPAN does not support versioning, so if you want real reproducibility you need to specify # the tarball path and the right order of different tarballs manually. - perl_packages.append( env_elem.text.strip() ) + perl_packages.append( dict( package=env_elem.text.strip(), + checksums=self.get_elem_checksums( env_elem ) ) ) if perl_packages: action_dict[ 'perl_packages' ] = perl_packages return action_dict @@ -1244,7 +1257,7 @@ class SetupREnvironment( Download, RecipeStep ): # # # - # https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz + # https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz # dir = None if initial_download: @@ -1260,10 +1273,12 @@ class SetupREnvironment( Download, RecipeStep ): log.debug( 'Handling setup_r_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \ ( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) ) tarball_names = [] - for url in action_dict[ 'r_packages' ]: + for r_package_dict in action_dict[ 'r_packages' ]: + url = r_package_dict['package'] filename = url.split( '/' )[ -1 ] tarball_names.append( filename ) - self.url_download( work_dir, filename, url, extract=False ) + checksums = r_package_dict.get('checksums', {}) + self.url_download( work_dir, filename, url, extract=False, checksums=checksums ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): @@ -1315,7 +1330,8 @@ class SetupREnvironment( Download, RecipeStep ): r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': - r_packages.append( env_elem.text.strip() ) + r_packages.append( dict( package=env_elem.text.strip(), + checksums=self.get_elem_checksums( env_elem ) ) ) if r_packages: action_dict[ 'r_packages' ] = r_packages return action_dict @@ -1364,7 +1380,8 @@ class SetupRubyEnvironment( Download, RecipeStep ): with lcd( current_dir ): with settings( warn_only=True ): ruby_package_tups = action_dict.get( 'ruby_package_tups', [] ) - for ruby_package_tup in ruby_package_tups: + for ruby_package_tup_dict in ruby_package_tups: + ruby_package_tup = ruby_package_tup_dict['package'] gem, gem_version, gem_parameters = ruby_package_tup if gem_parameters: gem_parameters = '-- %s' % gem_parameters @@ -1378,7 +1395,8 @@ class SetupRubyEnvironment( Download, RecipeStep ): # We assume a URL to a gem file. url = gem gem_name = url.split( '/' )[ -1 ] - self.url_download( work_dir, gem_name, url, extract=False ) + checksums = ruby_package_tup_dict.get('checksums', {}) + self.url_download( work_dir, gem_name, url, extract=False, checksums=checksums ) cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install --local %s %s''' % ( gem_name, gem_parameters ) else: @@ -1427,7 +1445,7 @@ class SetupRubyEnvironment( Download, RecipeStep ): # # protk # protk=1.2.4 - # http://url-to-some-gem-file.de/protk.gem + # http://url-to-some-gem-file.de/protk.gem # # Discover all child repository dependency tags and define the path to an env.sh file # associated with each repository. This will potentially update the value of the @@ -1450,11 +1468,13 @@ class SetupRubyEnvironment( Download, RecipeStep ): # version string gem_name = gem_token[ 0 ] gem_version = gem_token[ 1 ] - ruby_package_tups.append( ( gem_name, gem_version, gem_parameters ) ) + tup = ( gem_name, gem_version, gem_parameters ) else: # gem name for rubygems.org without version number gem = env_elem.text.strip() - ruby_package_tups.append( ( gem, None, gem_parameters ) ) + tup = ( gem, None, gem_parameters ) + ruby_package_tups.append( dict( package=tup, + checksums=self.get_elem_checksums( env_elem ) ) ) if ruby_package_tups: action_dict[ 'ruby_package_tups' ] = ruby_package_tups return action_dict @@ -1485,7 +1505,7 @@ class SetupPythonEnvironment( Download, RecipeStep ): # # # pysam.tar.gz - # http://url-to-some-python-package.de/pysam.tar.gz + # http://url-to-some-python-package.de/pysam.tar.gz # dir = None if initial_download: @@ -1505,7 +1525,8 @@ class SetupPythonEnvironment( Download, RecipeStep ): with lcd( current_dir ): with settings( warn_only=True ): python_package_tups = action_dict.get( 'python_package_tups', [] ) - for python_package_tup in python_package_tups: + for python_package_tup_dict in python_package_tups: + python_package_tup = python_package_tup_dict['package'] package, package_version = python_package_tup package_path = os.path.join( install_environment.tool_shed_repository_install_dir, package ) if os.path.isfile( package_path ): @@ -1519,7 +1540,8 @@ class SetupPythonEnvironment( Download, RecipeStep ): # We assume a URL to a python package. url = package package_name = url.split( '/' )[ -1 ] - self.url_download( work_dir, package_name, url, extract=False ) + checksums = python_package_tup_dict.get('checksums', {}) + self.url_download( work_dir, package_name, url, extract=False, checksums=checksums ) cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH; export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR; @@ -1582,13 +1604,16 @@ class SetupPythonEnvironment( Download, RecipeStep ): # version string package_name = python_token[ 0 ] package_version = python_token[ 1 ] - python_package_tups.append( ( package_name, package_version ) ) + tup = ( package_name, package_version ) else: # package name for pypi.org without version number package = env_elem.text.strip() - python_package_tups.append( ( package, None ) ) + tup = ( package, None ) + python_package_tups.append( dict( package=tup, + checksums=self.get_elem_checksums( env_elem ) ) ) if python_package_tups: action_dict[ 'python_package_tups' ] = python_package_tups + action_dict.update( self.get_elem_checksums( action_elem ) ) return action_dict