From 48548341fc4724f04d85a012a2745a5914636c15 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 28 Aug 2012 19:23:39 -0400 Subject: [PATCH] Handle non-ascii unicode in upload tool. Significant pre-existing refactoring still needed. --- tools/data_source/upload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 7792d0d5eeb..3d13f2bae5a 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -90,7 +90,8 @@ def add_file( dataset, registry, json_file, output_path ): if dataset.type == 'url': try: - temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dataset.path ), prefix='url_paste' ) + page = urllib.urlopen( dataset.path ) #page will be .close()ed by sniff methods + temp_name, dataset.is_multi_byte = sniff.stream_to_file( page, prefix='url_paste', source_encoding=util.get_charset_from_http_headers( page.headers ) ) except Exception, e: file_err( 'Unable to fetch %s\n%s' % ( dataset.path, str( e ) ), dataset, json_file ) return