From 71c79fda324877fb133f5f4194ac546be3041ee2 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Tue, 27 Feb 2018 17:52:02 +0100 Subject: [PATCH 1/2] File sizes were not being added --- scripts/grt/export.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index f5a3480c14b..879c1e4ffaa 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -78,7 +78,7 @@ class Sanitization: self.sanitization_config['tool_params'] = {} def blacklisted_tree(self, path): - if path.lstrip('.') in self.sanitization_config['tool_params'][self.tool_id]: + if self.tool_id in self.sanitization_config['tool_params'] and path.lstrip('.') in self.sanitization_config['tool_params'][self.tool_id]: return True return False @@ -88,17 +88,12 @@ class Sanitization: return 'null' # Thus, all tools below here are not blacklisted at the top level. - # If it isn't in tool_params, there are no keys being sanitized for - # this tool so we can return quickly without parsing. - if tool_id not in self.sanitization_config['tool_params']: - return value - # If the key is listed precisely (not a sub-tree), we can also return slightly more quickly. - if key in self.sanitization_config['tool_params'][tool_id]: + if tool_id in self.sanitization_config['tool_params'] and key in self.sanitization_config['tool_params'][tool_id]: return 'null' # If the key isn't a prefix for any of the keys being sanitized, then this is safe. - if not any(san_key.startswith(key) for san_key in self.sanitization_config['tool_params'][tool_id]): + if tool_id in self.sanitization_config['tool_params'] and not any(san_key.startswith(key) for san_key in self.sanitization_config['tool_params'][tool_id]): return value # Slow path. @@ -126,7 +121,8 @@ class Sanitization: self.filesize_cache[data['id']] = data return data else: - raise Exception("Cannot handle {src} yet".format(data)) + logging.warning("Cannot handle {src} yet".format(data)) + return data def _sanitize_dict(self, unsanitized_dict, path=""): # if it is a file dictionary, handle specially. From 6a20984c57501b49a74dccb7992799a19abf7e6e Mon Sep 17 00:00:00 2001 From: E Rasche Date: Tue, 27 Feb 2018 18:06:52 +0100 Subject: [PATCH 2/2] try harder to handle unexpected values --- scripts/grt/export.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 879c1e4ffaa..9874b7edf8b 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -97,7 +97,14 @@ class Sanitization: return value # Slow path. - unsanitized = {key: json.loads(value)} + if isinstance(value, str): + try: + unsanitized = {key: json.loads(value)} + except ValueError: + unsanitized = {key: value} + else: + unsanitized = {key: value} + self.tool_id = tool_id return json.dumps(self._sanitize_value(unsanitized))