update example & minor fix

This commit is contained in:
lich99
2024-05-30 20:54:15 +08:00
parent 6c52dfa268
commit d2c7d95390
3 changed files with 247 additions and 90 deletions
+3 -2
View File
@@ -16,8 +16,6 @@ from huggingface_hub import snapshot_download
logging.basicConfig(level = logging.INFO)
torch._dynamo.config.suppress_errors = True
torch.set_float32_matmul_precision('medium')
class Chat:
def __init__(self, ):
@@ -138,6 +136,9 @@ class Chat:
assert self.check_model(use_decoder=use_decoder)
if not isinstance(text, list):
text = [text]
if do_text_normalization:
for i, t in enumerate(text):
_lang = detect_language(t) if lang is None else lang
+1 -1
View File
@@ -48,7 +48,7 @@ class CustomRepetitionPenaltyLogitsProcessor():
def count_invalid_characters(s):
s = re.sub(r'\[uv_break\]|\[laugh\]|\[lbreak\]', '', s)
pattern = re.compile(r'[^\u4e00-\u9fffA-Za-z,。,\. ]')
pattern = re.compile(r'[^\u4e00-\u9fffA-Za-z,。,\. ]')
non_alphabetic_chinese_chars = pattern.findall(s)
return set(non_alphabetic_chinese_chars)
File diff suppressed because one or more lines are too long