opensource/dify
3
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-24 23:22:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix remove_leading_symbols remove [ (#34832)

Browse Source 
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
wangxiaolei
2026-04-09 08:22:09 +00:00
committed by GitHub
co-authored by autofix-ci[bot]
parent 03750b76ac
commit d042cbc62e
2 changed files with 65 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/core/tools/utils/text_processing_utils.py
+14 -1
View File
@@ -19,5 +19,18 @@ def remove_leading_symbols(text: str) -> str:
# Match Unicode ranges for punctuation and symbols
# FIXME this pattern is confused quick fix for #11868 maybe refactor it later
pattern = r'^[\[\]\u2000-\u2025\u2027-\u206F\u2E00-\u2E7F\u3000-\u300F\u3011-\u303F"#$%&\'()*+,./:;<=>?@^_`~]+'
pattern = re.compile(
r"""
^
(?:
[\u2000-\u2025] # General Punctuation: spaces, quotes, dashes
| [\u2027-\u206F] # General Punctuation: ellipsis, underscores, etc.
| [\u2E00-\u2E7F] # Supplemental Punctuation: medieval, ancient marks
| [\u3000-\u300F] # CJK Punctuation: 、。〃「」『》』 (excludes 【】)
| [\u3012-\u303F] # CJK Punctuation: 〖〗〔〕〘〙〚〛〜 etc.
| ["#$%&'()*+,./:;<=>?@^_`~] # ASCII punctuation (excludes []【】)
)+
""",
re.VERBOSE,
)
return re.sub(pattern, "", text)