mirror of
https://github.com/opendatalab/MinerU.git
synced 2026-11-02 21:42:09 +08:00
1015 lines
38 KiB
Python
1015 lines
38 KiB
Python
import os
|
||
import re
|
||
import numpy as np
|
||
|
||
from magic_pdf.libs.nlp_utils import NLPModels
|
||
|
||
from magic_pdf.para.commons import *
|
||
|
||
if sys.version_info[0] >= 3:
|
||
sys.stdout.reconfigure(encoding="utf-8") # type: ignore
|
||
|
||
|
||
class TitleProcessor:
|
||
def __init__(self, *doc_statistics) -> None:
|
||
if len(doc_statistics) > 0:
|
||
self.doc_statistics = doc_statistics[0]
|
||
|
||
self.nlp_model = NLPModels()
|
||
self.MAX_TITLE_LEVEL = 3
|
||
self.numbered_title_pattern = r"""
|
||
^ # 行首
|
||
( # 开始捕获组
|
||
[\(\（]\d+[\)\）] # 括号内数字，支持中文和英文括号，例如：(1) 或 （1）
|
||
|\d+[\)\）]\s # 数字后跟右括号和空格，支持中文和英文括号，例如：2) 或 2）
|
||
|[\(\（][A-Z][\)\）] # 括号内大写字母，支持中文和英文括号，例如：(A) 或 （A）
|
||
|[A-Z][\)\）]\s # 大写字母后跟右括号和空格，例如：A) 或 A）
|
||
|[\(\（][IVXLCDM]+[\)\）] # 括号内罗马数字，支持中文和英文括号，例如：(I) 或 （I）
|
||
|[IVXLCDM]+[\)\）]\s # 罗马数字后跟右括号和空格，例如：I) 或 I）
|
||
|\d+(\.\d+)*\s # 数字或复合数字编号后跟空格，例如：1. 或 3.2.1
|
||
|[一二三四五六七八九十百千]+[、\s] # 中文序号后跟顿号和空格，例如：一、
|
||
|[\（|\(][一二三四五六七八九十百千]+[\）|\)]\s* # 中文括号内中文序号后跟空格，例如：（一）
|
||
|[A-Z]\.\d+(\.\d+)?\s # 大写字母后跟点和数字，例如：A.1 或 A.1.1
|
||
|[\(\（][a-z][\)\）] # 括号内小写字母，支持中文和英文括号，例如：(a) 或 （a）
|
||
|[a-z]\)\s # 小写字母后跟右括号和空格，例如：a)
|
||
|[A-Z]-\s # 大写字母后跟短横线和空格，例如：A-
|
||
|\w+:\s # 英文序号词后跟冒号和空格，例如：First:
|
||
|第[一二三四五六七八九十百千]+[章节部分条款]\s # 以“第”开头的中文标题后跟空格
|
||
|[IVXLCDM]+\. # 罗马数字后跟点，例如：I.
|
||
|\d+\.\s # 单个数字后跟点和空格，例如：1.
|
||
) # 结束捕获组
|
||
.+ # 标题的其余部分
|
||
"""
|
||
|
||
def _is_potential_title(
|
||
self,
|
||
curr_line,
|
||
prev_line,
|
||
prev_line_is_title,
|
||
next_line,
|
||
avg_char_width,
|
||
avg_char_height,
|
||
median_font_size,
|
||
):
|
||
"""
|
||
This function checks if the line is a potential title.
|
||
|
||
Parameters
|
||
----------
|
||
curr_line : dict
|
||
current line
|
||
prev_line : dict
|
||
previous line
|
||
next_line : dict
|
||
next line
|
||
avg_char_width : float
|
||
average of char widths
|
||
avg_char_height : float
|
||
average of line heights
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the line is a potential title, False otherwise.
|
||
"""
|
||
|
||
def __is_line_centered(line_bbox, page_bbox, avg_char_width):
|
||
"""
|
||
This function checks if the line is centered on the page
|
||
|
||
Parameters
|
||
----------
|
||
line_bbox : list
|
||
bbox of the line
|
||
page_bbox : list
|
||
bbox of the page
|
||
avg_char_width : float
|
||
average of char widths
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the line is centered on the page, False otherwise.
|
||
"""
|
||
horizontal_ratio = 0.5
|
||
horizontal_thres = horizontal_ratio * avg_char_width
|
||
|
||
x0, _, x1, _ = line_bbox
|
||
_, _, page_x1, _ = page_bbox
|
||
|
||
return abs((x0 + x1) / 2 - page_x1 / 2) < horizontal_thres
|
||
|
||
def __is_bold_font_line(line):
|
||
"""
|
||
Check if a line contains any bold font style.
|
||
"""
|
||
|
||
def _is_bold_span(span):
|
||
# if span text is empty or only contains space, return False
|
||
if not span["text"].strip():
|
||
return False
|
||
|
||
return bool(span["flags"] & 2**4) # Check if the font is bold
|
||
|
||
for span in line["spans"]:
|
||
if not _is_bold_span(span):
|
||
return False
|
||
|
||
return True
|
||
|
||
def __is_italic_font_line(line):
|
||
"""
|
||
Check if a line contains any italic font style.
|
||
"""
|
||
|
||
def __is_italic_span(span):
|
||
return bool(span["flags"] & 2**1) # Check if the font is italic
|
||
|
||
for span in line["spans"]:
|
||
if not __is_italic_span(span):
|
||
return False
|
||
|
||
return True
|
||
|
||
def __is_punctuation_heavy(line_text):
|
||
"""
|
||
Check if the line contains a high ratio of punctuation marks, which may indicate
|
||
that the line is not a title.
|
||
|
||
Parameters:
|
||
line_text (str): Text of the line.
|
||
|
||
Returns:
|
||
bool: True if the line is heavy with punctuation, False otherwise.
|
||
"""
|
||
# Pattern for common title format like "X.Y. Title"
|
||
pattern = r"\b\d+\.\d+\..*\b"
|
||
|
||
# If the line matches the title format, return False
|
||
if re.match(pattern, line_text.strip()):
|
||
return False
|
||
|
||
# Find all punctuation marks in the line
|
||
punctuation_marks = re.findall(r"[^\w\s]", line_text)
|
||
number_of_punctuation_marks = len(punctuation_marks)
|
||
|
||
text_length = len(line_text)
|
||
|
||
if text_length == 0:
|
||
return False
|
||
|
||
punctuation_ratio = number_of_punctuation_marks / text_length
|
||
if punctuation_ratio >= 0.1:
|
||
return True
|
||
|
||
return False
|
||
|
||
def __has_mixed_font_styles(spans, strict_mode=False):
|
||
"""
|
||
This function checks if the line has mixed font styles, the strict mode will compare the font types
|
||
|
||
Parameters
|
||
----------
|
||
spans : list
|
||
spans of the line
|
||
strict_mode : bool
|
||
True for strict mode, the font types will be fully compared
|
||
False for non-strict mode, the font types will be compared by the most longest common prefix
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the line has mixed font styles, False otherwise.
|
||
"""
|
||
if strict_mode:
|
||
font_styles = set()
|
||
for span in spans:
|
||
font_style = span["font"].lower()
|
||
font_styles.add(font_style)
|
||
|
||
return len(font_styles) > 1
|
||
|
||
else: # non-strict mode
|
||
font_styles = []
|
||
for span in spans:
|
||
font_style = span["font"].lower()
|
||
font_styles.append(font_style)
|
||
|
||
if len(font_styles) > 1:
|
||
longest_common_prefix = os.path.commonprefix(font_styles)
|
||
if len(longest_common_prefix) > 0:
|
||
return False
|
||
else:
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
def __is_different_font_type_from_neighbors(curr_line_font_type, prev_line_font_type, next_line_font_type):
|
||
"""
|
||
This function checks if the current line has a different font type from the previous and next lines
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_font_type : str
|
||
font type of the current line
|
||
prev_line_font_type : str
|
||
font type of the previous line
|
||
next_line_font_type : str
|
||
font type of the next line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line has a different font type from the previous and next lines, False otherwise.
|
||
"""
|
||
return all(
|
||
curr_line_font_type != other_font_type.lower()
|
||
for other_font_type in [prev_line_font_type, next_line_font_type]
|
||
if other_font_type is not None
|
||
)
|
||
|
||
def __is_larger_font_size_from_neighbors(curr_line_font_size, prev_line_font_size, next_line_font_size):
|
||
"""
|
||
This function checks if the current line has a larger font size than the previous and next lines
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_font_size : float
|
||
font size of the current line
|
||
prev_line_font_size : float
|
||
font size of the previous line
|
||
next_line_font_size : float
|
||
font size of the next line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line has a larger font size than the previous and next lines, False otherwise.
|
||
"""
|
||
return all(
|
||
curr_line_font_size > other_font_size * 1.2
|
||
for other_font_size in [prev_line_font_size, next_line_font_size]
|
||
if other_font_size is not None
|
||
)
|
||
|
||
def __is_similar_to_pre_line(curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size):
|
||
"""
|
||
This function checks if the current line is similar to the previous line
|
||
|
||
Parameters
|
||
----------
|
||
curr_line : dict
|
||
current line
|
||
prev_line : dict
|
||
previous line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line is similar to the previous line, False otherwise.
|
||
"""
|
||
|
||
if curr_line_font_type == prev_line_font_type and curr_line_font_size == prev_line_font_size:
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
def __is_same_font_type_of_docAvg(curr_line_font_type):
|
||
"""
|
||
This function checks if the current line has the same font type as the document average font type
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_font_type : str
|
||
font type of the current line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line has the same font type as the document average font type, False otherwise.
|
||
"""
|
||
doc_most_common_font_type = safe_get(self.doc_statistics, "most_common_font_type", "").lower()
|
||
doc_second_most_common_font_type = safe_get(self.doc_statistics, "second_most_common_font_type", "").lower()
|
||
|
||
return curr_line_font_type.lower() in [doc_most_common_font_type, doc_second_most_common_font_type]
|
||
|
||
def __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio: float = 1):
|
||
"""
|
||
This function checks if the current line has a large enough font size
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_font_size : float
|
||
font size of the current line
|
||
ratio : float
|
||
ratio of the current line font size to the document average font size
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line has a large enough font size, False otherwise.
|
||
"""
|
||
doc_most_common_font_size = safe_get(self.doc_statistics, "most_common_font_size", 0)
|
||
doc_second_most_common_font_size = safe_get(self.doc_statistics, "second_most_common_font_size", 0)
|
||
doc_avg_font_size = min(doc_most_common_font_size, doc_second_most_common_font_size)
|
||
|
||
return curr_line_font_size >= doc_avg_font_size * ratio
|
||
|
||
def __is_sufficient_spacing_above_and_below(
|
||
curr_line_bbox,
|
||
prev_line_bbox,
|
||
next_line_bbox,
|
||
avg_char_height,
|
||
median_font_size,
|
||
):
|
||
"""
|
||
This function checks if the current line has sufficient spacing above and below
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_bbox : list
|
||
bbox of the current line
|
||
prev_line_bbox : list
|
||
bbox of the previous line
|
||
next_line_bbox : list
|
||
bbox of the next line
|
||
avg_char_width : float
|
||
average of char widths
|
||
avg_char_height : float
|
||
average of line heights
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line has sufficient spacing above and below, False otherwise.
|
||
"""
|
||
vertical_ratio = 1.25
|
||
vertical_thres = vertical_ratio * median_font_size
|
||
|
||
_, y0, _, y1 = curr_line_bbox
|
||
|
||
sufficient_spacing_above = False
|
||
if prev_line_bbox:
|
||
vertical_spacing_above = min(y0 - prev_line_bbox[1], y1 - prev_line_bbox[3])
|
||
sufficient_spacing_above = vertical_spacing_above > vertical_thres
|
||
else:
|
||
sufficient_spacing_above = True
|
||
|
||
sufficient_spacing_below = False
|
||
if next_line_bbox:
|
||
vertical_spacing_below = min(next_line_bbox[1] - y0, next_line_bbox[3] - y1)
|
||
sufficient_spacing_below = vertical_spacing_below > vertical_thres
|
||
else:
|
||
sufficient_spacing_below = True
|
||
|
||
return (sufficient_spacing_above, sufficient_spacing_below)
|
||
|
||
def __is_word_list_line_by_rules(curr_line_text):
|
||
"""
|
||
This function checks if the current line is a word list
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_text : str
|
||
text of the current line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line is a name list, False otherwise.
|
||
"""
|
||
# name_list_pattern = r"([a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]|[\u4e00-\u9fa5·]{2,16})(?=[，,;；\s]|$)"
|
||
name_list_pattern = r"(?<![\u4e00-\u9fa5])([A-Z][a-z]{0,19}\s[A-Z][a-z]{0,19}|[\u4e00-\u9fa5]{2,6})(?=[，,;；\s]|$)"
|
||
|
||
compiled_pattern = re.compile(name_list_pattern)
|
||
|
||
if compiled_pattern.search(curr_line_text):
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
# """
|
||
def __get_text_catgr_by_nlp(curr_line_text):
|
||
"""
|
||
This function checks if the current line is a name list using nlp model, such as spacy
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_text : str
|
||
text of the current line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line is a name list, False otherwise.
|
||
"""
|
||
|
||
result = self.nlp_model.detect_entity_catgr_using_nlp(curr_line_text)
|
||
|
||
return result
|
||
|
||
# """
|
||
|
||
def __is_numbered_title(curr_line_text):
|
||
"""
|
||
This function checks if the current line is a numbered list
|
||
|
||
Parameters
|
||
----------
|
||
curr_line_text : str
|
||
text of the current line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line is a numbered list, False otherwise.
|
||
"""
|
||
|
||
compiled_pattern = re.compile(self.numbered_title_pattern, re.VERBOSE)
|
||
|
||
if compiled_pattern.search(curr_line_text):
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
def __is_end_with_ending_puncs(line_text):
|
||
"""
|
||
This function checks if the current line ends with a ending punctuation mark
|
||
|
||
Parameters
|
||
----------
|
||
line_text : str
|
||
text of the current line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line ends with a punctuation mark, False otherwise.
|
||
"""
|
||
end_puncs = [".", "?", "!", "。", "？", "！", "…"]
|
||
|
||
line_text = line_text.rstrip()
|
||
if line_text[-1] in end_puncs:
|
||
return True
|
||
|
||
return False
|
||
|
||
def __contains_only_no_meaning_symbols(line_text):
|
||
"""
|
||
This function checks if the current line contains only symbols that have no meaning, if so, it is not a title.
|
||
Situation contains:
|
||
1. Only have punctuation marks
|
||
2. Only have other non-meaning symbols
|
||
|
||
Parameters
|
||
----------
|
||
line_text : str
|
||
text of the current line
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line contains only symbols that have no meaning, False otherwise.
|
||
"""
|
||
|
||
punctuation_marks = re.findall(r"[^\w\s]", line_text) # find all punctuation marks
|
||
number_of_punctuation_marks = len(punctuation_marks)
|
||
|
||
text_length = len(line_text)
|
||
|
||
if text_length == 0:
|
||
return False
|
||
|
||
punctuation_ratio = number_of_punctuation_marks / text_length
|
||
if punctuation_ratio >= 0.9:
|
||
return True
|
||
|
||
return False
|
||
|
||
def __is_equation(line_text):
|
||
"""
|
||
This function checks if the current line is an equation.
|
||
|
||
Parameters
|
||
----------
|
||
line_text : str
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line is an equation, False otherwise.
|
||
"""
|
||
equation_reg = r"\$.*?\\overline.*?\$" # to match interline equations
|
||
|
||
if re.search(equation_reg, line_text):
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
def __is_title_by_len(text, max_length=200):
|
||
"""
|
||
This function checks if the current line is a title by length.
|
||
|
||
Parameters
|
||
----------
|
||
text : str
|
||
text of the current line
|
||
|
||
max_length : int
|
||
max length of the title
|
||
|
||
Returns
|
||
-------
|
||
bool
|
||
True if the current line is a title, False otherwise.
|
||
|
||
"""
|
||
text = text.strip()
|
||
return len(text) <= max_length
|
||
|
||
def __compute_line_font_type_and_size(curr_line):
|
||
"""
|
||
This function computes the font type and font size of the line.
|
||
|
||
Parameters
|
||
----------
|
||
line : dict
|
||
line
|
||
|
||
Returns
|
||
-------
|
||
font_type : str
|
||
font type of the line
|
||
font_size : float
|
||
font size of the line
|
||
"""
|
||
spans = curr_line["spans"]
|
||
max_accumulated_length = 0
|
||
max_span_font_size = curr_line["spans"][0]["size"] # default value, float type
|
||
max_span_font_type = curr_line["spans"][0]["font"].lower() # default value, string type
|
||
for span in spans:
|
||
if span["text"].isspace():
|
||
continue
|
||
span_length = span["bbox"][2] - span["bbox"][0]
|
||
if span_length > max_accumulated_length:
|
||
max_accumulated_length = span_length
|
||
max_span_font_size = span["size"]
|
||
max_span_font_type = span["font"].lower()
|
||
|
||
return max_span_font_type, max_span_font_size
|
||
|
||
"""
|
||
Title detecting main Process.
|
||
"""
|
||
|
||
"""
|
||
Basic features about the current line.
|
||
"""
|
||
curr_line_bbox = curr_line["bbox"]
|
||
curr_line_text = curr_line["text"]
|
||
curr_line_font_type, curr_line_font_size = __compute_line_font_type_and_size(curr_line)
|
||
|
||
if len(curr_line_text.strip()) == 0: # skip empty lines
|
||
return False
|
||
|
||
prev_line_bbox = prev_line["bbox"] if prev_line else None
|
||
if prev_line:
|
||
prev_line_font_type, prev_line_font_size = __compute_line_font_type_and_size(prev_line)
|
||
else:
|
||
prev_line_font_type, prev_line_font_size = None, None
|
||
|
||
next_line_bbox = next_line["bbox"] if next_line else None
|
||
if next_line:
|
||
next_line_font_type, next_line_font_size = __compute_line_font_type_and_size(next_line)
|
||
else:
|
||
next_line_font_type, next_line_font_size = None, None
|
||
|
||
"""
|
||
Aggregated features about the current line.
|
||
"""
|
||
is_italc_font = __is_italic_font_line(curr_line)
|
||
is_bold_font = __is_bold_font_line(curr_line)
|
||
|
||
is_font_size_little_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=0.8)
|
||
is_font_size_not_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1)
|
||
is_much_larger_font_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1.6)
|
||
|
||
is_not_same_font_type_of_docAvg = not __is_same_font_type_of_docAvg(curr_line_font_type)
|
||
|
||
is_potential_title_font = is_bold_font or is_font_size_not_less_than_doc_avg or is_not_same_font_type_of_docAvg
|
||
|
||
is_mix_font_styles_strict = __has_mixed_font_styles(curr_line["spans"], strict_mode=True)
|
||
is_mix_font_styles_loose = __has_mixed_font_styles(curr_line["spans"], strict_mode=False)
|
||
|
||
is_punctuation_heavy = __is_punctuation_heavy(curr_line_text)
|
||
|
||
is_word_list_line_by_rules = __is_word_list_line_by_rules(curr_line_text)
|
||
is_person_or_org_list_line_by_nlp = __get_text_catgr_by_nlp(curr_line_text) in ["PERSON", "GPE", "ORG"]
|
||
|
||
is_font_size_larger_than_neighbors = __is_larger_font_size_from_neighbors(
|
||
curr_line_font_size, prev_line_font_size, next_line_font_size
|
||
)
|
||
|
||
is_font_type_diff_from_neighbors = __is_different_font_type_from_neighbors(
|
||
curr_line_font_type, prev_line_font_type, next_line_font_type
|
||
)
|
||
|
||
has_sufficient_spaces_above, has_sufficient_spaces_below = __is_sufficient_spacing_above_and_below(
|
||
curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_height, median_font_size
|
||
)
|
||
|
||
is_similar_to_pre_line = __is_similar_to_pre_line(
|
||
curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size
|
||
)
|
||
|
||
"""
|
||
Further aggregated features about the current line.
|
||
|
||
Attention:
|
||
Features that start with __ are for internal use.
|
||
"""
|
||
|
||
__is_line_left_aligned_from_neighbors = is_line_left_aligned_from_neighbors(
|
||
curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width
|
||
)
|
||
__is_font_diff_from_neighbors = is_font_size_larger_than_neighbors or is_font_type_diff_from_neighbors
|
||
is_a_left_inline_title = (
|
||
is_mix_font_styles_strict and __is_line_left_aligned_from_neighbors and __is_font_diff_from_neighbors
|
||
)
|
||
|
||
is_title_by_check_prev_line = prev_line is None and has_sufficient_spaces_above and is_potential_title_font
|
||
is_title_by_check_next_line = next_line is None and has_sufficient_spaces_below and is_potential_title_font
|
||
|
||
is_title_by_check_pre_and_next_line = (
|
||
(prev_line is not None or next_line is not None)
|
||
and has_sufficient_spaces_above
|
||
and has_sufficient_spaces_below
|
||
and is_potential_title_font
|
||
)
|
||
|
||
is_numbered_title = __is_numbered_title(curr_line_text) and (
|
||
(has_sufficient_spaces_above or prev_line is None) and (has_sufficient_spaces_below or next_line is None)
|
||
)
|
||
|
||
is_not_end_with_ending_puncs = not __is_end_with_ending_puncs(curr_line_text)
|
||
|
||
is_not_only_no_meaning_symbols = not __contains_only_no_meaning_symbols(curr_line_text)
|
||
|
||
is_equation = __is_equation(curr_line_text)
|
||
|
||
is_title_by_len = __is_title_by_len(curr_line_text)
|
||
|
||
"""
|
||
Decide if the line is a title.
|
||
"""
|
||
# is_title = False
|
||
# if prev_line_is_title:
|
||
|
||
is_title = (
|
||
is_not_end_with_ending_puncs # not end with ending punctuation marks
|
||
and is_not_only_no_meaning_symbols # not only have no meaning symbols
|
||
and is_title_by_len # is a title by length, default max length is 200
|
||
and not is_equation # an interline equation should never be a title
|
||
and is_potential_title_font # is a potential title font, which is bold or larger than the document average font size or not the same font type as the document average font type
|
||
and (
|
||
(is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
|
||
or (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
|
||
or (
|
||
is_much_larger_font_than_doc_avg
|
||
and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
|
||
)
|
||
or (
|
||
is_font_size_little_less_than_doc_avg
|
||
and is_bold_font
|
||
and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
|
||
)
|
||
) # not the same font type as the document average font type, which includes the most common font type and the second most common font type
|
||
and (
|
||
(
|
||
not is_person_or_org_list_line_by_nlp
|
||
and (
|
||
is_much_larger_font_than_doc_avg
|
||
or (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
|
||
)
|
||
)
|
||
or (
|
||
not (is_word_list_line_by_rules and is_person_or_org_list_line_by_nlp)
|
||
and not is_a_left_inline_title
|
||
and not is_punctuation_heavy
|
||
and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
|
||
)
|
||
or (
|
||
is_person_or_org_list_line_by_nlp
|
||
and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
|
||
and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
|
||
)
|
||
or (is_numbered_title and not is_a_left_inline_title)
|
||
)
|
||
)
|
||
# ) or (is_similar_to_pre_line and prev_line_is_title)
|
||
|
||
is_name_or_org_list_to_be_removed = (
|
||
(is_person_or_org_list_line_by_nlp)
|
||
and is_punctuation_heavy
|
||
and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
|
||
) and not is_title
|
||
|
||
if is_name_or_org_list_to_be_removed:
|
||
is_author_or_org_list = True
|
||
# print curr_line_text to check
|
||
# print_yellow(f"Text of is_author_or_org_list: {curr_line_text}")
|
||
else:
|
||
is_author_or_org_list = False
|
||
"""
|
||
# print reason why the line is a title
|
||
if is_title:
|
||
print_green("This line is a title.")
|
||
print_green("↓" * 10)
|
||
print()
|
||
print("curr_line_text: ", curr_line_text)
|
||
print()
|
||
|
||
# print reason why the line is not a title
|
||
line_text = curr_line_text.strip()
|
||
test_text = "Career/Personal Life"
|
||
text_content_condition = line_text == test_text
|
||
|
||
if not is_title and text_content_condition: # Print specific line
|
||
# if not is_title: # Print each line
|
||
print_red("This line is not a title.")
|
||
print_red("↓" * 10)
|
||
|
||
print()
|
||
print("curr_line_text: ", curr_line_text)
|
||
print()
|
||
|
||
if is_not_end_with_ending_puncs:
|
||
print_green(f"is_not_end_with_ending_puncs")
|
||
else:
|
||
print_red(f"is_end_with_ending_puncs")
|
||
|
||
if is_not_only_no_meaning_symbols:
|
||
print_green(f"is_not_only_no_meaning_symbols")
|
||
else:
|
||
print_red(f"is_only_no_meaning_symbols")
|
||
|
||
if is_title_by_len:
|
||
print_green(f"is_title_by_len: {is_title_by_len}")
|
||
else:
|
||
print_red(f"is_not_title_by_len: {is_title_by_len}")
|
||
|
||
if is_equation:
|
||
print_red(f"is_equation")
|
||
else:
|
||
print_green(f"is_not_equation")
|
||
|
||
if is_potential_title_font:
|
||
print_green(f"is_potential_title_font")
|
||
else:
|
||
print_red(f"is_not_potential_title_font")
|
||
|
||
if is_punctuation_heavy:
|
||
print_red("is_punctuation_heavy")
|
||
else:
|
||
print_green("is_not_punctuation_heavy")
|
||
|
||
if is_bold_font:
|
||
print_green(f"is_bold_font")
|
||
else:
|
||
print_red(f"is_not_bold_font")
|
||
|
||
if is_font_size_not_less_than_doc_avg:
|
||
print_green(f"is_larger_font_than_doc_avg")
|
||
else:
|
||
print_red(f"is_not_larger_font_than_doc_avg")
|
||
|
||
if is_much_larger_font_than_doc_avg:
|
||
print_green(f"is_much_larger_font_than_doc_avg")
|
||
else:
|
||
print_red(f"is_not_much_larger_font_than_doc_avg")
|
||
|
||
if is_not_same_font_type_of_docAvg:
|
||
print_green(f"is_not_same_font_type_of_docAvg")
|
||
else:
|
||
print_red(f"is_same_font_type_of_docAvg")
|
||
|
||
if is_word_list_line_by_rules:
|
||
print_red("is_word_list_line_by_rules")
|
||
else:
|
||
print_green("is_not_name_list_by_rules")
|
||
|
||
if is_person_or_org_list_line_by_nlp:
|
||
print_red("is_person_or_org_list_line_by_nlp")
|
||
else:
|
||
print_green("is_not_person_or_org_list_line_by_nlp")
|
||
|
||
if not is_numbered_title:
|
||
print_red("is_not_numbered_title")
|
||
else:
|
||
print_green("is_numbered_title")
|
||
|
||
if is_a_left_inline_title:
|
||
print_red("is_a_left_inline_title")
|
||
else:
|
||
print_green("is_not_a_left_inline_title")
|
||
|
||
if not is_title_by_check_prev_line:
|
||
print_red("is_not_title_by_check_prev_line")
|
||
else:
|
||
print_green("is_title_by_check_prev_line")
|
||
|
||
if not is_title_by_check_next_line:
|
||
print_red("is_not_title_by_check_next_line")
|
||
else:
|
||
print_green("is_title_by_check_next_line")
|
||
|
||
if not is_title_by_check_pre_and_next_line:
|
||
print_red("is_not_title_by_check_pre_and_next_line")
|
||
else:
|
||
print_green("is_title_by_check_pre_and_next_line")
|
||
|
||
# print_green("Common features:")
|
||
# print_green("↓" * 10)
|
||
|
||
# print(f" curr_line_font_type: {curr_line_font_type}")
|
||
# print(f" curr_line_font_size: {curr_line_font_size}")
|
||
# print()
|
||
|
||
"""
|
||
|
||
return is_title, is_author_or_org_list
|
||
|
||
def _detect_block_title(self, input_block):
|
||
"""
|
||
Use the functions 'is_potential_title' to detect titles of each paragraph block.
|
||
If a line is a title, then the value of key 'is_title' of the line will be set to True.
|
||
"""
|
||
|
||
raw_lines = input_block["lines"]
|
||
|
||
prev_line_is_title_flag = False
|
||
|
||
for i, curr_line in enumerate(raw_lines):
|
||
prev_line = raw_lines[i - 1] if i > 0 else None
|
||
next_line = raw_lines[i + 1] if i < len(raw_lines) - 1 else None
|
||
|
||
blk_avg_char_width = input_block["avg_char_width"]
|
||
blk_avg_char_height = input_block["avg_char_height"]
|
||
blk_media_font_size = input_block["median_font_size"]
|
||
|
||
is_title, is_author_or_org_list = self._is_potential_title(
|
||
curr_line,
|
||
prev_line,
|
||
prev_line_is_title_flag,
|
||
next_line,
|
||
blk_avg_char_width,
|
||
blk_avg_char_height,
|
||
blk_media_font_size,
|
||
)
|
||
|
||
if is_title:
|
||
curr_line["is_title"] = is_title
|
||
prev_line_is_title_flag = True
|
||
else:
|
||
curr_line["is_title"] = False
|
||
prev_line_is_title_flag = False
|
||
|
||
if is_author_or_org_list:
|
||
curr_line["is_author_or_org_list"] = is_author_or_org_list
|
||
else:
|
||
curr_line["is_author_or_org_list"] = False
|
||
|
||
return input_block
|
||
|
||
def batch_process_blocks_detect_titles(self, pdf_dic):
|
||
"""
|
||
This function batch process the blocks to detect titles.
|
||
|
||
Parameters
|
||
----------
|
||
pdf_dict : dict
|
||
result dictionary
|
||
|
||
Returns
|
||
-------
|
||
pdf_dict : dict
|
||
result dictionary
|
||
"""
|
||
num_titles = 0
|
||
|
||
for page_id, blocks in pdf_dic.items():
|
||
if page_id.startswith("page_"):
|
||
para_blocks = []
|
||
if "para_blocks" in blocks.keys():
|
||
para_blocks = blocks["para_blocks"]
|
||
|
||
all_single_line_blocks = []
|
||
for block in para_blocks:
|
||
if len(block["lines"]) == 1:
|
||
all_single_line_blocks.append(block)
|
||
|
||
new_para_blocks = []
|
||
if not len(all_single_line_blocks) == len(para_blocks): # Not all blocks are single line blocks.
|
||
for para_block in para_blocks:
|
||
new_block = self._detect_block_title(para_block)
|
||
new_para_blocks.append(new_block)
|
||
num_titles += sum([line.get("is_title", 0) for line in new_block["lines"]])
|
||
else: # All blocks are single line blocks.
|
||
for para_block in para_blocks:
|
||
new_para_blocks.append(para_block)
|
||
num_titles += sum([line.get("is_title", 0) for line in para_block["lines"]])
|
||
para_blocks = new_para_blocks
|
||
|
||
blocks["para_blocks"] = para_blocks
|
||
|
||
for para_block in para_blocks:
|
||
all_titles = all(safe_get(line, "is_title", False) for line in para_block["lines"])
|
||
para_text_len = sum([len(line["text"]) for line in para_block["lines"]])
|
||
if (
|
||
all_titles and para_text_len < 200
|
||
): # total length of the paragraph is less than 200, more than this should not be a title
|
||
para_block["is_block_title"] = 1
|
||
else:
|
||
para_block["is_block_title"] = 0
|
||
|
||
all_name_or_org_list_to_be_removed = all(
|
||
safe_get(line, "is_author_or_org_list", False) for line in para_block["lines"]
|
||
)
|
||
if all_name_or_org_list_to_be_removed and page_id == "page_0":
|
||
para_block["is_block_an_author_or_org_list"] = 1
|
||
else:
|
||
para_block["is_block_an_author_or_org_list"] = 0
|
||
|
||
pdf_dic["statistics"]["num_titles"] = num_titles
|
||
|
||
return pdf_dic
|
||
|
||
def __determine_size_based_level(self, title_blocks):
|
||
"""
|
||
This function determines the title level based on the font size of the title.
|
||
|
||
Parameters
|
||
----------
|
||
title_blocks : list
|
||
|
||
Returns
|
||
-------
|
||
title_blocks : list
|
||
"""
|
||
|
||
font_sizes = np.array([safe_get(tb["block"], "block_font_size", 0) for tb in title_blocks])
|
||
|
||
# Use the mean and std of font sizes to remove extreme values
|
||
mean_font_size = np.mean(font_sizes)
|
||
std_font_size = np.std(font_sizes)
|
||
min_extreme_font_size = mean_font_size - std_font_size # type: ignore
|
||
max_extreme_font_size = mean_font_size + std_font_size # type: ignore
|
||
|
||
# Compute the threshold for title level
|
||
middle_font_sizes = font_sizes[(font_sizes > min_extreme_font_size) & (font_sizes < max_extreme_font_size)]
|
||
if middle_font_sizes.size > 0:
|
||
middle_mean_font_size = np.mean(middle_font_sizes)
|
||
level_threshold = middle_mean_font_size
|
||
else:
|
||
level_threshold = mean_font_size
|
||
|
||
for tb in title_blocks:
|
||
title_block = tb["block"]
|
||
title_font_size = safe_get(title_block, "block_font_size", 0)
|
||
|
||
current_level = 1 # Initialize title level, the biggest level is 1
|
||
|
||
# print(f"Before adjustment by font size, {current_level}")
|
||
if title_font_size >= max_extreme_font_size:
|
||
current_level = 1
|
||
elif title_font_size <= min_extreme_font_size:
|
||
current_level = 3
|
||
elif float(title_font_size) >= float(level_threshold):
|
||
current_level = 2
|
||
else:
|
||
current_level = 3
|
||
# print(f"After adjustment by font size, {current_level}")
|
||
|
||
title_block["block_title_level"] = current_level
|
||
|
||
return title_blocks
|
||
|
||
def batch_process_blocks_recog_title_level(self, pdf_dic):
|
||
title_blocks = []
|
||
|
||
# Collect all titles
|
||
for page_id, blocks in pdf_dic.items():
|
||
if page_id.startswith("page_"):
|
||
para_blocks = blocks.get("para_blocks", [])
|
||
for block in para_blocks:
|
||
if block.get("is_block_title"):
|
||
title_obj = {"page_id": page_id, "block": block}
|
||
title_blocks.append(title_obj)
|
||
|
||
# Determine title level
|
||
if title_blocks:
|
||
# Determine title level based on font size
|
||
title_blocks = self.__determine_size_based_level(title_blocks)
|
||
|
||
return pdf_dic
|