Briq/camelot-py
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add table regions support

Browse Source
This commit is contained in:
Vinayak Mehta
2019-01-04 19:17:54 +05:30
parent a5027e81c5
commit 03f301b25c
8 changed files with 100 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
camelot/parsers/lattice.py
+39 -16
View File
@@ -16,7 +16,7 @@ from ..utils import (scale_image, scale_pdf, segments_in_bbox, text_in_bbox,
merge_close_lines, get_table_index, compute_accuracy,
compute_whitespace)
from ..image_processing import (adaptive_threshold, find_lines,
find_table_contours, find_table_joints)
find_contours, find_joints)
logger = logging.getLogger('camelot')
@@ -28,13 +28,17 @@ class Lattice(BaseParser):
Parameters
----------
table_regions : list, optional (default: None)
List of page regions that may contain tables of the form x1,y1,x2,y2
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
in PDF coordinate space.
table_areas : list, optional (default: None)
List of table area strings of the form x1,y1,x2,y2
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
in PDF coordinate space.
process_background : bool, optional (default: False)
Process background lines.
line_size_scaling : int, optional (default: 15)
line_scale : int, optional (default: 15)
Line size scaling factor. The larger the value the smaller
the detected lines. Making it very large will lead to text
being detected as lines.
@@ -77,14 +81,15 @@ class Lattice(BaseParser):
Resolution used for PDF to PNG conversion.
"""
def __init__(self, table_areas=None, process_background=False,
line_size_scaling=15, copy_text=None, shift_text=['l', 't'],
def __init__(self, table_regions=None, table_areas=None, process_background=False,
line_scale=15, copy_text=None, shift_text=['l', 't'],
split_text=False, flag_size=False, strip_text='', line_tol=2,
joint_tol=2, threshold_blocksize=15, threshold_constant=-2,
iterations=0, resolution=300, **kwargs):
self.table_regions = table_regions
self.table_areas = table_areas
self.process_background = process_background
self.line_size_scaling = line_size_scaling
self.line_scale = line_scale
self.copy_text = copy_text
self.shift_text = shift_text
self.split_text = split_text
@@ -239,14 +244,35 @@ class Lattice(BaseParser):
image_scalers = (image_width_scaler, image_height_scaler, self.pdf_height)
pdf_scalers = (pdf_width_scaler, pdf_height_scaler, image_height)
vertical_mask, vertical_segments = find_lines(
self.threshold, direction='vertical',
line_size_scaling=self.line_size_scaling, iterations=self.iterations)
horizontal_mask, horizontal_segments = find_lines(
self.threshold, direction='horizontal',
line_size_scaling=self.line_size_scaling, iterations=self.iterations)
if self.table_areas is None:
regions = None
if self.table_regions is not None:
regions = []
for region in self.table_regions:
x1, y1, x2, y2 = region.split(",")
x1 = float(x1)
y1 = float(y1)
x2 = float(x2)
y2 = float(y2)
x1, y1, x2, y2 = scale_pdf((x1, y1, x2, y2), image_scalers)
regions.append((x1, y1, abs(x2 - x1), abs(y2 - y1)))
vertical_mask, vertical_segments = find_lines(
self.threshold, regions=regions, direction='vertical',
line_scale=self.line_scale, iterations=self.iterations)
horizontal_mask, horizontal_segments = find_lines(
self.threshold, regions=regions, direction='horizontal',
line_scale=self.line_scale, iterations=self.iterations)
contours = find_contours(vertical_mask, horizontal_mask)
table_bbox = find_joints(contours, vertical_mask, horizontal_mask)
else:
vertical_mask, vertical_segments = find_lines(
self.threshold, direction='vertical', line_scale=self.line_scale,
iterations=self.iterations)
horizontal_mask, horizontal_segments = find_lines(
self.threshold, direction='horizontal', line_scale=self.line_scale,
iterations=self.iterations)
if self.table_areas is not None:
areas = []
for area in self.table_areas:
x1, y1, x2, y2 = area.split(",")
@@ -256,10 +282,7 @@ class Lattice(BaseParser):
y2 = float(y2)
x1, y1, x2, y2 = scale_pdf((x1, y1, x2, y2), image_scalers)
areas.append((x1, y1, abs(x2 - x1), abs(y2 - y1)))
table_bbox = find_table_joints(areas, vertical_mask, horizontal_mask)
else:
contours = find_table_contours(vertical_mask, horizontal_mask)
table_bbox = find_table_joints(contours, vertical_mask, horizontal_mask)
table_bbox = find_joints(areas, vertical_mask, horizontal_mask)
self.table_bbox_unscaled = copy.deepcopy(table_bbox)
camelot/parsers/stream.py
+13 -5
View File
@@ -26,6 +26,10 @@ class Stream(BaseParser):
Parameters
----------
table_regions : list, optional (default: None)
List of page regions that may contain tables of the form x1,y1,x2,y2
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
in PDF coordinate space.
table_areas : list, optional (default: None)
List of table area strings of the form x1,y1,x2,y2
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
@@ -51,9 +55,10 @@ class Stream(BaseParser):
to generate columns.
"""
def __init__(self, table_areas=None, columns=None, split_text=False,
def __init__(self, table_regions=None, table_areas=None, columns=None, split_text=False,
flag_size=False, strip_text='', edge_tol=50, row_tol=2,
column_tol=0, **kwargs):
self.table_regions = table_regions
self.table_areas = table_areas
self.columns = columns
self._validate_columns()
@@ -275,7 +280,13 @@ class Stream(BaseParser):
def _generate_table_bbox(self):
self.textedges = []
if self.table_areas is not None:
if self.table_areas is None:
if self.table_regions is not None:
# filter horizontal text
pass
# find tables based on nurminen's detection algorithm
table_bbox = self._nurminen_table_detection(self.horizontal_text)
else:
table_bbox = {}
for area in self.table_areas:
x1, y1, x2, y2 = area.split(",")
@@ -284,9 +295,6 @@ class Stream(BaseParser):
x2 = float(x2)
y2 = float(y2)
table_bbox[(x1, y2, x2, y1)] = None
else:
# find tables based on nurminen's detection algorithm
table_bbox = self._nurminen_table_detection(self.horizontal_text)
self.table_bbox = table_bbox
def _generate_columns_and_rows(self, table_idx, tk):