Briq/camelot-py
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rename kwargs and add tests

Browse Source
This commit is contained in:
Vinayak Mehta
2018-12-21 15:09:37 +05:30
parent f6aa21c31f
commit 50b4468aff
10 changed files with 193 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
camelot/parsers/lattice.py
+15 -13
View File
@@ -47,16 +47,16 @@ class Lattice(BaseParser):
Direction in which text in a spanning cell will flow.
split_text : bool, optional (default: False)
Split text that spans across multiple cells.
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
flag_size : bool, optional (default: False)
Flag text based on font size. Useful to detect
super/subscripts. Adds <s></s> around flagged text.
line_close_tol : int, optional (default: 2)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
line_tol : int, optional (default: 2)
Tolerance parameter used to merge close vertical and horizontal
lines.
joint_close_tol : int, optional (default: 2)
joint_tol : int, optional (default: 2)
Tolerance parameter used to decide whether the detected lines
and points lie close to each other.
threshold_blocksize : int, optional (default: 15)
@@ -73,12 +73,14 @@ class Lattice(BaseParser):
Number of times for erosion/dilation is applied.
For more information, refer `OpenCV's dilate <https://docs.opencv.org/2.4/modules/imgproc/doc/filtering.html#dilate>`_.
resolution : int, optional (default: 300)
Resolution used for PDF to PNG conversion.
"""
def __init__(self, table_areas=None, process_background=False,
line_size_scaling=15, copy_text=None, shift_text=['l', 't'],
split_text=False, flag_size=False, strip_text='', line_close_tol=2,
joint_close_tol=2, threshold_blocksize=15, threshold_constant=-2,
split_text=False, flag_size=False, strip_text='', line_tol=2,
joint_tol=2, threshold_blocksize=15, threshold_constant=-2,
iterations=0, resolution=300, **kwargs):
self.table_areas = table_areas
self.process_background = process_background
@@ -88,8 +90,8 @@ class Lattice(BaseParser):
self.split_text = split_text
self.flag_size = flag_size
self.strip_text = strip_text
self.line_close_tol = line_close_tol
self.joint_close_tol = joint_close_tol
self.line_tol = line_tol
self.joint_tol = joint_tol
self.threshold_blocksize = threshold_blocksize
self.threshold_constant = threshold_constant
self.iterations = iterations
@@ -283,9 +285,9 @@ class Lattice(BaseParser):
rows.extend([tk[1], tk[3]])
# sort horizontal and vertical segments
cols = merge_close_lines(
sorted(cols), line_close_tol=self.line_close_tol)
sorted(cols), line_tol=self.line_tol)
rows = merge_close_lines(
sorted(rows, reverse=True), line_close_tol=self.line_close_tol)
sorted(rows, reverse=True), line_tol=self.line_tol)
# make grid using x and y coord of shortlisted rows and cols
cols = [(cols[i], cols[i + 1])
for i in range(0, len(cols) - 1)]
@@ -302,7 +304,7 @@ class Lattice(BaseParser):
table = Table(cols, rows)
# set table edges to True using ver+hor lines
table = table.set_edges(v_s, h_s, joint_close_tol=self.joint_close_tol)
table = table.set_edges(v_s, h_s, joint_tol=self.joint_tol)
# set table border edges to True
table = table.set_border()
# set spanning cells to True
@@ -315,7 +317,7 @@ class Lattice(BaseParser):
for t in self.t_bbox[direction]:
indices, error = get_table_index(
table, t, direction, split_text=self.split_text,
flag_size=self.flag_size)
flag_size=self.flag_size, strip_text=self.strip_text)
if indices[:2] != (-1, -1):
pos_errors.append(error)
indices = Lattice._reduce_index(table, indices, shift_text=self.shift_text)
camelot/parsers/stream.py
+27 -27
View File
@@ -35,34 +35,34 @@ class Stream(BaseParser):
are comma-separated.
split_text : bool, optional (default: False)
Split text that spans across multiple cells.
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
flag_size : bool, optional (default: False)
Flag text based on font size. Useful to detect
super/subscripts. Adds <s></s> around flagged text.
edge_close_tol : int, optional (default: 50)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
edge_tol : int, optional (default: 50)
Tolerance parameter for extending textedges vertically.
row_close_tol : int, optional (default: 2)
row_tol : int, optional (default: 2)
Tolerance parameter used to combine text vertically,
to generate rows.
col_close_tol : int, optional (default: 0)
column_tol : int, optional (default: 0)
Tolerance parameter used to combine text horizontally,
to generate columns.
"""
def __init__(self, table_areas=None, columns=None, split_text=False,
flag_size=False, strip_text='', edge_close_tol=50, row_close_tol=2,
col_close_tol=0, **kwargs):
flag_size=False, strip_text='', edge_tol=50, row_tol=2,
column_tol=0, **kwargs):
self.table_areas = table_areas
self.columns = columns
self._validate_columns()
self.split_text = split_text
self.flag_size = flag_size
self.strip_text = strip_text
self.edge_close_tol = edge_close_tol
self.row_close_tol = row_close_tol
self.col_close_tol = col_close_tol
self.edge_tol = edge_tol
self.row_tol = row_tol
self.column_tol = column_tol
@staticmethod
def _text_bbox(t_bbox):
@@ -88,7 +88,7 @@ class Stream(BaseParser):
return text_bbox
@staticmethod
def _group_rows(text, row_close_tol=2):
def _group_rows(text, row_tol=2):
"""Groups PDFMiner text objects into rows vertically
within a tolerance.
@@ -96,7 +96,7 @@ class Stream(BaseParser):
----------
text : list
List of PDFMiner text objects.
row_close_tol : int, optional (default: 2)
row_tol : int, optional (default: 2)
Returns
-------
@@ -112,7 +112,7 @@ class Stream(BaseParser):
# if t.get_text().strip() and all([obj.upright for obj in t._objs if
# type(obj) is LTChar]):
if t.get_text().strip():
if not np.isclose(row_y, t.y0, atol=row_close_tol):
if not np.isclose(row_y, t.y0, atol=row_tol):
rows.append(sorted(temp, key=lambda t: t.x0))
temp = []
row_y = t.y0
@@ -122,7 +122,7 @@ class Stream(BaseParser):
return rows
@staticmethod
def _merge_columns(l, col_close_tol=0):
def _merge_columns(l, column_tol=0):
"""Merges column boundaries horizontally if they overlap
or lie within a tolerance.
@@ -130,7 +130,7 @@ class Stream(BaseParser):
----------
l : list
List of column x-coordinate tuples.
col_close_tol : int, optional (default: 0)
column_tol : int, optional (default: 0)
Returns
-------
@@ -144,17 +144,17 @@ class Stream(BaseParser):
merged.append(higher)
else:
lower = merged[-1]
if col_close_tol >= 0:
if column_tol >= 0:
if (higher[0] <= lower[1] or
np.isclose(higher[0], lower[1], atol=col_close_tol)):
np.isclose(higher[0], lower[1], atol=column_tol)):
upper_bound = max(lower[1], higher[1])
lower_bound = min(lower[0], higher[0])
merged[-1] = (lower_bound, upper_bound)
else:
merged.append(higher)
elif col_close_tol < 0:
elif column_tol < 0:
if higher[0] <= lower[1]:
if np.isclose(higher[0], lower[1], atol=abs(col_close_tol)):
if np.isclose(higher[0], lower[1], atol=abs(column_tol)):
merged.append(higher)
else:
upper_bound = max(lower[1], higher[1])
@@ -191,7 +191,7 @@ class Stream(BaseParser):
return rows
@staticmethod
def _add_columns(cols, text, row_close_tol):
def _add_columns(cols, text, row_tol):
"""Adds columns to existing list by taking into account
the text that lies outside the current column x-coordinates.
@@ -210,7 +210,7 @@ class Stream(BaseParser):
"""
if text:
text = Stream._group_rows(text, row_close_tol=row_close_tol)
text = Stream._group_rows(text, row_tol=row_tol)
elements = [len(r) for r in text]
new_cols = [(t.x0, t.x1)
for r in text if len(r) == max(elements) for t in r]
@@ -259,7 +259,7 @@ class Stream(BaseParser):
# TODO: add support for arabic text #141
# sort textlines in reading order
textlines.sort(key=lambda x: (-x.y0, x.x0))
textedges = TextEdges(edge_close_tol=self.edge_close_tol)
textedges = TextEdges(edge_tol=self.edge_tol)
# generate left, middle and right textedges
textedges.generate(textlines)
# select relevant edges
@@ -301,7 +301,7 @@ class Stream(BaseParser):
self.t_bbox = t_bbox
text_x_min, text_y_min, text_x_max, text_y_max = self._text_bbox(self.t_bbox)
rows_grouped = self._group_rows(self.t_bbox['horizontal'], row_close_tol=self.row_close_tol)
rows_grouped = self._group_rows(self.t_bbox['horizontal'], row_tol=self.row_tol)
rows = self._join_rows(rows_grouped, text_y_max, text_y_min)
elements = [len(r) for r in rows_grouped]
@@ -332,7 +332,7 @@ class Stream(BaseParser):
warnings.warn("No tables found in table area {}".format(
table_idx + 1))
cols = [(t.x0, t.x1) for r in rows_grouped if len(r) == ncols for t in r]
cols = self._merge_columns(sorted(cols), col_close_tol=self.col_close_tol)
cols = self._merge_columns(sorted(cols), column_tol=self.column_tol)
inner_text = []
for i in range(1, len(cols)):
left = cols[i - 1][1]
@@ -344,7 +344,7 @@ class Stream(BaseParser):
for t in self.t_bbox[direction]
if t.x0 > cols[-1][1] or t.x1 < cols[0][0]]
inner_text.extend(outer_text)
cols = self._add_columns(cols, inner_text, self.row_close_tol)
cols = self._add_columns(cols, inner_text, self.row_tol)
cols = self._join_columns(cols, text_x_min, text_x_max)
return cols, rows
@@ -360,7 +360,7 @@ class Stream(BaseParser):
for t in self.t_bbox[direction]:
indices, error = get_table_index(
table, t, direction, split_text=self.split_text,
flag_size=self.flag_size)
flag_size=self.flag_size, strip_text=self.strip_text)
if indices[:2] != (-1, -1):
pos_errors.append(error)
for r_idx, c_idx, text in indices: