Briq/camelot-py
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix merge conflict

Browse Source
This commit is contained in:
Vinayak Mehta
2018-12-22 11:07:24 +05:30
parent 736fb25b56 175ba32d38
commit 27fa226c71
20 changed files with 356 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
camelot/cli.py
+10 -4
View File
@@ -43,6 +43,8 @@ pass_config = click.make_pass_decorator(Config)
help='Split text that spans across multiple cells.')
@click.option('-flag', '--flag_size', is_flag=True, help='Flag text based on'
' font size. Useful to detect super/subscripts.')
@click.option('-strip', '--strip_text', help='Characters that should be stripped from a string before'
' assigning it to a cell.')
@click.option('-M', '--margins', nargs=3, default=(1.0, 0.5, 0.1),
help='PDFMiner char_margin, line_margin and word_margin.')
@click.pass_context
@@ -68,10 +70,10 @@ def cli(ctx, *args, **kwargs):
@click.option('-shift', '--shift_text', default=['l', 't'],
type=click.Choice(['', 'l', 'r', 't', 'b']), multiple=True,
help='Direction in which text in a spanning cell will flow.')
@click.option('-l', '--line_close_tol', default=2,
@click.option('-l', '--line_tol', default=2,
help='Tolerance parameter used to merge close vertical'
' and horizontal lines.')
@click.option('-j', '--joint_close_tol', default=2,
@click.option('-j', '--joint_tol', default=2,
help='Tolerance parameter used to decide whether'
' the detected lines and points lie close to each other.')
@click.option('-block', '--threshold_blocksize', default=15,
@@ -84,6 +86,8 @@ def cli(ctx, *args, **kwargs):
' may be zero or negative as well.')
@click.option('-I', '--iterations', default=0,
help='Number of times for erosion/dilation will be applied.')
@click.option('-res', '--resolution', default=300,
help='Resolution used for PDF to PNG conversion.')
@click.option('-plot', '--plot_type',
type=click.Choice(['text', 'grid', 'contour', 'joint', 'line']),
help='Plot elements found on PDF page for visual debugging.')
@@ -133,9 +137,11 @@ def lattice(c, *args, **kwargs):
' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
@click.option('-C', '--columns', default=[], multiple=True,
help='X coordinates of column separators.')
@click.option('-r', '--row_close_tol', default=2, help='Tolerance parameter'
@click.option('-e', '--edge_tol', default=50, help='Tolerance parameter'
' for extending textedges vertically.')
@click.option('-r', '--row_tol', default=2, help='Tolerance parameter'
' used to combine text vertically, to generate rows.')
@click.option('-c', '--col_close_tol', default=0, help='Tolerance parameter'
@click.option('-c', '--column_tol', default=0, help='Tolerance parameter'
' used to combine text horizontally, to generate columns.')
@click.option('-plot', '--plot_type',
type=click.Choice(['text', 'grid', 'contour', 'textedge']),
camelot/core.py
+13 -13
View File
@@ -13,8 +13,6 @@ import pandas as pd
# minimum number of vertical textline intersections for a textedge
# to be considered valid
TEXTEDGE_REQUIRED_ELEMENTS = 4
# y coordinate tolerance for extending textedge
TEXTEDGE_EXTEND_TOLERANCE = 50
# padding added to table area on the left, right and bottom
TABLE_AREA_PADDING = 10
@@ -55,11 +53,11 @@ class TextEdge(object):
return '<TextEdge x={} y0={} y1={} align={} valid={}>'.format(
round(self.x, 2), round(self.y0, 2), round(self.y1, 2), self.align, self.is_valid)
def update_coords(self, x, y0):
def update_coords(self, x, y0, edge_tol=50):
"""Updates the text edge's x and bottom y coordinates and sets
the is_valid attribute.
"""
if np.isclose(self.y0, y0, atol=TEXTEDGE_EXTEND_TOLERANCE):
if np.isclose(self.y0, y0, atol=edge_tol):
self.x = (self.intersections * self.x + x) / float(self.intersections + 1)
self.y0 = y0
self.intersections += 1
@@ -74,7 +72,8 @@ class TextEdges(object):
the PDF page. The dict has three keys based on the alignments,
and each key's value is a list of camelot.core.TextEdge objects.
"""
def __init__(self):
def __init__(self, edge_tol=50):
self.edge_tol = edge_tol
self._textedges = {'left': [], 'right': [], 'middle': []}
@staticmethod
@@ -115,7 +114,8 @@ class TextEdges(object):
if idx is None:
self.add(textline, align)
else:
self._textedges[align][idx].update_coords(x_coord, textline.y0)
self._textedges[align][idx].update_coords(
x_coord, textline.y0, edge_tol=self.edge_tol)
def generate(self, textlines):
"""Generates the text edges dict based on horizontal text
@@ -359,7 +359,7 @@ class Table(object):
cell.left = cell.right = cell.top = cell.bottom = True
return self
def set_edges(self, vertical, horizontal, joint_close_tol=2):
def set_edges(self, vertical, horizontal, joint_tol=2):
"""Sets a cell's edges to True depending on whether the cell's
coordinates overlap with the line's coordinates within a
tolerance.
@@ -376,11 +376,11 @@ class Table(object):
# find closest x coord
# iterate over y coords and find closest start and end points
i = [i for i, t in enumerate(self.cols)
if np.isclose(v[0], t[0], atol=joint_close_tol)]
if np.isclose(v[0], t[0], atol=joint_tol)]
j = [j for j, t in enumerate(self.rows)
if np.isclose(v[3], t[0], atol=joint_close_tol)]
if np.isclose(v[3], t[0], atol=joint_tol)]
k = [k for k, t in enumerate(self.rows)
if np.isclose(v[1], t[0], atol=joint_close_tol)]
if np.isclose(v[1], t[0], atol=joint_tol)]
if not j:
continue
J = j[0]
@@ -427,11 +427,11 @@ class Table(object):
# find closest y coord
# iterate over x coords and find closest start and end points
i = [i for i, t in enumerate(self.rows)
if np.isclose(h[1], t[0], atol=joint_close_tol)]
if np.isclose(h[1], t[0], atol=joint_tol)]
j = [j for j, t in enumerate(self.cols)
if np.isclose(h[0], t[0], atol=joint_close_tol)]
if np.isclose(h[0], t[0], atol=joint_tol)]
k = [k for k, t in enumerate(self.cols)
if np.isclose(h[2], t[0], atol=joint_close_tol)]
if np.isclose(h[2], t[0], atol=joint_tol)]
if not j:
continue
J = j[0]
camelot/handlers.py
+5 -2
View File
@@ -125,7 +125,7 @@ class PDFHandler(object):
with open(fpath, 'wb') as f:
outfile.write(f)
def parse(self, flavor='lattice', suppress_stdout=False, **kwargs):
def parse(self, flavor='lattice', suppress_stdout=False, layout_kwargs={}, **kwargs):
"""Extracts tables by calling parser.get_tables on all single
page PDFs.
@@ -136,6 +136,8 @@ class PDFHandler(object):
Lattice is used by default.
suppress_stdout : str (default: False)
Suppress logs and warnings.
layout_kwargs : dict, optional (default: {})
A dict of `pdfminer.layout.LAParams <https://github.com/euske/pdfminer/blob/master/pdfminer/layout.py#L33>`_ kwargs.
kwargs : dict
See camelot.read_pdf kwargs.
@@ -153,6 +155,7 @@ class PDFHandler(object):
for p in self.pages]
parser = Lattice(**kwargs) if flavor == 'lattice' else Stream(**kwargs)
for p in pages:
t = parser.extract_tables(p, suppress_stdout=suppress_stdout)
t = parser.extract_tables(p, suppress_stdout=suppress_stdout,
layout_kwargs=layout_kwargs)
tables.extend(t)
return TableList(tables)
camelot/io.py
+14 -10
View File
@@ -6,7 +6,7 @@ from .utils import validate_input, remove_extra
def read_pdf(filepath, pages='1', password=None, flavor='lattice',
suppress_stdout=False, **kwargs):
suppress_stdout=False, layout_kwargs={}, **kwargs):
"""Read PDF and return extracted tables.
Note: kwargs annotated with ^ can only be used with flavor='stream'
@@ -26,6 +26,8 @@ def read_pdf(filepath, pages='1', password=None, flavor='lattice',
Lattice is used by default.
suppress_stdout : bool, optional (default: True)
Print all logs and warnings.
layout_kwargs : dict, optional (default: {})
A dict of `pdfminer.layout.LAParams <https://github.com/euske/pdfminer/blob/master/pdfminer/layout.py#L33>`_ kwargs.
table_areas : list, optional (default: None)
List of table area strings of the form x1,y1,x2,y2
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
@@ -38,10 +40,13 @@ def read_pdf(filepath, pages='1', password=None, flavor='lattice',
flag_size : bool, optional (default: False)
Flag text based on font size. Useful to detect
super/subscripts. Adds <s></s> around flagged text.
row_close_tol^ : int, optional (default: 2)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
row_tol^ : int, optional (default: 2)
Tolerance parameter used to combine text vertically,
to generate rows.
col_close_tol^ : int, optional (default: 0)
column_tol^ : int, optional (default: 0)
Tolerance parameter used to combine text horizontally,
to generate columns.
process_background* : bool, optional (default: False)
@@ -57,10 +62,10 @@ def read_pdf(filepath, pages='1', password=None, flavor='lattice',
shift_text* : list, optional (default: ['l', 't'])
{'l', 'r', 't', 'b'}
Direction in which text in a spanning cell will flow.
line_close_tol* : int, optional (default: 2)
line_tol* : int, optional (default: 2)
Tolerance parameter used to merge close vertical and horizontal
lines.
joint_close_tol* : int, optional (default: 2)
joint_tol* : int, optional (default: 2)
Tolerance parameter used to decide whether the detected lines
and points lie close to each other.
threshold_blocksize* : int, optional (default: 15)
@@ -77,10 +82,8 @@ def read_pdf(filepath, pages='1', password=None, flavor='lattice',
Number of times for erosion/dilation is applied.
For more information, refer `OpenCV's dilate <https://docs.opencv.org/2.4/modules/imgproc/doc/filtering.html#dilate>`_.
margins : tuple
PDFMiner char_margin, line_margin and word_margin.
For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
resolution* : int, optional (default: 300)
Resolution used for PDF to PNG conversion.
Returns
-------
@@ -98,5 +101,6 @@ def read_pdf(filepath, pages='1', password=None, flavor='lattice',
validate_input(kwargs, flavor=flavor)
p = PDFHandler(filepath, pages=pages, password=password)
kwargs = remove_extra(kwargs, flavor=flavor)
tables = p.parse(flavor=flavor, suppress_stdout=suppress_stdout, **kwargs)
tables = p.parse(flavor=flavor, suppress_stdout=suppress_stdout,
layout_kwargs=layout_kwargs, **kwargs)
return tables
camelot/parsers/base.py
+3 -5
View File
@@ -8,13 +8,11 @@ from ..utils import get_page_layout, get_text_objects
class BaseParser(object):
"""Defines a base parser.
"""
def _generate_layout(self, filename):
def _generate_layout(self, filename, layout_kwargs):
self.filename = filename
self.layout_kwargs = layout_kwargs
self.layout, self.dimensions = get_page_layout(
self.filename,
char_margin=self.char_margin,
line_margin=self.line_margin,
word_margin=self.word_margin)
filename, **layout_kwargs)
self.horizontal_text = get_text_objects(self.layout, ltype="lh")
self.vertical_text = get_text_objects(self.layout, ltype="lv")
self.pdf_width, self.pdf_height = self.dimensions
camelot/parsers/lattice.py
+20 -18
View File
@@ -53,10 +53,13 @@ class Lattice(BaseParser):
flag_size : bool, optional (default: False)
Flag text based on font size. Useful to detect
super/subscripts. Adds <s></s> around flagged text.
line_close_tol : int, optional (default: 2)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
line_tol : int, optional (default: 2)
Tolerance parameter used to merge close vertical and horizontal
lines.
joint_close_tol : int, optional (default: 2)
joint_tol : int, optional (default: 2)
Tolerance parameter used to decide whether the detected lines
and points lie close to each other.
threshold_blocksize : int, optional (default: 15)
@@ -73,17 +76,15 @@ class Lattice(BaseParser):
Number of times for erosion/dilation is applied.
For more information, refer `OpenCV's dilate <https://docs.opencv.org/2.4/modules/imgproc/doc/filtering.html#dilate>`_.
margins : tuple
PDFMiner char_margin, line_margin and word_margin.
For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
resolution : int, optional (default: 300)
Resolution used for PDF to PNG conversion.
"""
def __init__(self, table_areas=None, process_background=False,
line_size_scaling=15, copy_text=None, shift_text=['l', 't'],
split_text=False, flag_size=False, line_close_tol=2,
joint_close_tol=2, threshold_blocksize=15, threshold_constant=-2,
iterations=0, margins=(1.0, 0.5, 0.1), **kwargs):
split_text=False, flag_size=False, strip_text='', line_tol=2,
joint_tol=2, threshold_blocksize=15, threshold_constant=-2,
iterations=0, resolution=300, **kwargs):
self.table_areas = table_areas
self.process_background = process_background
self.line_size_scaling = line_size_scaling
@@ -91,12 +92,13 @@ class Lattice(BaseParser):
self.shift_text = shift_text
self.split_text = split_text
self.flag_size = flag_size
self.line_close_tol = line_close_tol
self.joint_close_tol = joint_close_tol
self.strip_text = strip_text
self.line_tol = line_tol
self.joint_tol = joint_tol
self.threshold_blocksize = threshold_blocksize
self.threshold_constant = threshold_constant
self.iterations = iterations
self.char_margin, self.line_margin, self.word_margin = margins
self.resolution = resolution
@staticmethod
def _reduce_index(t, idx, shift_text):
@@ -245,9 +247,9 @@ class Lattice(BaseParser):
rows.extend([tk[1], tk[3]])
# sort horizontal and vertical segments
cols = merge_close_lines(
sorted(cols), line_close_tol=self.line_close_tol)
sorted(cols), line_tol=self.line_tol)
rows = merge_close_lines(
sorted(rows, reverse=True), line_close_tol=self.line_close_tol)
sorted(rows, reverse=True), line_tol=self.line_tol)
# make grid using x and y coord of shortlisted rows and cols
cols = [(cols[i], cols[i + 1])
for i in range(0, len(cols) - 1)]
@@ -264,7 +266,7 @@ class Lattice(BaseParser):
table = Table(cols, rows)
# set table edges to True using ver+hor lines
table = table.set_edges(v_s, h_s, joint_close_tol=self.joint_close_tol)
table = table.set_edges(v_s, h_s, joint_tol=self.joint_tol)
# set table border edges to True
table = table.set_border()
# set spanning cells to True
@@ -277,7 +279,7 @@ class Lattice(BaseParser):
for t in self.t_bbox[direction]:
indices, error = get_table_index(
table, t, direction, split_text=self.split_text,
flag_size=self.flag_size)
flag_size=self.flag_size, strip_text=self.strip_text)
if indices[:2] != (-1, -1):
pos_errors.append(error)
indices = Lattice._reduce_index(table, indices, shift_text=self.shift_text)
@@ -310,8 +312,8 @@ class Lattice(BaseParser):
return table
def extract_tables(self, filename, suppress_stdout=False):
self._generate_layout(filename)
def extract_tables(self, filename, suppress_stdout=False, layout_kwargs={}):
self._generate_layout(filename, layout_kwargs)
if not suppress_stdout:
logger.info('Processing {}'.format(os.path.basename(self.rootname)))
camelot/parsers/stream.py
+31 -30
View File
@@ -38,29 +38,31 @@ class Stream(BaseParser):
flag_size : bool, optional (default: False)
Flag text based on font size. Useful to detect
super/subscripts. Adds <s></s> around flagged text.
row_close_tol : int, optional (default: 2)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
edge_tol : int, optional (default: 50)
Tolerance parameter for extending textedges vertically.
row_tol : int, optional (default: 2)
Tolerance parameter used to combine text vertically,
to generate rows.
col_close_tol : int, optional (default: 0)
column_tol : int, optional (default: 0)
Tolerance parameter used to combine text horizontally,
to generate columns.
margins : tuple, optional (default: (1.0, 0.5, 0.1))
PDFMiner char_margin, line_margin and word_margin.
For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
"""
def __init__(self, table_areas=None, columns=None, split_text=False,
flag_size=False, row_close_tol=2, col_close_tol=0,
margins=(1.0, 0.5, 0.1), **kwargs):
flag_size=False, strip_text='', edge_tol=50, row_tol=2,
column_tol=0, **kwargs):
self.table_areas = table_areas
self.columns = columns
self._validate_columns()
self.split_text = split_text
self.flag_size = flag_size
self.row_close_tol = row_close_tol
self.col_close_tol = col_close_tol
self.char_margin, self.line_margin, self.word_margin = margins
self.strip_text = strip_text
self.edge_tol = edge_tol
self.row_tol = row_tol
self.column_tol = column_tol
@staticmethod
def _text_bbox(t_bbox):
@@ -86,7 +88,7 @@ class Stream(BaseParser):
return text_bbox
@staticmethod
def _group_rows(text, row_close_tol=2):
def _group_rows(text, row_tol=2):
"""Groups PDFMiner text objects into rows vertically
within a tolerance.
@@ -94,7 +96,7 @@ class Stream(BaseParser):
----------
text : list
List of PDFMiner text objects.
row_close_tol : int, optional (default: 2)
row_tol : int, optional (default: 2)
Returns
-------
@@ -110,7 +112,7 @@ class Stream(BaseParser):
# if t.get_text().strip() and all([obj.upright for obj in t._objs if
# type(obj) is LTChar]):
if t.get_text().strip():
if not np.isclose(row_y, t.y0, atol=row_close_tol):
if not np.isclose(row_y, t.y0, atol=row_tol):
rows.append(sorted(temp, key=lambda t: t.x0))
temp = []
row_y = t.y0
@@ -120,7 +122,7 @@ class Stream(BaseParser):
return rows
@staticmethod
def _merge_columns(l, col_close_tol=0):
def _merge_columns(l, column_tol=0):
"""Merges column boundaries horizontally if they overlap
or lie within a tolerance.
@@ -128,7 +130,7 @@ class Stream(BaseParser):
----------
l : list
List of column x-coordinate tuples.
col_close_tol : int, optional (default: 0)
column_tol : int, optional (default: 0)
Returns
-------
@@ -142,17 +144,17 @@ class Stream(BaseParser):
merged.append(higher)
else:
lower = merged[-1]
if col_close_tol >= 0:
if column_tol >= 0:
if (higher[0] <= lower[1] or
np.isclose(higher[0], lower[1], atol=col_close_tol)):
np.isclose(higher[0], lower[1], atol=column_tol)):
upper_bound = max(lower[1], higher[1])
lower_bound = min(lower[0], higher[0])
merged[-1] = (lower_bound, upper_bound)
else:
merged.append(higher)
elif col_close_tol < 0:
elif column_tol < 0:
if higher[0] <= lower[1]:
if np.isclose(higher[0], lower[1], atol=abs(col_close_tol)):
if np.isclose(higher[0], lower[1], atol=abs(column_tol)):
merged.append(higher)
else:
upper_bound = max(lower[1], higher[1])
@@ -189,7 +191,7 @@ class Stream(BaseParser):
return rows
@staticmethod
def _add_columns(cols, text, row_close_tol):
def _add_columns(cols, text, row_tol):
"""Adds columns to existing list by taking into account
the text that lies outside the current column x-coordinates.
@@ -208,7 +210,7 @@ class Stream(BaseParser):
"""
if text:
text = Stream._group_rows(text, row_close_tol=row_close_tol)
text = Stream._group_rows(text, row_tol=row_tol)
elements = [len(r) for r in text]
new_cols = [(t.x0, t.x1)
for r in text if len(r) == max(elements) for t in r]
@@ -254,11 +256,10 @@ class Stream(BaseParser):
Assumes that tables are situated relatively far apart
vertically.
"""
# TODO: add support for arabic text #141
# sort textlines in reading order
textlines.sort(key=lambda x: (-x.y0, x.x0))
textedges = TextEdges()
textedges = TextEdges(edge_tol=self.edge_tol)
# generate left, middle and right textedges
textedges.generate(textlines)
# select relevant edges
@@ -300,7 +301,7 @@ class Stream(BaseParser):
self.t_bbox = t_bbox
text_x_min, text_y_min, text_x_max, text_y_max = self._text_bbox(self.t_bbox)
rows_grouped = self._group_rows(self.t_bbox['horizontal'], row_close_tol=self.row_close_tol)
rows_grouped = self._group_rows(self.t_bbox['horizontal'], row_tol=self.row_tol)
rows = self._join_rows(rows_grouped, text_y_max, text_y_min)
elements = [len(r) for r in rows_grouped]
@@ -331,7 +332,7 @@ class Stream(BaseParser):
warnings.warn("No tables found in table area {}".format(
table_idx + 1))
cols = [(t.x0, t.x1) for r in rows_grouped if len(r) == ncols for t in r]
cols = self._merge_columns(sorted(cols), col_close_tol=self.col_close_tol)
cols = self._merge_columns(sorted(cols), column_tol=self.column_tol)
inner_text = []
for i in range(1, len(cols)):
left = cols[i - 1][1]
@@ -343,7 +344,7 @@ class Stream(BaseParser):
for t in self.t_bbox[direction]
if t.x0 > cols[-1][1] or t.x1 < cols[0][0]]
inner_text.extend(outer_text)
cols = self._add_columns(cols, inner_text, self.row_close_tol)
cols = self._add_columns(cols, inner_text, self.row_tol)
cols = self._join_columns(cols, text_x_min, text_x_max)
return cols, rows
@@ -359,7 +360,7 @@ class Stream(BaseParser):
for t in self.t_bbox[direction]:
indices, error = get_table_index(
table, t, direction, split_text=self.split_text,
flag_size=self.flag_size)
flag_size=self.flag_size, strip_text=self.strip_text)
if indices[:2] != (-1, -1):
pos_errors.append(error)
for r_idx, c_idx, text in indices:
@@ -388,8 +389,8 @@ class Stream(BaseParser):
return table
def extract_tables(self, filename, suppress_stdout=False):
self._generate_layout(filename)
def extract_tables(self, filename, suppress_stdout=False, layout_kwargs={}):
self._generate_layout(filename, layout_kwargs)
if not suppress_stdout:
logger.info('Processing {}'.format(os.path.basename(self.rootname)))
camelot/utils.py
+33 -18
View File
@@ -20,16 +20,16 @@ from pdfminer.layout import (LAParams, LTAnno, LTChar, LTTextLineHorizontal,
stream_kwargs = [
'columns',
'row_close_tol',
'col_close_tol'
'row_tol',
'column_tol'
]
lattice_kwargs = [
'process_background',
'line_size_scaling',
'copy_text',
'shift_text',
'line_close_tol',
'joint_close_tol',
'line_tol',
'joint_tol',
'threshold_blocksize',
'threshold_constant',
'iterations'
@@ -281,14 +281,14 @@ def text_in_bbox(bbox, text):
return t_bbox
def merge_close_lines(ar, line_close_tol=2):
def merge_close_lines(ar, line_tol=2):
"""Merges lines which are within a tolerance by calculating a
moving mean, based on their x or y axis projections.
Parameters
----------
ar : list
line_close_tol : int, optional (default: 2)
line_tol : int, optional (default: 2)
Returns
-------
@@ -301,7 +301,7 @@ def merge_close_lines(ar, line_close_tol=2):
ret.append(a)
else:
temp = ret[-1]
if np.isclose(temp, a, atol=line_close_tol):
if np.isclose(temp, a, atol=line_tol):
temp = (temp + a) / 2.0
ret[-1] = temp
else:
@@ -309,7 +309,12 @@ def merge_close_lines(ar, line_close_tol=2):
return ret
def flag_font_size(textline, direction):
# TODO: combine the following functions into a TextProcessor class which
# applies corresponding transformations sequentially
# (inspired from sklearn.pipeline.Pipeline)
def flag_font_size(textline, direction, strip_text=''):
"""Flags super/subscripts in text by enclosing them with <s></s>.
May give false positives.
@@ -319,6 +324,9 @@ def flag_font_size(textline, direction):
List of PDFMiner LTChar objects.
direction : string
Direction of the PDFMiner LTTextLine object.
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
Returns
-------
@@ -344,13 +352,13 @@ def flag_font_size(textline, direction):
fchars = [t[0] for t in chars]
if ''.join(fchars).strip():
flist.append(''.join(fchars))
fstring = ''.join(flist)
fstring = ''.join(flist).strip(strip_text)
else:
fstring = ''.join([t.get_text() for t in textline])
fstring = ''.join([t.get_text() for t in textline]).strip(strip_text)
return fstring
def split_textline(table, textline, direction, flag_size=False):
def split_textline(table, textline, direction, flag_size=False, strip_text=''):
"""Splits PDFMiner LTTextLine into substrings if it spans across
multiple rows/columns.
@@ -365,6 +373,9 @@ def split_textline(table, textline, direction, flag_size=False):
Whether or not to highlight a substring using <s></s>
if its size is different from rest of the string. (Useful for
super and subscripts.)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
Returns
-------
@@ -416,14 +427,15 @@ def split_textline(table, textline, direction, flag_size=False):
grouped_chars = []
for key, chars in groupby(cut_text, itemgetter(0, 1)):
if flag_size:
grouped_chars.append((key[0], key[1], flag_font_size([t[2] for t in chars], direction)))
grouped_chars.append((key[0], key[1],
flag_font_size([t[2] for t in chars], direction, strip_text=strip_text)))
else:
gchars = [t[2].get_text() for t in chars]
grouped_chars.append((key[0], key[1], ''.join(gchars)))
grouped_chars.append((key[0], key[1], ''.join(gchars).strip(strip_text)))
return grouped_chars
def get_table_index(table, t, direction, split_text=False, flag_size=False):
def get_table_index(table, t, direction, split_text=False, flag_size=False, strip_text='',):
"""Gets indices of the table cell where given text object lies by
comparing their y and x-coordinates.
@@ -441,6 +453,9 @@ def get_table_index(table, t, direction, split_text=False, flag_size=False):
Whether or not to highlight a substring using <s></s>
if its size is different from rest of the string. (Useful for
super and subscripts)
strip_text : str, optional (default: '')
Characters that should be stripped from a string before
assigning it to a cell.
Returns
-------
@@ -495,12 +510,12 @@ def get_table_index(table, t, direction, split_text=False, flag_size=False):
error = ((X * (y0_offset + y1_offset)) + (Y * (x0_offset + x1_offset))) / charea
if split_text:
return split_textline(table, t, direction, flag_size=flag_size), error
return split_textline(table, t, direction, flag_size=flag_size, strip_text=strip_text), error
else:
if flag_size:
return [(r_idx, c_idx, flag_font_size(t._objs, direction))], error
return [(r_idx, c_idx, flag_font_size(t._objs, direction, strip_text=strip_text))], error
else:
return [(r_idx, c_idx, t.get_text())], error
return [(r_idx, c_idx, t.get_text().strip(strip_text))], error
def compute_accuracy(error_weights):
@@ -558,7 +573,7 @@ def compute_whitespace(d):
def get_page_layout(filename, char_margin=1.0, line_margin=0.5, word_margin=0.1,
detect_vertical=True, all_texts=True):
detect_vertical=True, all_texts=True):
"""Returns a PDFMiner LTPage object and page dimension of a single
page pdf. See https://euske.github.io/pdfminer/ to get definitions
of kwargs.