Briq/camelot-py
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support for vertical tables in Stream

Browse Source 
* Change var names

* Add test pdf

* Add tests for Lattice rotation

* Add support for vertical tables in Stream, test pdfs

* Add tests for Stream rotation
This commit is contained in:
Vinayak Mehta
2016-09-15 20:51:59 +05:30
committed by GitHub
parent 8ce7b74671
commit 79afb45e2e
11 changed files with 222 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
camelot/lattice.py
+18 -11
View File
@@ -9,9 +9,10 @@ from .imgproc import (adaptive_threshold, find_lines, find_table_contours,
find_table_joints)
from .table import Table
from .utils import (scale_to_pdf, scale_to_image, segments_bbox, text_bbox,
detect_vertical, merge_close_values, get_row_index,
get_rotation, merge_close_values, get_row_index,
get_column_index, get_score, reduce_index, outline,
fill_spanning, count_empty, encode_list, pdf_to_text)
fill_spanning, count_empty, encode_list, get_page_layout,
get_text_objects)
__all__ = ['Lattice']
@@ -62,7 +63,7 @@ class Lattice:
page as value.
"""
def __init__(self, table_area=None, fill=None, mtol=[2], scale=15,
invert=False, margins=(2.0, 0.5, 0.1), debug=None):
invert=False, margins=(1.0, 0.5, 0.1), debug=None):
self.method = 'lattice'
self.table_area = table_area
@@ -82,10 +83,14 @@ class Lattice:
Dictionary with page number as key and list of tables on that
page as value.
"""
text, __, width, height = pdf_to_text(pdfname, self.char_margin,
self.line_margin, self.word_margin)
layout, dim = get_page_layout(pdfname, char_margin=self.char_margin,
line_margin=self.line_margin, word_margin=self.word_margin)
ltchar = get_text_objects(layout, LTType="char")
lttextlh = get_text_objects(layout, LTType="lh")
lttextlv = get_text_objects(layout, LTType="lv")
width, height = dim
bname, __ = os.path.splitext(pdfname)
if not text:
if not ltchar:
logging.warning("{0}: PDF has no text. It may be an image.".format(
os.path.basename(bname)))
return None
@@ -156,9 +161,11 @@ class Lattice:
# select elements which lie within table_bbox
table_data = {}
v_s, h_s = segments_bbox(k, v_segments, h_segments)
t_bbox = text_bbox(k, text)
table_data['text_p'] = 100 * (1 - (len(t_bbox) / len(text)))
table_rotation = detect_vertical(t_bbox)
char_bbox = text_bbox(k, ltchar)
lh_bbox = text_bbox(k, lttextlh)
lv_bbox = text_bbox(k, lttextlv)
table_data['text_p'] = 100 * (1 - (len(char_bbox) / len(ltchar)))
table_rotation = get_rotation(char_bbox, lh_bbox, lv_bbox)
cols, rows = zip(*table_bbox[k])
cols, rows = list(cols), list(rows)
cols.extend([k[0], k[2]])
@@ -187,7 +194,7 @@ class Lattice:
rerror = []
cerror = []
for t in text:
for t in char_bbox:
try:
r_idx, rass_error = get_row_index(t, rows)
except TypeError:
@@ -207,7 +214,7 @@ class Lattice:
for j in range(len(table.cells[i])):
t_bbox = table.cells[i][j].get_objects()
try:
cell_rotation = detect_vertical(t_bbox)
cell_rotation = get_rotation(t_bbox)
except ZeroDivisionError:
cell_rotation = ''
pass
camelot/stream.py
+46 -14
View File
@@ -7,8 +7,8 @@ import copy_reg
import numpy as np
from .table import Table
from .utils import (get_row_index, get_score, count_empty, encode_list,
pdf_to_text, text_bbox)
from .utils import (rotate, get_row_index, get_score, count_empty, encode_list,
get_page_layout, get_text_objects, text_bbox, get_rotation)
__all__ = ['Stream']
@@ -199,7 +199,7 @@ class Stream:
page as value.
"""
def __init__(self, table_area=None, columns=None, ncolumns=None, ytol=[2],
mtol=[2], margins=(2.0, 0.5, 0.1), debug=False):
mtol=[0], margins=(1.0, 0.5, 0.1), debug=False):
self.method = 'stream'
self.table_area = table_area
@@ -219,17 +219,20 @@ class Stream:
Dictionary with page number as key and list of tables on that
page as value.
"""
__, text, width, height = pdf_to_text(pdfname, self.char_margin,
self.line_margin, self.word_margin)
layout, dim = get_page_layout(pdfname, char_margin=self.char_margin,
line_margin=self.line_margin, word_margin=self.word_margin)
ltchar = get_text_objects(layout, LTType="char")
lttextlh = get_text_objects(layout, LTType="lh")
lttextlv = get_text_objects(layout, LTType="lv")
width, height = dim
bname, __ = os.path.splitext(pdfname)
if not text:
if not lttextlh:
logging.warning("{0}: PDF has no text. It may be an image.".format(
os.path.basename(bname)))
return None
if self.debug:
self.debug_text = [(t.x0, t.y0, t.x1, t.y1) for t in text]
return None
self.debug_text = []
if self.table_area is not None:
if self.columns is not None:
@@ -261,11 +264,35 @@ class Stream:
for k in sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True):
# select elements which lie within table_bbox
table_data = {}
t_bbox = text_bbox(k, text)
table_rotation = get_rotation(ltchar, lttextlh, lttextlv)
if table_rotation != '':
t_bbox = text_bbox(k, lttextlv)
if table_rotation == 'left':
if self.debug:
self.debug_text.extend([(t.x0, t.y0, t.x1, t.y1) for t in lttextlv])
for t in t_bbox:
x0, y0, x1, y1 = t.bbox
x0, y0 = rotate(0, 0, x0, y0, -np.pi / 2)
x1, y1 = rotate(0, 0, x1, y1, -np.pi / 2)
t.set_bbox((x0, y1, x1, y0))
elif table_rotation == 'right':
for t in t_bbox:
x0, y0, x1, y1 = t.bbox
x0, y0 = rotate(0, 0, x0, y0, np.pi / 2)
x1, y1 = rotate(0, 0, x1, y1, np.pi / 2)
t.set_bbox((x1, y0, x0, y1))
else:
if self.debug:
self.debug_text.extend([(t.x0, t.y0, t.x1, t.y1) for t in lttextlh])
t_bbox = text_bbox(k, lttextlh)
t_bbox.sort(key=lambda x: (-x.y0, x.x0))
text_x_min = min([t.x0 for t in t_bbox])
text_y_min = min([t.y0 for t in t_bbox])
text_x_max = max([t.x1 for t in t_bbox])
text_y_max = max([t.y1 for t in t_bbox])
rows_grouped = _group_rows(t_bbox, ytol=self.ytol[table_no])
rows = _join_rows(rows_grouped, k[3], k[1])
rows = _join_rows(rows_grouped, text_y_max, text_y_min)
elements = [len(r) for r in rows_grouped]
guess = False
@@ -275,8 +302,13 @@ class Stream:
# similar to else condition
# len can't be 1
cols = self.columns[table_no].split(',')
cols = [(float(cols[i]), float(cols[i + 1]))
for i in range(0, len(cols) - 1)]
cols = [float(c) for c in cols]
if table_rotation != '':
if table_rotation == 'left':
cols = [rotate(0, 0, 0, c, -np.pi / 2)[0] for c in cols]
elif table_rotation == 'right':
cols = [rotate(0, 0, 0, c, np.pi / 2)[0] for c in cols]
cols = [(cols[i], cols[i + 1]) for i in range(0, len(cols) - 1)]
else:
if self.ncolumns is not None and self.ncolumns[table_no] != -1:
ncols = self.ncolumns[table_no]
@@ -288,7 +320,7 @@ class Stream:
" isn't the same as what you specified."
" Change the value of mtol.".format(
os.path.basename(bname)))
cols = _join_columns(cols, k[0], k[2])
cols = _join_columns(cols, text_x_min, text_x_max)
else:
guess = True
ncols = max(set(elements), key=elements.count)
@@ -310,7 +342,7 @@ class Stream:
outer_text = [t for t in t_bbox if t.x0 > cols[-1][1] or t.x1 < cols[0][0]]
inner_text.extend(outer_text)
cols = _add_columns(cols, inner_text, self.ytol[table_no])
cols = _join_columns(cols, k[0], k[2])
cols = _join_columns(cols, text_x_min, text_x_max)
table = Table(cols, rows)
rerror = []
camelot/utils.py
+28 -18
View File
@@ -11,7 +11,7 @@ from pdfminer.pdfinterp import PDFResourceManager
from pdfminer.pdfinterp import PDFPageInterpreter
from pdfminer.pdfdevice import PDFDevice
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LAParams, LTChar, LTTextLineHorizontal
from pdfminer.layout import LAParams, LTChar, LTTextLineHorizontal, LTTextLineVertical
def translate(x1, x2):
@@ -144,7 +144,7 @@ def scale_to_pdf(tables, v_segments, h_segments, factors):
return tables_new, v_segments_new, h_segments_new
def detect_vertical(text):
def get_rotation(ltchar, lttextlh=None, lttextlv=None):
"""Detects if text in table is vertical or not and returns
its orientation.
@@ -156,13 +156,18 @@ def detect_vertical(text):
-------
rotation : string
"""
num_v = [t for t in text if (not t.upright) and t.get_text().strip()]
num_h = [t for t in text if t.upright and t.get_text().strip()]
vger = len(num_v) / float(len(num_v) + len(num_h))
rotation = ''
if vger > 0.8:
clockwise = sum(t.matrix[1] < 0 and t.matrix[2] > 0 for t in text)
anticlockwise = sum(t.matrix[1] > 0 and t.matrix[2] < 0 for t in text)
if lttextlh is not None and lttextlv is not None:
hlen = len([t for t in lttextlh if t.get_text().strip()])
vlen = len([t for t in lttextlv if t.get_text().strip()])
vger = 0.0
else:
hlen = len([t for t in ltchar if t.upright and t.get_text().strip()])
vlen = len([t for t in ltchar if (not t.upright) and t.get_text().strip()])
vger = vlen / float(hlen+vlen)
if hlen < vlen or vger > 0.8:
clockwise = sum(t.matrix[1] < 0 and t.matrix[2] > 0 for t in ltchar)
anticlockwise = sum(t.matrix[1] > 0 and t.matrix[2] < 0 for t in ltchar)
rotation = 'left' if clockwise < anticlockwise else 'right'
return rotation
@@ -520,7 +525,7 @@ def encode_list(ar):
return ar
def extract_text_objects(layout, LTObject, t=None):
def get_text_objects(layout, LTType="char", t=None):
"""Recursively parses pdf layout to get a list of
text objects.
@@ -539,6 +544,12 @@ def extract_text_objects(layout, LTObject, t=None):
t : list
List of text objects.
"""
if LTType == "char":
LTObject = LTChar
elif LTType == "lh":
LTObject = LTTextLineHorizontal
elif LTType == "lv":
LTObject = LTTextLineVertical
if t is None:
t = []
try:
@@ -546,15 +557,14 @@ def extract_text_objects(layout, LTObject, t=None):
if isinstance(obj, LTObject):
t.append(obj)
else:
t += extract_text_objects(obj, LTObject)
t += get_text_objects(obj, LTType=LTType)
except AttributeError:
pass
return t
def pdf_to_text(pname, char_margin, line_margin, word_margin):
# pkey = 'page-{0}'.format(p)
# pname = os.path.join(self.temp, '{}.pdf'.format(pkey))
def get_page_layout(pname, char_margin=2.0, line_margin=0.5, word_margin=0.1,
detect_vertical=True, all_texts=True):
with open(pname, 'r') as f:
parser = PDFParser(f)
document = PDFDocument(parser)
@@ -562,16 +572,16 @@ def pdf_to_text(pname, char_margin, line_margin, word_margin):
raise PDFTextExtractionNotAllowed
laparams = LAParams(char_margin=char_margin,
line_margin=line_margin,
word_margin=word_margin)
word_margin=word_margin,
detect_vertical=detect_vertical,
all_texts=all_texts)
rsrcmgr = PDFResourceManager()
device = PDFPageAggregator(rsrcmgr, laparams=laparams)
interpreter = PDFPageInterpreter(rsrcmgr, device)
for page in PDFPage.create_pages(document):
interpreter.process_page(page)
layout = device.get_result()
lattice_objects = extract_text_objects(layout, LTChar)
stream_objects = extract_text_objects(
layout, LTTextLineHorizontal)
width = layout.bbox[2]
height = layout.bbox[3]
return lattice_objects, stream_objects, width, height
dim = (width, height)
return layout, dim