Create python package
Add version support Add new test file [RFC] First phase [RFC] Second phase [RFC] Third phase Add logging Update README Add debug Add debug, fixes Add pep8 changes Add fix Rename CLI tool Add csv fix Update README Add fix for numpages Update README Update requirements.txt Use yield Add tuple unpacking fix Fix n00b mistake Add check for None Fix check for None Fix unicode Add relative imports
This commit is contained in:
Vinayak Mehta
@@ -0,0 +1,7 @@
|
||||
from .pdf import Pdf
|
||||
from .lattice import Lattice
|
||||
from .stream import Stream
|
||||
|
||||
__version__ = '0.1'
|
||||
|
||||
__all__ = ['Pdf', 'Lattice', 'Stream']
|
||||
@@ -0,0 +1,85 @@
|
||||
class Cell:
|
||||
"""Cell
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x1 : int
|
||||
|
||||
y1 : int
|
||||
|
||||
x2 : int
|
||||
|
||||
y2 : int
|
||||
|
||||
Attributes
|
||||
----------
|
||||
lb : tuple
|
||||
|
||||
lt : tuple
|
||||
|
||||
rb : tuple
|
||||
|
||||
rt : tuple
|
||||
|
||||
bbox : tuple
|
||||
|
||||
left : bool
|
||||
|
||||
right : bool
|
||||
|
||||
top : bool
|
||||
|
||||
bottom : bool
|
||||
|
||||
text : string
|
||||
|
||||
spanning_h : bool
|
||||
|
||||
spanning_v : bool
|
||||
"""
|
||||
|
||||
def __init__(self, x1, y1, x2, y2):
|
||||
|
||||
self.x1 = x1
|
||||
self.y1 = y1
|
||||
self.x2 = x2
|
||||
self.y2 = y2
|
||||
self.lb = (x1, y1)
|
||||
self.lt = (x1, y2)
|
||||
self.rb = (x2, y1)
|
||||
self.rt = (x2, y2)
|
||||
self.bbox = (x1, y1, x2, y2)
|
||||
self.left = False
|
||||
self.right = False
|
||||
self.top = False
|
||||
self.bottom = False
|
||||
self.text = ''
|
||||
self.spanning_h = False
|
||||
self.spanning_v = False
|
||||
|
||||
def add_text(self, text):
|
||||
"""Adds text to cell object.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : string
|
||||
"""
|
||||
self.text = ''.join([self.text, text])
|
||||
|
||||
def get_text(self):
|
||||
"""Returns text from cell object.
|
||||
|
||||
Returns
|
||||
-------
|
||||
text : string
|
||||
"""
|
||||
return self.text
|
||||
|
||||
def get_bounded_edges(self):
|
||||
"""Returns number of edges by which a cell is bounded.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bounded_edges : int
|
||||
"""
|
||||
return self.top + self.bottom + self.left + self.right
|
||||
@@ -0,0 +1,354 @@
|
||||
import os
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from .table import Table
|
||||
from .utils import (transform, elements_bbox, detect_vertical, merge_close_values,
|
||||
get_row_index, get_column_index, reduce_index, outline,
|
||||
fill_spanning, remove_empty, encode_list)
|
||||
|
||||
|
||||
__all__ = ['Lattice']
|
||||
|
||||
|
||||
def _morph_transform(imagename, scale=15, invert=False):
|
||||
"""Morphological Transformation
|
||||
|
||||
Applies a series of morphological operations on the image
|
||||
to find table contours and line segments.
|
||||
http://answers.opencv.org/question/63847/how-to-extract-tables-from-an-image/
|
||||
|
||||
Empirical result for adaptiveThreshold's blockSize=5 and C=-0.2
|
||||
taken from http://pequan.lip6.fr/~bereziat/pima/2012/seuillage/sezgin04.pdf
|
||||
|
||||
Parameters
|
||||
----------
|
||||
imagename : Path to image.
|
||||
|
||||
scale : int
|
||||
Scaling factor. Large scaling factor leads to smaller lines
|
||||
being detected. (optional, default: 15)
|
||||
|
||||
invert : bool
|
||||
Invert pdf image to make sure that lines are in foreground.
|
||||
(optional, default: False)
|
||||
|
||||
Returns
|
||||
-------
|
||||
img : ndarray
|
||||
|
||||
tables : dict
|
||||
Dictionary with table bounding box as key and list of
|
||||
joints found in the table as value.
|
||||
|
||||
v_segments : list
|
||||
List of vertical line segments found in the image.
|
||||
|
||||
h_segments : list
|
||||
List of horizontal line segments found in the image.
|
||||
"""
|
||||
img = cv2.imread(imagename)
|
||||
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
||||
|
||||
if invert:
|
||||
threshold = cv2.adaptiveThreshold(
|
||||
gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY,
|
||||
15, -0.2)
|
||||
else:
|
||||
threshold = cv2.adaptiveThreshold(
|
||||
np.invert(gray), 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
|
||||
cv2.THRESH_BINARY,
|
||||
15, -0.2)
|
||||
|
||||
vertical = threshold
|
||||
horizontal = threshold
|
||||
|
||||
verticalsize = vertical.shape[0] / scale
|
||||
horizontalsize = horizontal.shape[1] / scale
|
||||
|
||||
ver = cv2.getStructuringElement(cv2.MORPH_RECT, (1, verticalsize))
|
||||
hor = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontalsize, 1))
|
||||
|
||||
vertical = cv2.erode(vertical, ver, (-1, -1))
|
||||
vertical = cv2.dilate(vertical, ver, (-1, -1))
|
||||
|
||||
horizontal = cv2.erode(horizontal, hor, (-1, -1))
|
||||
horizontal = cv2.dilate(horizontal, hor, (-1, -1))
|
||||
|
||||
mask = vertical + horizontal
|
||||
joints = np.bitwise_and(vertical, horizontal)
|
||||
__, contours, __ = cv2.findContours(
|
||||
mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||
contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]
|
||||
|
||||
tables = {}
|
||||
for c in contours:
|
||||
c_poly = cv2.approxPolyDP(c, 3, True)
|
||||
x, y, w, h = cv2.boundingRect(c_poly)
|
||||
roi = joints[y : y + h, x : x + w]
|
||||
__, jc, __ = cv2.findContours(
|
||||
roi, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
|
||||
if len(jc) <= 4: # remove contours with less than <=4 joints
|
||||
continue
|
||||
joint_coords = []
|
||||
for j in jc:
|
||||
jx, jy, jw, jh = cv2.boundingRect(j)
|
||||
c1, c2 = x + (2 * jx + jw) / 2, y + (2 * jy + jh) / 2
|
||||
joint_coords.append((c1, c2))
|
||||
tables[(x, y + h, x + w, y)] = joint_coords
|
||||
|
||||
v_segments, h_segments = [], []
|
||||
_, vcontours, _ = cv2.findContours(
|
||||
vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||
for vc in vcontours:
|
||||
x, y, w, h = cv2.boundingRect(vc)
|
||||
x1, x2 = x, x + w
|
||||
y1, y2 = y, y + h
|
||||
v_segments.append(((x1 + x2) / 2, y2, (x1 + x2) / 2, y1))
|
||||
|
||||
_, hcontours, _ = cv2.findContours(
|
||||
horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||
for hc in hcontours:
|
||||
x, y, w, h = cv2.boundingRect(hc)
|
||||
x1, x2 = x, x + w
|
||||
y1, y2 = y, y + h
|
||||
h_segments.append((x1, (y1 + y2) / 2, x2, (y1 + y2) / 2))
|
||||
|
||||
return img, tables, v_segments, h_segments
|
||||
|
||||
|
||||
class Lattice:
|
||||
"""Lattice algorithm
|
||||
|
||||
Makes use of pdf geometry by processing its image, to make a table
|
||||
and fills text objects in table cells.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pdfobject : camelot.pdf.Pdf
|
||||
|
||||
fill : None, 'h', 'v', 'hv'
|
||||
Fill data in horizontal and/or vertical spanning
|
||||
cells. (optional)
|
||||
|
||||
scale : int
|
||||
Scaling factor. Large scaling factor leads to smaller lines
|
||||
being detected. (optional, default: 15)
|
||||
|
||||
jtol : int
|
||||
Tolerance to account for when comparing joint and line
|
||||
coordinates. (optional, default: 2)
|
||||
|
||||
mtol : int
|
||||
Tolerance to account for when merging lines which are
|
||||
very close. (optional, default: 2)
|
||||
|
||||
invert : bool
|
||||
Invert pdf image to make sure that lines are in foreground.
|
||||
(optional, default: False)
|
||||
|
||||
debug : 'contour', 'line', 'joint', 'table'
|
||||
Debug by visualizing pdf geometry.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
tables : dict
|
||||
Dictionary with page number as key and list of tables on that
|
||||
page as value.
|
||||
"""
|
||||
|
||||
def __init__(self, pdfobject, fill=None, scale=15, jtol=2, mtol=2,
|
||||
invert=False, debug=None):
|
||||
|
||||
self.pdfobject = pdfobject
|
||||
self.fill = fill
|
||||
self.scale = scale
|
||||
self.jtol = jtol
|
||||
self.mtol = mtol
|
||||
self.invert = invert
|
||||
self.debug = debug
|
||||
self.tables = {}
|
||||
if self.debug is not None:
|
||||
self.debug_images = {}
|
||||
self.debug_segments = {}
|
||||
self.debug_tables = {}
|
||||
|
||||
def get_tables(self):
|
||||
"""Returns all tables found in given pdf.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tables : dict
|
||||
Dictionary with page number as key and list of tables on that
|
||||
page as value.
|
||||
"""
|
||||
self.pdfobject.split()
|
||||
self.pdfobject.convert()
|
||||
for page in self.pdfobject.extract():
|
||||
p, text, __, width, height = page
|
||||
pkey = 'pg-{0}'.format(p)
|
||||
imagename = os.path.join(
|
||||
self.pdfobject.temp, '{}.png'.format(pkey))
|
||||
pdf_x = width
|
||||
pdf_y = height
|
||||
img, table_bbox, v_segments, h_segments = _morph_transform(
|
||||
imagename, scale=self.scale, invert=self.invert)
|
||||
img_x = img.shape[1]
|
||||
img_y = img.shape[0]
|
||||
scaling_factor_x = pdf_x / float(img_x)
|
||||
scaling_factor_y = pdf_y / float(img_y)
|
||||
|
||||
if self.debug is not None:
|
||||
self.debug_images[pkey] = (img, table_bbox)
|
||||
|
||||
factors = (scaling_factor_x, scaling_factor_y, img_y)
|
||||
table_bbox, v_segments, h_segments = transform(table_bbox, v_segments,
|
||||
h_segments, factors)
|
||||
|
||||
if self.debug is not None:
|
||||
self.debug_segments[pkey] = (v_segments, h_segments)
|
||||
|
||||
if self.debug is not None:
|
||||
debug_page_tables = []
|
||||
page_tables = []
|
||||
# sort tables based on y-coord
|
||||
for k in sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True):
|
||||
# select edges which lie within table_bbox
|
||||
text_bbox, v_s, h_s = elements_bbox(k, text, v_segments,
|
||||
h_segments)
|
||||
rotated = detect_vertical(text_bbox)
|
||||
cols, rows = zip(*table_bbox[k])
|
||||
cols, rows = list(cols), list(rows)
|
||||
cols.extend([k[0], k[2]])
|
||||
rows.extend([k[1], k[3]])
|
||||
# sort horizontal and vertical segments
|
||||
cols = merge_close_values(sorted(cols), mtol=self.mtol)
|
||||
rows = merge_close_values(
|
||||
sorted(rows, reverse=True), mtol=self.mtol)
|
||||
# make grid using x and y coord of shortlisted rows and cols
|
||||
cols = [(cols[i], cols[i + 1])
|
||||
for i in range(0, len(cols) - 1)]
|
||||
rows = [(rows[i], rows[i + 1])
|
||||
for i in range(0, len(rows) - 1)]
|
||||
table = Table(cols, rows)
|
||||
# set table edges to True using ver+hor lines
|
||||
table = table.set_edges(v_s, h_s, jtol=self.jtol)
|
||||
# set spanning cells to True
|
||||
table = table.set_spanning()
|
||||
# set table border edges to True
|
||||
table = outline(table)
|
||||
|
||||
if self.debug is not None:
|
||||
debug_page_tables.append(table)
|
||||
|
||||
# fill text after sorting it
|
||||
if rotated == '':
|
||||
text_bbox.sort(key=lambda x: (-x.y0, x.x0))
|
||||
elif rotated == 'left':
|
||||
text_bbox.sort(key=lambda x: (x.x0, x.y0))
|
||||
elif rotated == 'right':
|
||||
text_bbox.sort(key=lambda x: (-x.x0, -x.y0))
|
||||
for t in text_bbox:
|
||||
r_idx = get_row_index(t, rows)
|
||||
c_idx = get_column_index(t, cols)
|
||||
if None in [r_idx, c_idx]:
|
||||
# couldn't assign LTChar to any cell
|
||||
pass
|
||||
else:
|
||||
r_idx, c_idx = reduce_index(
|
||||
table, rotated, r_idx, c_idx)
|
||||
table.cells[r_idx][c_idx].add_text(
|
||||
t.get_text().strip('\n'))
|
||||
|
||||
if self.fill is not None:
|
||||
table = fill_spanning(table, fill=self.fill)
|
||||
ar = table.get_list()
|
||||
if rotated == 'left':
|
||||
ar = zip(*ar[::-1])
|
||||
elif rotated == 'right':
|
||||
ar = zip(*ar[::1])
|
||||
ar.reverse()
|
||||
ar = remove_empty(ar)
|
||||
ar = [list(o) for o in ar]
|
||||
page_tables.append(encode_list(ar))
|
||||
print pkey # verbose
|
||||
self.tables[pkey] = page_tables
|
||||
|
||||
if self.debug is not None:
|
||||
self.debug_tables[pkey] = debug_page_tables
|
||||
|
||||
if self.pdfobject.clean:
|
||||
self.pdfobject.remove_tempdir()
|
||||
|
||||
if self.debug is not None:
|
||||
return None
|
||||
|
||||
return self.tables
|
||||
|
||||
def plot_geometry(self, geometry):
|
||||
"""Plots various pdf geometries that are detected so user can choose
|
||||
tweak scale, jtol, mtol parameters.
|
||||
"""
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
if geometry == 'contour':
|
||||
for pkey in self.debug_images.keys():
|
||||
img, table_bbox = self.debug_images[pkey]
|
||||
for t in table_bbox.keys():
|
||||
cv2.rectangle(img, (t[0], t[1]),
|
||||
(t[2], t[3]), (255, 0, 0), 3)
|
||||
plt.imshow(img)
|
||||
plt.axis('off')
|
||||
plt.show()
|
||||
elif geometry == 'joint':
|
||||
x_coord = []
|
||||
y_coord = []
|
||||
for pkey in self.debug_images.keys():
|
||||
img, table_bbox = self.debug_images[pkey]
|
||||
for k in table_bbox.keys():
|
||||
for coord in table_bbox[k]:
|
||||
x_coord.append(coord[0])
|
||||
y_coord.append(coord[1])
|
||||
max_x, max_y = max(x_coord), max(y_coord)
|
||||
plt.plot(x_coord, y_coord, 'ro')
|
||||
plt.axis([0, max_x + 100, max_y + 100, 0])
|
||||
plt.imshow(img)
|
||||
plt.axis('off')
|
||||
plt.show()
|
||||
elif geometry == 'line':
|
||||
for pkey in self.debug_segments.keys():
|
||||
v_s, h_s = self.debug_segments[pkey]
|
||||
for v in v_s:
|
||||
plt.plot([v[0], v[2]], [v[1], v[3]])
|
||||
for h in h_s:
|
||||
plt.plot([h[0], h[2]], [h[1], h[3]])
|
||||
plt.axis('off')
|
||||
plt.show()
|
||||
elif geometry == 'table':
|
||||
for pkey in self.debug_tables.keys():
|
||||
for table in self.debug_tables[pkey]:
|
||||
for i in range(len(table.cells)):
|
||||
for j in range(len(table.cells[i])):
|
||||
if table.cells[i][j].left:
|
||||
plt.plot([table.cells[i][j].lb[0],
|
||||
table.cells[i][j].lt[0]],
|
||||
[table.cells[i][j].lb[1],
|
||||
table.cells[i][j].lt[1]])
|
||||
if table.cells[i][j].right:
|
||||
plt.plot([table.cells[i][j].rb[0],
|
||||
table.cells[i][j].rt[0]],
|
||||
[table.cells[i][j].rb[1],
|
||||
table.cells[i][j].rt[1]])
|
||||
if table.cells[i][j].top:
|
||||
plt.plot([table.cells[i][j].lt[0],
|
||||
table.cells[i][j].rt[0]],
|
||||
[table.cells[i][j].lt[1],
|
||||
table.cells[i][j].rt[1]])
|
||||
if table.cells[i][j].bottom:
|
||||
plt.plot([table.cells[i][j].lb[0],
|
||||
table.cells[i][j].rb[0]],
|
||||
[table.cells[i][j].lb[1],
|
||||
table.cells[i][j].rb[1]])
|
||||
plt.axis('off')
|
||||
plt.show()
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from PyPDF2 import PdfFileReader, PdfFileWriter
|
||||
from pdfminer.pdfparser import PDFParser
|
||||
from pdfminer.pdfdocument import PDFDocument
|
||||
from pdfminer.pdfpage import PDFPage
|
||||
from pdfminer.pdfpage import PDFTextExtractionNotAllowed
|
||||
from pdfminer.pdfinterp import PDFResourceManager
|
||||
from pdfminer.pdfinterp import PDFPageInterpreter
|
||||
from pdfminer.pdfdevice import PDFDevice
|
||||
from pdfminer.converter import PDFPageAggregator
|
||||
from pdfminer.layout import LAParams, LTChar, LTTextLineHorizontal
|
||||
from wand.image import Image
|
||||
|
||||
|
||||
__all__ = ['Pdf']
|
||||
|
||||
|
||||
def _parse_page_numbers(pagenos):
|
||||
"""Converts list of page ranges to a list of page numbers.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pagenos : list
|
||||
List of dicts containing page ranges.
|
||||
|
||||
Returns
|
||||
-------
|
||||
page_numbers : list
|
||||
List of page numbers.
|
||||
"""
|
||||
page_numbers = []
|
||||
for p in pagenos:
|
||||
page_numbers.extend(range(p['start'], p['end'] + 1))
|
||||
page_numbers = sorted(set(page_numbers))
|
||||
return page_numbers
|
||||
|
||||
|
||||
def _extract_text_objects(layout, LTObject, t=None):
|
||||
"""Recursively parses pdf layout to get a list of
|
||||
text objects.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
layout : object
|
||||
Layout object.
|
||||
|
||||
LTObject : object
|
||||
Text object, either LTChar or LTTextLineHorizontal.
|
||||
|
||||
t : list
|
||||
|
||||
Returns
|
||||
-------
|
||||
t : list
|
||||
List of text objects.
|
||||
"""
|
||||
if t is None:
|
||||
t = []
|
||||
try:
|
||||
for obj in layout._objs:
|
||||
if isinstance(obj, LTObject):
|
||||
t.append(obj)
|
||||
else:
|
||||
t += _extract_text_objects(obj, LTObject)
|
||||
except AttributeError:
|
||||
pass
|
||||
return t
|
||||
|
||||
|
||||
class Pdf:
|
||||
"""Handles all pdf operations which include:
|
||||
|
||||
1. Split pdf into single page pdfs using given page numbers
|
||||
2. Convert single page pdfs into images
|
||||
3. Extract text from single page pdfs
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pdfname : string
|
||||
Path to pdf.
|
||||
|
||||
pagenos : list
|
||||
List of dicts which specify pdf page ranges.
|
||||
|
||||
char_margin : float
|
||||
Chars closer than char_margin are grouped together to form a
|
||||
word. (optional, default: 2.0)
|
||||
|
||||
line_margin : float
|
||||
Lines closer than line_margin are grouped together to form a
|
||||
textbox. (optional, default: 0.5)
|
||||
|
||||
word_margin : float
|
||||
Insert blank spaces between chars if distance between words
|
||||
is greater than word_margin. (optional, default: 0.1)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
temp : string
|
||||
Path to temporary directory.
|
||||
|
||||
lattice_objects : dict
|
||||
List of text objects.
|
||||
|
||||
stream_objects : dict
|
||||
List of text objects.
|
||||
|
||||
width : dict
|
||||
List of dicts with width of each pdf page.
|
||||
|
||||
height : dict
|
||||
List of dicts with height of each pdf page.
|
||||
"""
|
||||
|
||||
def __init__(self, pdfname, pagenos=[{'start': 1, 'end': 1}],
|
||||
char_margin=2.0, line_margin=0.5, word_margin=0.1,
|
||||
clean=False):
|
||||
|
||||
self.pdfname = pdfname
|
||||
self.pagenos = _parse_page_numbers(pagenos)
|
||||
self.char_margin = char_margin
|
||||
self.line_margin = line_margin
|
||||
self.word_margin = word_margin
|
||||
self.clean = clean
|
||||
self.temp = tempfile.mkdtemp()
|
||||
|
||||
def split(self):
|
||||
"""Splits pdf into single page pdfs.
|
||||
"""
|
||||
infile = PdfFileReader(open(self.pdfname, 'rb'), strict=False)
|
||||
for p in self.pagenos:
|
||||
page = infile.getPage(p - 1)
|
||||
outfile = PdfFileWriter()
|
||||
outfile.addPage(page)
|
||||
with open(os.path.join(self.temp, 'pg-{0}.pdf'.format(p)), 'wb') as f:
|
||||
outfile.write(f)
|
||||
|
||||
def extract(self):
|
||||
"""Extracts text objects, width, height from a pdf.
|
||||
"""
|
||||
for p in self.pagenos:
|
||||
pkey = 'pg-{0}'.format(p)
|
||||
pname = os.path.join(self.temp, '{}.pdf'.format(pkey))
|
||||
with open(pname, 'r') as f:
|
||||
parser = PDFParser(f)
|
||||
document = PDFDocument(parser)
|
||||
if not document.is_extractable:
|
||||
raise PDFTextExtractionNotAllowed
|
||||
laparams = LAParams(char_margin=self.char_margin,
|
||||
line_margin=self.line_margin,
|
||||
word_margin=self.word_margin)
|
||||
rsrcmgr = PDFResourceManager()
|
||||
device = PDFPageAggregator(rsrcmgr, laparams=laparams)
|
||||
interpreter = PDFPageInterpreter(rsrcmgr, device)
|
||||
for page in PDFPage.create_pages(document):
|
||||
interpreter.process_page(page)
|
||||
layout = device.get_result()
|
||||
lattice_objects = _extract_text_objects(layout, LTChar)
|
||||
stream_objects = _extract_text_objects(
|
||||
layout, LTTextLineHorizontal)
|
||||
width = layout.bbox[2]
|
||||
height = layout.bbox[3]
|
||||
yield p, lattice_objects, stream_objects, width, height
|
||||
|
||||
def convert(self):
|
||||
"""Converts single page pdfs to images.
|
||||
"""
|
||||
for p in self.pagenos:
|
||||
pdfname = os.path.join(self.temp, 'pg-{0}.pdf'.format(p))
|
||||
imagename = os.path.join(self.temp, 'pg-{0}.png'.format(p))
|
||||
with Image(filename=pdfname, depth=8, resolution=300) as png:
|
||||
png.save(filename=imagename)
|
||||
|
||||
def remove_tempdir(self):
|
||||
shutil.rmtree(self.temp)
|
||||
@@ -0,0 +1,210 @@
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .utils import get_column_index, encode_list
|
||||
|
||||
|
||||
__all__ = ['Stream']
|
||||
|
||||
|
||||
def _group_rows(text, ytol=2):
|
||||
"""Groups text objects into rows using ytol.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : list
|
||||
List of text objects.
|
||||
|
||||
ytol : int
|
||||
Tolerance to account for when grouping rows
|
||||
together. (default: 2, optional)
|
||||
|
||||
Returns
|
||||
-------
|
||||
rows : list
|
||||
List of grouped text rows.
|
||||
"""
|
||||
row_y = 0
|
||||
rows = []
|
||||
temp = []
|
||||
for t in text:
|
||||
# is checking for upright necessary?
|
||||
# if t.get_text().strip() and all([obj.upright for obj in t._objs if
|
||||
# type(obj) is LTChar]):
|
||||
if t.get_text().strip():
|
||||
if not np.isclose(row_y, t.y0, atol=ytol):
|
||||
row_y = t.y0
|
||||
rows.append(temp)
|
||||
temp = []
|
||||
temp.append(t)
|
||||
return rows
|
||||
|
||||
|
||||
def _merge_columns(l):
|
||||
"""Merges overlapping columns and returns list with updated
|
||||
columns boundaries.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
l : list
|
||||
List of column x-coordinates.
|
||||
|
||||
Returns
|
||||
-------
|
||||
merged : list
|
||||
List of merged column x-coordinates.
|
||||
"""
|
||||
merged = []
|
||||
for higher in l:
|
||||
if not merged:
|
||||
merged.append(higher)
|
||||
else:
|
||||
lower = merged[-1]
|
||||
if higher[0] <= lower[1]:
|
||||
upper_bound = max(lower[1], higher[1])
|
||||
lower_bound = min(lower[0], higher[0])
|
||||
merged[-1] = (lower_bound, upper_bound)
|
||||
else:
|
||||
merged.append(higher)
|
||||
return merged
|
||||
|
||||
|
||||
class Stream:
|
||||
"""Stream algorithm
|
||||
|
||||
Groups text objects into rows and guesses number of columns
|
||||
using mode of the number of text objects in each row.
|
||||
|
||||
The number of columns can be passed explicitly or specified by a
|
||||
list of column x-coordinates.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pdfobject : camelot.pdf.Pdf
|
||||
|
||||
ncolumns : int
|
||||
Number of columns. (optional, default: 0)
|
||||
|
||||
columns : string
|
||||
Comma-separated list of column x-coordinates.
|
||||
(optional, default: None)
|
||||
|
||||
ytol : int
|
||||
Tolerance to account for when grouping rows
|
||||
together. (optional, default: 2)
|
||||
|
||||
debug : bool
|
||||
Debug by visualizing textboxes. (optional, default: False)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
tables : dict
|
||||
Dictionary with page number as key and list of tables on that
|
||||
page as value.
|
||||
"""
|
||||
|
||||
def __init__(self, pdfobject, ncolumns=0, columns=None, ytol=2,
|
||||
debug=False):
|
||||
|
||||
self.pdfobject = pdfobject
|
||||
self.ncolumns = ncolumns
|
||||
self.columns = columns
|
||||
self.ytol = ytol
|
||||
self.debug = debug
|
||||
self.tables = {}
|
||||
if self.debug:
|
||||
self.debug_text = {}
|
||||
|
||||
def get_tables(self):
|
||||
"""Returns all tables found in given pdf.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tables : dict
|
||||
Dictionary with page number as key and list of tables on that
|
||||
page as value.
|
||||
"""
|
||||
self.pdfobject.split()
|
||||
for page in self.pdfobject.extract():
|
||||
p, __, text, __, __ = page
|
||||
pkey = 'pg-{0}'.format(p)
|
||||
text.sort(key=lambda x: (-x.y0, x.x0))
|
||||
|
||||
if self.debug:
|
||||
self.debug_text[pkey] = text
|
||||
|
||||
rows = _group_rows(text, ytol=self.ytol)
|
||||
elements = [len(r) for r in rows]
|
||||
# a table can't have just 1 column, can it?
|
||||
elements = filter(lambda x: x != 1, elements)
|
||||
|
||||
guess = False
|
||||
if self.columns:
|
||||
cols = self.columns.split(',')
|
||||
cols = [(float(cols[i]), float(cols[i + 1]))
|
||||
for i in range(0, len(cols) - 1)]
|
||||
else:
|
||||
guess = True
|
||||
ncols = self.ncolumns if self.ncolumns else max(
|
||||
set(elements), key=elements.count)
|
||||
if ncols == 0:
|
||||
# no tables detected
|
||||
continue
|
||||
cols = [(t.x0, t.x1)
|
||||
for r in rows for t in r if len(r) == ncols]
|
||||
cols = _merge_columns(sorted(cols))
|
||||
cols = [(c[0] + c[1]) / 2.0 for c in cols]
|
||||
|
||||
ar = [['' for c in cols] for r in rows]
|
||||
for r_idx, r in enumerate(rows):
|
||||
for t in r:
|
||||
if guess:
|
||||
cog = (t.x0 + t.x1) / 2.0
|
||||
diff = [abs(cog - c) for c in cols]
|
||||
c_idx = diff.index(min(diff))
|
||||
else:
|
||||
c_idx = get_column_index(t, cols)
|
||||
if None in [r_idx, c_idx]: # couldn't assign LTTextLH to any cell
|
||||
continue
|
||||
if ar[r_idx][c_idx]:
|
||||
ar[r_idx][c_idx] = ' '.join(
|
||||
[ar[r_idx][c_idx], t.get_text().strip()])
|
||||
else:
|
||||
ar[r_idx][c_idx] = t.get_text().strip()
|
||||
print pkey # verbose
|
||||
self.tables[pkey] = [encode_list(ar)]
|
||||
|
||||
if self.pdfobject.clean:
|
||||
self.pdfobject.remove_tempdir()
|
||||
|
||||
if self.debug:
|
||||
return None
|
||||
|
||||
return self.tables
|
||||
|
||||
def plot_text(self):
|
||||
"""Plots all text objects so user can choose number of columns
|
||||
or columns x-coordinates using the matplotlib interface.
|
||||
"""
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as patches
|
||||
|
||||
for pkey in sorted(self.debug_text.keys()):
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
xs, ys = [], []
|
||||
for t in self.debug_text[pkey]:
|
||||
xs.extend([t.x0, t.x1])
|
||||
ys.extend([t.y0, t.y1])
|
||||
ax.add_patch(
|
||||
patches.Rectangle(
|
||||
(t.x0, t.y0),
|
||||
t.x1 - t.x0,
|
||||
t.y1 - t.y0
|
||||
)
|
||||
)
|
||||
ax.set_xlim(min(xs) - 10, max(xs) + 10)
|
||||
ax.set_ylim(min(ys) - 10, max(ys) + 10)
|
||||
plt.axis('off')
|
||||
plt.show()
|
||||
@@ -0,0 +1,209 @@
|
||||
import numpy as np
|
||||
|
||||
from .cell import Cell
|
||||
|
||||
|
||||
class Table:
|
||||
"""Table
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cols : list
|
||||
List of column x-coordinates.
|
||||
|
||||
rows : list
|
||||
List of row y-coordinates.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
cells : list
|
||||
2-D list of cell objects.
|
||||
"""
|
||||
|
||||
def __init__(self, cols, rows):
|
||||
|
||||
self.cols = cols
|
||||
self.rows = rows
|
||||
self.cells = [[Cell(c[0], r[1], c[1], r[0])
|
||||
for c in cols] for r in rows]
|
||||
|
||||
def set_edges(self, vertical, horizontal, jtol=2):
|
||||
"""Sets cell edges to True if corresponding line segments
|
||||
are detected in the pdf image.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
vertical : list
|
||||
List of vertical line segments.
|
||||
|
||||
horizontal : list
|
||||
List of horizontal line segments.
|
||||
|
||||
jtol : int, default: 2, optional
|
||||
Tolerance to account for when comparing joint and line
|
||||
coordinates.
|
||||
"""
|
||||
for v in vertical:
|
||||
# find closest x coord
|
||||
# iterate over y coords and find closest points
|
||||
i = [i for i, t in enumerate(self.cols)
|
||||
if np.isclose(v[0], t[0], atol=jtol)]
|
||||
j = [j for j, t in enumerate(self.rows)
|
||||
if np.isclose(v[3], t[0], atol=jtol)]
|
||||
k = [k for k, t in enumerate(self.rows)
|
||||
if np.isclose(v[1], t[0], atol=jtol)]
|
||||
if not j:
|
||||
continue
|
||||
J = j[0]
|
||||
if i == [0]: # only left edge
|
||||
I = i[0]
|
||||
if k:
|
||||
K = k[0]
|
||||
while J < K:
|
||||
self.cells[J][I].left = True
|
||||
J += 1
|
||||
else:
|
||||
K = len(self.rows)
|
||||
while J < K:
|
||||
self.cells[J][I].left = True
|
||||
J += 1
|
||||
elif i == []: # only right edge
|
||||
I = len(self.cols) - 1
|
||||
if k:
|
||||
K = k[0]
|
||||
while J < K:
|
||||
self.cells[J][I].right = True
|
||||
J += 1
|
||||
else:
|
||||
K = len(self.rows)
|
||||
while J < K:
|
||||
self.cells[J][I].right = True
|
||||
J += 1
|
||||
else: # both left and right edges
|
||||
I = i[0]
|
||||
if k:
|
||||
K = k[0]
|
||||
while J < K:
|
||||
self.cells[J][I].left = True
|
||||
self.cells[J][I - 1].right = True
|
||||
J += 1
|
||||
else:
|
||||
K = len(self.rows)
|
||||
while J < K:
|
||||
self.cells[J][I].left = True
|
||||
self.cells[J][I - 1].right = True
|
||||
J += 1
|
||||
|
||||
for h in horizontal:
|
||||
# find closest y coord
|
||||
# iterate over x coords and find closest points
|
||||
i = [i for i, t in enumerate(self.rows)
|
||||
if np.isclose(h[1], t[0], atol=jtol)]
|
||||
j = [j for j, t in enumerate(self.cols)
|
||||
if np.isclose(h[0], t[0], atol=jtol)]
|
||||
k = [k for k, t in enumerate(self.cols)
|
||||
if np.isclose(h[2], t[0], atol=jtol)]
|
||||
if not j:
|
||||
continue
|
||||
J = j[0]
|
||||
if i == [0]: # only top edge
|
||||
I = i[0]
|
||||
if k:
|
||||
K = k[0]
|
||||
while J < K:
|
||||
self.cells[I][J].top = True
|
||||
J += 1
|
||||
else:
|
||||
K = len(self.cols)
|
||||
while J < K:
|
||||
self.cells[I][J].top = True
|
||||
J += 1
|
||||
elif i == []: # only bottom edge
|
||||
I = len(self.rows) - 1
|
||||
if k:
|
||||
K = k[0]
|
||||
while J < K:
|
||||
self.cells[I][J].bottom = True
|
||||
J += 1
|
||||
else:
|
||||
K = len(self.cols)
|
||||
while J < K:
|
||||
self.cells[I][J].bottom = True
|
||||
J += 1
|
||||
else: # both top and bottom edges
|
||||
I = i[0]
|
||||
if k:
|
||||
K = k[0]
|
||||
while J < K:
|
||||
self.cells[I][J].top = True
|
||||
self.cells[I - 1][J].bottom = True
|
||||
J += 1
|
||||
else:
|
||||
K = len(self.cols)
|
||||
while J < K:
|
||||
self.cells[I][J].top = True
|
||||
self.cells[I - 1][J].bottom = True
|
||||
J += 1
|
||||
|
||||
return self
|
||||
|
||||
def set_spanning(self):
|
||||
"""Sets spanning values of a cell to True if it isn't
|
||||
bounded by four edges.
|
||||
"""
|
||||
for i in range(len(self.cells)):
|
||||
for j in range(len(self.cells[i])):
|
||||
bound = self.cells[i][j].get_bounded_edges()
|
||||
if bound == 4:
|
||||
continue
|
||||
|
||||
elif bound == 3:
|
||||
if not self.cells[i][j].left:
|
||||
if (self.cells[i][j].right and
|
||||
self.cells[i][j].top and
|
||||
self.cells[i][j].bottom):
|
||||
self.cells[i][j].spanning_h = True
|
||||
|
||||
elif not self.cells[i][j].right:
|
||||
if (self.cells[i][j].left and
|
||||
self.cells[i][j].top and
|
||||
self.cells[i][j].bottom):
|
||||
self.cells[i][j].spanning_h = True
|
||||
|
||||
elif not self.cells[i][j].top:
|
||||
if (self.cells[i][j].left and
|
||||
self.cells[i][j].right and
|
||||
self.cells[i][j].bottom):
|
||||
self.cells[i][j].spanning_v = True
|
||||
|
||||
elif not self.cells[i][j].bottom:
|
||||
if (self.cells[i][j].left and
|
||||
self.cells[i][j].right and
|
||||
self.cells[i][j].top):
|
||||
self.cells[i][j].spanning_v = True
|
||||
|
||||
elif bound == 2:
|
||||
if self.cells[i][j].left and self.cells[i][j].right:
|
||||
if (not self.cells[i][j].top and
|
||||
not self.cells[i][j].bottom):
|
||||
self.cells[i][j].spanning_v = True
|
||||
|
||||
elif self.cells[i][j].top and self.cells[i][j].bottom:
|
||||
if (not self.cells[i][j].left and
|
||||
not self.cells[i][j].right):
|
||||
self.cells[i][j].spanning_h = True
|
||||
|
||||
return self
|
||||
|
||||
def get_list(self):
|
||||
"""Returns text from all cells as list of lists.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ar : list
|
||||
"""
|
||||
ar = []
|
||||
for i in range(len(self.cells)):
|
||||
ar.append([self.cells[i][j].get_text().strip()
|
||||
for j in range(len(self.cells[i]))])
|
||||
return ar
|
||||
@@ -0,0 +1,399 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
def translate(x1, x2):
|
||||
"""Translates x2 by x1.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x1 : float
|
||||
|
||||
x2 : float
|
||||
|
||||
Returns
|
||||
-------
|
||||
x2 : float
|
||||
"""
|
||||
x2 += x1
|
||||
return x2
|
||||
|
||||
|
||||
def scale(x, s):
|
||||
"""Scales x by scaling factor s.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : float
|
||||
|
||||
s : float
|
||||
|
||||
Returns
|
||||
-------
|
||||
x : float
|
||||
"""
|
||||
x *= s
|
||||
return x
|
||||
|
||||
|
||||
def rotate(x1, y1, x2, y2, angle):
|
||||
"""Rotates point x2, y2 about point x1, y1 by angle.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x1 : float
|
||||
|
||||
y1 : float
|
||||
|
||||
x2 : float
|
||||
|
||||
y2 : float
|
||||
|
||||
angle : float
|
||||
Angle in radians.
|
||||
|
||||
Returns
|
||||
-------
|
||||
xnew : float
|
||||
|
||||
ynew : float
|
||||
"""
|
||||
s = np.sin(angle)
|
||||
c = np.cos(angle)
|
||||
x2 = translate(-x1, x2)
|
||||
y2 = translate(-y1, y2)
|
||||
xnew = c * x2 - s * y2
|
||||
ynew = s * x2 + c * y2
|
||||
xnew = translate(x1, xnew)
|
||||
ynew = translate(y1, ynew)
|
||||
return xnew, ynew
|
||||
|
||||
|
||||
def transform(tables, v_segments, h_segments, factors):
|
||||
"""Translates and scales OpenCV coordinates to PDFMiner coordinate
|
||||
space.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tables : dict
|
||||
|
||||
v_segments : list
|
||||
|
||||
h_segments : list
|
||||
|
||||
factors : tuple
|
||||
|
||||
Returns
|
||||
-------
|
||||
tables_new : dict
|
||||
|
||||
v_segments_new : dict
|
||||
|
||||
h_segments_new : dict
|
||||
"""
|
||||
scaling_factor_x, scaling_factor_y, img_y = factors
|
||||
tables_new = {}
|
||||
for k in tables.keys():
|
||||
x1, y1, x2, y2 = k
|
||||
x1 = scale(x1, scaling_factor_x)
|
||||
y1 = scale(abs(translate(-img_y, y1)), scaling_factor_y)
|
||||
x2 = scale(x2, scaling_factor_x)
|
||||
y2 = scale(abs(translate(-img_y, y2)), scaling_factor_y)
|
||||
j_x, j_y = zip(*tables[k])
|
||||
j_x = [scale(j, scaling_factor_x) for j in j_x]
|
||||
j_y = [scale(abs(translate(-img_y, j)), scaling_factor_y) for j in j_y]
|
||||
joints = zip(j_x, j_y)
|
||||
tables_new[(x1, y1, x2, y2)] = joints
|
||||
|
||||
v_segments_new = []
|
||||
for v in v_segments:
|
||||
x1, x2 = scale(v[0], scaling_factor_x), scale(v[2], scaling_factor_x)
|
||||
y1, y2 = scale(abs(translate(-img_y, v[1])), scaling_factor_y), scale(
|
||||
abs(translate(-img_y, v[3])), scaling_factor_y)
|
||||
v_segments_new.append((x1, y1, x2, y2))
|
||||
|
||||
h_segments_new = []
|
||||
for h in h_segments:
|
||||
x1, x2 = scale(h[0], scaling_factor_x), scale(h[2], scaling_factor_x)
|
||||
y1, y2 = scale(abs(translate(-img_y, h[1])), scaling_factor_y), scale(
|
||||
abs(translate(-img_y, h[3])), scaling_factor_y)
|
||||
h_segments_new.append((x1, y1, x2, y2))
|
||||
|
||||
return tables_new, v_segments_new, h_segments_new
|
||||
|
||||
|
||||
def detect_vertical(text):
|
||||
"""Detects if text in table is vertical or not and returns
|
||||
its orientation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : list
|
||||
|
||||
Returns
|
||||
-------
|
||||
rotated : string
|
||||
"""
|
||||
num_v = [t for t in text if (not t.upright) and t.get_text().strip()]
|
||||
num_h = [t for t in text if t.upright and t.get_text().strip()]
|
||||
vger = len(num_v) / float(len(num_v) + len(num_h))
|
||||
rotated = ''
|
||||
if vger > 0.8:
|
||||
clockwise = sum(t.matrix[1] < 0 and t.matrix[2] > 0 for t in text)
|
||||
anticlockwise = sum(t.matrix[1] > 0 and t.matrix[2] < 0 for t in text)
|
||||
rotated = 'left' if clockwise < anticlockwise else 'right'
|
||||
return rotated
|
||||
|
||||
|
||||
def elements_bbox(bbox, text, v_segments, h_segments):
|
||||
"""Returns all text objects and line segments present inside a
|
||||
table's bounding box.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
bbox : tuple
|
||||
|
||||
text : list
|
||||
|
||||
v_segments : list
|
||||
|
||||
h_segments : list
|
||||
|
||||
Returns
|
||||
-------
|
||||
text_bbox : list
|
||||
|
||||
v_s : list
|
||||
|
||||
h_s : list
|
||||
"""
|
||||
lb = (bbox[0], bbox[1])
|
||||
rt = (bbox[2], bbox[3])
|
||||
text_bbox = [t for t in text if lb[0] - 2 <= (t.x0 + t.x1) / 2.0
|
||||
<= rt[0] + 2 and lb[1] - 2 <= (t.y0 + t.y1) / 2.0
|
||||
<= rt[1] + 2]
|
||||
v_s = [v for v in v_segments if v[1] > lb[1] - 2 and
|
||||
v[3] < rt[1] + 2 and lb[0] - 2 <= v[0] <= rt[0] + 2]
|
||||
h_s = [h for h in h_segments if h[0] > lb[0] - 2 and
|
||||
h[2] < rt[0] + 2 and lb[1] - 2 <= h[1] <= rt[1] + 2]
|
||||
return text_bbox, v_s, h_s
|
||||
|
||||
|
||||
def remove_close_values(ar, mtol=2):
|
||||
"""Removes values which are within a tolerance of mtol of another value
|
||||
present in list.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ar : list
|
||||
|
||||
mtol : int
|
||||
(optional, default: 2)
|
||||
|
||||
Returns
|
||||
-------
|
||||
ret : list
|
||||
"""
|
||||
ret = []
|
||||
for a in ar:
|
||||
if not ret:
|
||||
ret.append(a)
|
||||
else:
|
||||
temp = ret[-1]
|
||||
if np.isclose(temp, a, atol=mtol):
|
||||
pass
|
||||
else:
|
||||
ret.append(a)
|
||||
return ret
|
||||
|
||||
|
||||
def merge_close_values(ar, mtol=2):
|
||||
"""Merges values which are within a tolerance of mtol by calculating
|
||||
a moving mean.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ar : list
|
||||
|
||||
mtol : int
|
||||
(optional, default: 2)
|
||||
|
||||
Returns
|
||||
-------
|
||||
ret : list
|
||||
"""
|
||||
ret = []
|
||||
for a in ar:
|
||||
if not ret:
|
||||
ret.append(a)
|
||||
else:
|
||||
temp = ret[-1]
|
||||
if np.isclose(temp, a, atol=mtol):
|
||||
temp = (temp + a) / 2.0
|
||||
ret[-1] = temp
|
||||
else:
|
||||
ret.append(a)
|
||||
return ret
|
||||
|
||||
|
||||
def get_row_index(t, rows):
|
||||
"""Gets index of the row in which the given object falls by
|
||||
comparing their co-ordinates.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
t : object
|
||||
|
||||
rows : list
|
||||
|
||||
Returns
|
||||
-------
|
||||
r : int
|
||||
"""
|
||||
for r in range(len(rows)):
|
||||
if (t.y0 + t.y1) / 2.0 < rows[r][0] and (t.y0 + t.y1) / 2.0 > rows[r][1]:
|
||||
return r
|
||||
|
||||
|
||||
def get_column_index(t, columns):
|
||||
"""Gets index of the column in which the given object falls by
|
||||
comparing their co-ordinates.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
t : object
|
||||
|
||||
columns : list
|
||||
|
||||
Returns
|
||||
-------
|
||||
c : int
|
||||
"""
|
||||
for c in range(len(columns)):
|
||||
if (t.x0 + t.x1) / 2.0 > columns[c][0] and (t.x0 + t.x1) / 2.0 < columns[c][1]:
|
||||
return c
|
||||
|
||||
|
||||
def reduce_index(t, rotated, r_idx, c_idx):
|
||||
"""Reduces index of a text object if it lies within a spanning
|
||||
cell taking in account table rotation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
t : object
|
||||
|
||||
rotated : string
|
||||
|
||||
r_idx : int
|
||||
|
||||
c_idx : int
|
||||
|
||||
Returns
|
||||
-------
|
||||
r_idx : int
|
||||
|
||||
c_idx : int
|
||||
"""
|
||||
if not rotated:
|
||||
if t.cells[r_idx][c_idx].spanning_h:
|
||||
while not t.cells[r_idx][c_idx].left:
|
||||
c_idx -= 1
|
||||
if t.cells[r_idx][c_idx].spanning_v:
|
||||
while not t.cells[r_idx][c_idx].top:
|
||||
r_idx -= 1
|
||||
elif rotated == 'left':
|
||||
if t.cells[r_idx][c_idx].spanning_h:
|
||||
while not t.cells[r_idx][c_idx].left:
|
||||
c_idx -= 1
|
||||
if t.cells[r_idx][c_idx].spanning_v:
|
||||
while not t.cells[r_idx][c_idx].bottom:
|
||||
r_idx += 1
|
||||
elif rotated == 'right':
|
||||
if t.cells[r_idx][c_idx].spanning_h:
|
||||
while not t.cells[r_idx][c_idx].right:
|
||||
c_idx += 1
|
||||
if t.cells[r_idx][c_idx].spanning_v:
|
||||
while not t.cells[r_idx][c_idx].top:
|
||||
r_idx -= 1
|
||||
return r_idx, c_idx
|
||||
|
||||
|
||||
def outline(t):
|
||||
"""Sets table border edges to True.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
t : object
|
||||
|
||||
Returns
|
||||
-------
|
||||
t : object
|
||||
"""
|
||||
for i in range(len(t.cells)):
|
||||
t.cells[i][0].left = True
|
||||
t.cells[i][len(t.cells[i]) - 1].right = True
|
||||
for i in range(len(t.cells[0])):
|
||||
t.cells[0][i].top = True
|
||||
t.cells[len(t.cells) - 1][i].bottom = True
|
||||
return t
|
||||
|
||||
|
||||
def fill_spanning(t, fill=None):
|
||||
"""Fills spanning cells.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
t : object
|
||||
|
||||
f : string
|
||||
(optional, default: None)
|
||||
|
||||
Returns
|
||||
-------
|
||||
t : object
|
||||
"""
|
||||
if fill == "h":
|
||||
for i in range(len(t.cells)):
|
||||
for j in range(len(t.cells[i])):
|
||||
if t.cells[i][j].get_text().strip() == '':
|
||||
if t.cells[i][j].spanning_h:
|
||||
t.cells[i][j].add_text(t.cells[i][j - 1].get_text())
|
||||
elif fill == "v":
|
||||
for i in range(len(t.cells)):
|
||||
for j in range(len(t.cells[i])):
|
||||
if t.cells[i][j].get_text().strip() == '':
|
||||
if t.cells[i][j].spanning_v:
|
||||
t.cells[i][j].add_text(t.cells[i - 1][j].get_text())
|
||||
elif fill == "hv":
|
||||
for i in range(len(t.cells)):
|
||||
for j in range(len(t.cells[i])):
|
||||
if t.cells[i][j].get_text().strip() == '':
|
||||
if t.cells[i][j].spanning_h:
|
||||
t.cells[i][j].add_text(t.cells[i][j - 1].get_text())
|
||||
elif t.cells[i][j].spanning_v:
|
||||
t.cells[i][j].add_text(t.cells[i - 1][j].get_text())
|
||||
return t
|
||||
|
||||
|
||||
def remove_empty(d):
|
||||
"""Removes empty rows and columns from list of lists.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
d : list
|
||||
|
||||
Returns
|
||||
-------
|
||||
d : list
|
||||
"""
|
||||
for i, row in enumerate(d):
|
||||
if row == [''] * len(row):
|
||||
d.pop(i)
|
||||
d = zip(*d)
|
||||
d = [list(row) for row in d if any(row)]
|
||||
d = zip(*d)
|
||||
return d
|
||||
|
||||
|
||||
def encode_list(ar):
|
||||
ar = [[r.encode('utf-8') for r in row] for row in ar]
|
||||
return ar
|
||||
Reference in New Issue
Block a user