Briq/camelot-py
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Don't let processes modify instance attributes

Browse Source
This commit is contained in:
Vinayak Mehta
2017-02-07 22:13:33 +05:30
parent 970256e19d
commit bc86346154
5 changed files with 77 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
camelot/lattice.py
+12 -10
View File
@@ -1,5 +1,6 @@
from __future__ import division
import os
import sys
import types
import logging
import copy_reg
@@ -15,6 +16,8 @@ from .utils import (scale_to_pdf, scale_to_image, segments_bbox, text_in_bbox,
__all__ = ['Lattice']
logger = logging.getLogger("app_logger")
def _reduce_method(m):
if m.im_self is None:
@@ -209,8 +212,9 @@ class Lattice:
ltchar = get_text_objects(layout, ltype="char")
width, height = dim
bname, __ = os.path.splitext(pdfname)
logger.info('Parsing tables from {0}.'.format(bname))
if not ltchar:
logging.warning("{0}: PDF has no text. It may be an image.".format(
logger.warning("{0}: PDF has no text. It may be an image.".format(
os.path.basename(bname)))
return {os.path.basename(bname): None}
@@ -265,7 +269,7 @@ class Lattice:
table_bbox = find_table_joints(contours, vmask, hmask)
if len(self.mtol) == 1 and self.mtol[0] == 2:
self.mtol = self.mtol * len(table_bbox)
mtolerance = self.mtol * len(table_bbox)
if self.debug:
self.debug_images = (img, table_bbox)
@@ -279,9 +283,8 @@ class Lattice:
page = {}
tables = {}
table_no = 0
# sort tables based on y-coord
for k in sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True):
for table_no, k in enumerate(sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True)):
# select elements which lie within table_bbox
table_data = {}
t_bbox = {}
@@ -297,9 +300,9 @@ class Lattice:
cols.extend([k[0], k[2]])
rows.extend([k[1], k[3]])
# sort horizontal and vertical segments
cols = merge_close_values(sorted(cols), mtol=self.mtol[table_no])
cols = merge_close_values(sorted(cols), mtol=mtolerance[table_no])
rows = merge_close_values(
sorted(rows, reverse=True), mtol=self.mtol[table_no])
sorted(rows, reverse=True), mtol=mtolerance[table_no])
# make grid using x and y coord of shortlisted rows and cols
cols = [(cols[i], cols[i + 1])
for i in range(0, len(cols) - 1)]
@@ -309,9 +312,9 @@ class Lattice:
if self.headers is not None and self.headers[table_no] != [""]:
self.headers[table_no] = self.headers[table_no].split(',')
if len(self.headers[table_no]) != len(cols):
logging.warning("Length of header ({0}) specified for table is not"
" equal to the number of columns ({1}) detected.".format(
len(self.headers[table_no]), len(cols)))
logger.warning("Length of header ({0}) specified for table is not"
" equal to the number of columns ({1}) detected.".format(
len(self.headers[table_no]), len(cols)))
while len(self.headers[table_no]) != len(cols):
self.headers[table_no].append('')
@@ -361,7 +364,6 @@ class Lattice:
table_data['nrows'] = len(ar)
table_data['ncols'] = len(ar[0])
tables['table-{0}'.format(table_no + 1)] = table_data
table_no += 1
page[os.path.basename(bname)] = tables
if self.debug:
camelot/pdf.py
+7 -2
View File
@@ -1,5 +1,6 @@
import os
import shutil
import logging
import tempfile
import itertools
import multiprocessing as mp
@@ -12,6 +13,8 @@ from .utils import get_page_layout, get_text_objects, get_rotation
__all__ = ['Pdf']
logger = logging.getLogger("app_logger")
def _parse_page_numbers(pagenos):
"""Converts list of dicts to list of ints.
@@ -72,14 +75,16 @@ class Pdf:
raise TypeError("Only PDF format is supported right now.")
self.pagenos = _parse_page_numbers(pagenos)
self.parallel = parallel
self.cpu_count = mp.cpu_count()
self.pool = mp.Pool(processes=self.cpu_count)
if self.parallel:
self.cpu_count = mp.cpu_count()
self.pool = mp.Pool(processes=self.cpu_count)
self.clean = clean
self.temp = tempfile.mkdtemp()
def split(self):
"""Splits file into single page pdfs.
"""
logger.info('Splitting pages...')
infile = PdfFileReader(open(self.pdfname, 'rb'), strict=False)
for p in self.pagenos:
sp_path = os.path.join(self.temp, 'page-{0}.pdf'.format(p))
camelot/stream.py
+16 -15
View File
@@ -13,6 +13,8 @@ from .utils import (text_in_bbox, get_table_index, get_score, count_empty,
__all__ = ['Stream']
logger = logging.getLogger("app_logger")
def _reduce_method(m):
if m.im_self is None:
@@ -299,8 +301,9 @@ class Stream:
ltchar = get_text_objects(layout, ltype="char")
width, height = dim
bname, __ = os.path.splitext(pdfname)
logger.info('Parsing tables from {0}'.format(bname))
if not lttextlh:
logging.warning("{0}: PDF has no text. It may be an image.".format(
logger.warning("{0}: PDF has no text. It may be an image.".format(
os.path.basename(bname)))
return {os.path.basename(bname): None}
@@ -329,15 +332,14 @@ class Stream:
table_bbox = {(0, 0, width, height): None}
if len(self.ytol) == 1 and self.ytol[0] == 2:
self.ytol = self.ytol * len(table_bbox)
ytolerance = self.ytol * len(table_bbox)
if len(self.mtol) == 1 and self.mtol[0] == 0:
self.mtol = self.mtol * len(table_bbox)
mtolerance = self.mtol * len(table_bbox)
page = {}
tables = {}
table_no = 0
# sort tables based on y-coord
for k in sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True):
for table_no, k in enumerate(sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True)):
# select elements which lie within table_bbox
table_data = {}
t_bbox = {}
@@ -348,7 +350,7 @@ class Stream:
for direction in t_bbox:
t_bbox[direction].sort(key=lambda x: (-x.y0, x.x0))
text_x_min, text_y_min, text_x_max, text_y_max = _text_bbox(t_bbox)
rows_grouped = _group_rows(t_bbox['horizontal'], ytol=self.ytol[table_no])
rows_grouped = _group_rows(t_bbox['horizontal'], ytol=ytolerance[table_no])
rows = _join_rows(rows_grouped, text_y_max, text_y_min)
elements = [len(r) for r in rows_grouped]
@@ -369,13 +371,13 @@ class Stream:
len_non_mode = len(filter(lambda x: x != ncols, elements))
if ncols == 1 and not self.debug:
# no tables detected
logging.warning("{}: Only one column was detected, the pdf"
" may have no tables. Specify ncols if"
" the pdf has tables.".format(
logger.warning("{}: Only one column was detected, the pdf"
" may have no tables. Specify ncols if"
" the pdf has tables.".format(
os.path.basename(bname)))
cols = [(t.x0, t.x1)
for r in rows_grouped if len(r) == ncols for t in r]
cols = _merge_columns(sorted(cols), mtol=self.mtol[table_no])
cols = _merge_columns(sorted(cols), mtol=mtolerance[table_no])
inner_text = []
for i in range(1, len(cols)):
left = cols[i - 1][1]
@@ -387,15 +389,15 @@ class Stream:
for t in t_bbox[direction]
if t.x0 > cols[-1][1] or t.x1 < cols[0][0]]
inner_text.extend(outer_text)
cols = _add_columns(cols, inner_text, self.ytol[table_no])
cols = _add_columns(cols, inner_text, ytolerance[table_no])
cols = _join_columns(cols, text_x_min, text_x_max)
if self.headers is not None and self.headers[table_no] != [""]:
self.headers[table_no] = self.headers[table_no].split(',')
if len(self.headers[table_no]) != len(cols):
logging.warning("Length of header ({0}) specified for table is not"
" equal to the number of columns ({1}) detected.".format(
len(self.headers[table_no]), len(cols)))
logger.warning("Length of header ({0}) specified for table is not"
" equal to the number of columns ({1}) detected.".format(
len(self.headers[table_no]), len(cols)))
while len(self.headers[table_no]) != len(cols):
self.headers[table_no].append('')
@@ -434,7 +436,6 @@ class Stream:
table_data['nrows'] = len(ar)
table_data['ncols'] = len(ar[0])
tables['table-{0}'.format(table_no + 1)] = table_data
table_no += 1
page[os.path.basename(bname)] = tables
return page
camelot/utils.py
+27
View File
@@ -178,6 +178,33 @@ def scale_to_pdf(tables, v_segments, h_segments, factors):
return tables_new, v_segments_new, h_segments_new
def setup_logging(log_filepath):
"""Setup logging
Args:
log_filepath (string): Path to log file
Returns:
logging.Logger: Logger object
"""
logger = logging.getLogger("app_logger")
logger.setLevel(logging.DEBUG)
# Log File Handler (Associating one log file per webservice run)
log_file_handler = logging.FileHandler(log_filepath,
mode='a',
encoding='utf-8')
log_file_handler.setLevel(logging.DEBUG)
format_string = '%(asctime)s - %(levelname)s - %(funcName)s - %(message)s'
formatter = logging.Formatter(format_string, datefmt='%Y-%m-%dT%H:%M:%S')
log_file_handler.setFormatter(formatter)
logger.addHandler(log_file_handler)
# Stream Log Handler (For console)
stream_log_handler = logging.StreamHandler()
stream_log_handler.setLevel(logging.INFO)
formatter = logging.Formatter(format_string, datefmt='%Y-%m-%dT%H:%M:%S')
stream_log_handler.setFormatter(formatter)
logger.addHandler(stream_log_handler)
return logger
def get_rotation(lttextlh, lttextlv, ltchar):
"""Detects if text in table is vertical or not using the current
transformation matrix (CTM) and returns its orientation.