[ENH] Add OCR and better joint detection
* Add iterations for dilation * Add OCRLattice and OCRStream * Add debug
This commit is contained in:
Vinayak Mehta
+124
-33
@@ -18,7 +18,7 @@ from PyPDF2 import PdfFileReader
|
||||
from camelot.pdf import Pdf
|
||||
from camelot.lattice import Lattice
|
||||
from camelot.stream import Stream
|
||||
from camelot.ocr import OCR
|
||||
from camelot.ocr import OCRLattice, OCRStream
|
||||
from camelot import utils
|
||||
|
||||
|
||||
@@ -54,7 +54,8 @@ options:
|
||||
camelot methods:
|
||||
lattice Looks for lines between data.
|
||||
stream Looks for spaces between data.
|
||||
ocr Looks for lines in image based pdfs.
|
||||
ocrl Lattice, but for images.
|
||||
ocrs Stream, but for images.
|
||||
|
||||
See 'camelot <method> -h' for more information on a specific method.
|
||||
"""
|
||||
@@ -63,20 +64,22 @@ lattice_doc = """
|
||||
Lattice method looks for lines between text to form a table.
|
||||
|
||||
usage:
|
||||
camelot lattice [-t <tarea>...] [-F <fill>...] [-H <header>...]
|
||||
[-m <mtol>...] [options] [--] <file>
|
||||
camelot lattice [-t <tarea>...] [-F <fill>...] [-m <mtol>...]
|
||||
[-j <jtol>...] [options] [--] <file>
|
||||
|
||||
options:
|
||||
-t, --tarea <tarea> Specific table areas to analyze.
|
||||
-F, --fill <fill> Fill data in horizontal and/or vertical spanning
|
||||
cells. Example: -F h, -F v, -F hv
|
||||
-H, --header <header> Specify header for each table.
|
||||
-m, --mtol <mtol> Tolerance to account for when merging lines
|
||||
which are very close. [default: 2]
|
||||
-j, --jtol <jtol> Tolerance to account for when matching line endings
|
||||
with intersections. [default: 2]
|
||||
-b, --blocksize <blocksize> See adaptive threshold doc. [default: 15]
|
||||
-c, --constant <constant> See adaptive threshold doc. [default: -2]
|
||||
-C, --constant <constant> See adaptive threshold doc. [default: -2]
|
||||
-s, --scale <scale> Scaling factor. Large scaling factor leads to
|
||||
smaller lines being detected. [default: 15]
|
||||
-I, --iterations <iterations> Number of iterations for dilation. [default: 2]
|
||||
-i, --invert Invert pdf image to make sure that lines are
|
||||
in foreground.
|
||||
-T, --shift_text <shift_text> Specify where the text in a spanning cell
|
||||
@@ -89,41 +92,61 @@ stream_doc = """
|
||||
Stream method looks for whitespaces between text to form a table.
|
||||
|
||||
usage:
|
||||
camelot stream [-t <tarea>...] [-c <columns>...] [-H <header>...]
|
||||
[-y <ytol>...] [-m <mtol>...] [options] [--] <file>
|
||||
camelot stream [-t <tarea>...] [-c <columns>...] [-m <mtol>...]
|
||||
[-y <ytol>...] [options] [--] <file>
|
||||
|
||||
options:
|
||||
-t, --tarea <tarea> Specific table areas to analyze.
|
||||
-c, --columns <columns> Comma-separated list of column x-coordinates.
|
||||
Example: -c 10.1,20.2,30.3
|
||||
-H, --header <header> Specify header for each table.
|
||||
-y, --ytol <ytol> Tolerance to account for when grouping rows
|
||||
together. [default: 2]
|
||||
-m, --mtol <mtol> Tolerance to account for when merging columns
|
||||
together. [default: 0]
|
||||
-y, --ytol <ytol> Tolerance to account for when grouping rows
|
||||
together. [default: 2]
|
||||
-d, --debug Debug by visualizing textboxes.
|
||||
"""
|
||||
|
||||
|
||||
ocr_doc = """
|
||||
OCR method looks for lines in image based pdfs.
|
||||
ocrl_doc = """
|
||||
Lattice, but for images.
|
||||
|
||||
usage:
|
||||
camelot ocr [-t <tarea>] [-m <mtol>] [options] [--] <file>
|
||||
camelot ocrl [-t <tarea>...] [-m <mtol>...] [options] [--] <file>
|
||||
|
||||
options:
|
||||
-t, --tarea <tarea> Specific table areas to analyze.
|
||||
-m, --mtol <mtol> Tolerance to account for when merging lines
|
||||
which are very close. [default: 2]
|
||||
-b, --blocksize <blocksize> See adaptive threshold doc. [default: 15]
|
||||
-c, --constant <constant> See adaptive threshold doc. [default: -2]
|
||||
-D, --dpi <dpi> Dots per inch, specify image quality to be used for OCR.
|
||||
[default: 300]
|
||||
-l, --lang <lang> Specify language to be used for OCR. [default: eng]
|
||||
-s, --scale <scale> Scaling factor. Large scaling factor leads to
|
||||
smaller lines being detected. [default: 15]
|
||||
-d, --debug <debug> Debug by visualizing pdf geometry.
|
||||
(contour,line,joint,table) Example: -d table
|
||||
-t, --tarea <tarea> Specific table areas to analyze.
|
||||
-m, --mtol <mtol> Tolerance to account for when merging lines
|
||||
which are very close. [default: 2]
|
||||
-b, --blocksize <blocksize> See adaptive threshold doc. [default: 15]
|
||||
-C, --constant <constant> See adaptive threshold doc. [default: -2]
|
||||
-D, --dpi <dpi> Dots per inch, specify image quality to be used for OCR.
|
||||
[default: 300]
|
||||
-l, --lang <lang> Specify language to be used for OCR. [default: eng]
|
||||
-s, --scale <scale> Scaling factor. Large scaling factor leads to
|
||||
smaller lines being detected. [default: 15]
|
||||
-I, --iterations <iterations> Number of iterations for dilation. [default: 2]
|
||||
-d, --debug <debug> Debug by visualizing pdf geometry.
|
||||
(contour,line,joint,table) Example: -d table
|
||||
"""
|
||||
|
||||
ocrs_doc = """
|
||||
Stream, but for images.
|
||||
|
||||
usage:
|
||||
camelot ocrs [-t <tarea>...] [-c <columns>...] [options] [--] <file>
|
||||
|
||||
options:
|
||||
-t, --tarea <tarea> Specific table areas to analyze.
|
||||
-c, --columns <columns> Comma-separated list of column x-coordinates.
|
||||
Example: -c 10.1,20.2,30.3
|
||||
-b, --blocksize <blocksize> See adaptive threshold doc. [default: 15]
|
||||
-C, --constant <constant> See adaptive threshold doc. [default: -2]
|
||||
-N, --line-threshold <line_threshold> Maximum intensity of projections on y-axis.
|
||||
[default: 100]
|
||||
-D, --dpi <dpi> Dots per inch, specify image quality to be used for OCR.
|
||||
[default: 300]
|
||||
-l, --lang <lang> Specify language to be used for OCR. [default: eng]
|
||||
-d, --debug Debug by visualizing image.
|
||||
"""
|
||||
|
||||
|
||||
@@ -351,8 +374,10 @@ if __name__ == '__main__':
|
||||
args.update(docopt(lattice_doc, argv=argv))
|
||||
elif args['<method>'] == 'stream':
|
||||
args.update(docopt(stream_doc, argv=argv))
|
||||
elif args['<method>'] == 'ocr':
|
||||
args.update(docopt(ocr_doc, argv=argv))
|
||||
elif args['<method>'] == 'ocrl':
|
||||
args.update(docopt(ocrl_doc, argv=argv))
|
||||
elif args['<method>'] == 'ocrs':
|
||||
args.update(docopt(ocrs_doc, argv=argv))
|
||||
|
||||
filename = args['<file>']
|
||||
filedir = os.path.dirname(args['<file>'])
|
||||
@@ -392,11 +417,12 @@ if __name__ == '__main__':
|
||||
kwargs = {
|
||||
'table_area': args['--tarea'] if args['--tarea'] else None,
|
||||
'fill': args['--fill'] if args['--fill'] else None,
|
||||
'headers': args['--header'] if args['--header'] else None,
|
||||
'mtol': [int(m) for m in args['--mtol']],
|
||||
'jtol': [int(j) for j in args['--jtol']],
|
||||
'blocksize': int(args['--blocksize']),
|
||||
'threshold_constant': float(args['--constant']),
|
||||
'scale': int(args['--scale']),
|
||||
'iterations': int(args['--iterations']),
|
||||
'invert': args['--invert'],
|
||||
'margins': margins,
|
||||
'split_text': args['--split_text'],
|
||||
@@ -462,7 +488,6 @@ if __name__ == '__main__':
|
||||
kwargs = {
|
||||
'table_area': args['--tarea'] if args['--tarea'] else None,
|
||||
'columns': args['--columns'] if args['--columns'] else None,
|
||||
'headers': args['--header'] if args['--header'] else None,
|
||||
'ytol': [int(y) for y in args['--ytol']],
|
||||
'mtol': [int(m) for m in args['--mtol']],
|
||||
'margins': margins,
|
||||
@@ -522,7 +547,7 @@ if __name__ == '__main__':
|
||||
except Exception as e:
|
||||
logger.exception(e.message, exc_info=True)
|
||||
sys.exit()
|
||||
elif args['<method>'] == 'ocr':
|
||||
elif args['<method>'] == 'ocrl':
|
||||
try:
|
||||
kwargs = {
|
||||
'table_area': args['--tarea'] if args['--tarea'] else None,
|
||||
@@ -532,9 +557,75 @@ if __name__ == '__main__':
|
||||
'dpi': int(args['--dpi']),
|
||||
'lang': args['--lang'],
|
||||
'scale': int(args['--scale']),
|
||||
'iterations': int(args['--iterations']),
|
||||
'debug': args['--debug']
|
||||
}
|
||||
manager = Pdf(OCR(**kwargs), filename, pagenos=p, clean=True,
|
||||
manager = Pdf(OCRLattice(**kwargs), filename, pagenos=p, clean=True,
|
||||
parallel=args['--parallel'])
|
||||
data = manager.extract()
|
||||
|
||||
processing_time = time.time() - start_time
|
||||
logger.info("Finished processing in " + str(processing_time) + " seconds")
|
||||
|
||||
if args['--plot']:
|
||||
if args['--output']:
|
||||
pngname = os.path.join(args['--output'], os.path.basename(pngname))
|
||||
plot_type = args['--plot'].split(',')
|
||||
if 'page' in plot_type:
|
||||
for page_number in sorted(data.keys(), key=lambda x: int(x[5:])):
|
||||
page = data[page_number]
|
||||
for table_number in sorted(page.keys(), key=lambda x: int(x[6:])):
|
||||
table = page[table_number]
|
||||
plot_table_barchart(table['r_nempty_cells'],
|
||||
table['c_nempty_cells'],
|
||||
table['empty_p'],
|
||||
page_number,
|
||||
table_number)
|
||||
|
||||
if 'all' in plot_type:
|
||||
plot_all_barchart(data, pngname)
|
||||
|
||||
if 'rc' in plot_type:
|
||||
plot_rc_piechart(data, pngname)
|
||||
|
||||
if args['--print-stats']:
|
||||
print_stats(data, processing_time)
|
||||
|
||||
if args['--save-stats']:
|
||||
if args['--output']:
|
||||
scorename = os.path.join(args['--output'], os.path.basename(scorename))
|
||||
with open(scorename, 'w') as score_file:
|
||||
score_file.write('table,nrows,ncols,empty_p,line_p,text_p,score\n')
|
||||
for page_number in sorted(data.keys(), key=lambda x: int(x[5:])):
|
||||
page = data[page_number]
|
||||
for table_number in sorted(page.keys(), key=lambda x: int(x[6:])):
|
||||
table = page[table_number]
|
||||
score_file.write('{0},{1},{2},{3},{4},{5},{6}\n'.format(
|
||||
''.join([page_number, '_', table_number]),
|
||||
table['nrows'],
|
||||
table['ncols'],
|
||||
table['empty_p'],
|
||||
table['line_p'],
|
||||
table['text_p'],
|
||||
table['score']))
|
||||
if args['--debug']:
|
||||
manager.debug_plot()
|
||||
except Exception as e:
|
||||
logger.exception(e.message, exc_info=True)
|
||||
sys.exit()
|
||||
elif args['<method>'] == 'ocrs':
|
||||
try:
|
||||
kwargs = {
|
||||
'table_area': args['--tarea'] if args['--tarea'] else None,
|
||||
'columns': args['--columns'] if args['--columns'] else None,
|
||||
'blocksize': int(args['--blocksize']),
|
||||
'threshold_constant': float(args['--constant']),
|
||||
'line_threshold': int(args['--line-threshold']),
|
||||
'dpi': int(args['--dpi']),
|
||||
'lang': args['--lang'],
|
||||
'debug': args['--debug']
|
||||
}
|
||||
manager = Pdf(OCRStream(**kwargs), filename, pagenos=p, clean=True,
|
||||
parallel=args['--parallel'])
|
||||
data = manager.extract()
|
||||
|
||||
@@ -588,7 +679,7 @@ if __name__ == '__main__':
|
||||
logger.exception(e.message, exc_info=True)
|
||||
sys.exit()
|
||||
|
||||
if args['--debug']:
|
||||
if args.get('--debug') is not None and args['--debug']:
|
||||
print("See 'camelot <method> -h' for various parameters you can tweak.")
|
||||
else:
|
||||
output = filedir if args['--output'] is None else args['--output']
|
||||
|
||||
Reference in New Issue
Block a user