Fix merge conflict

camelot/__init__.py

@@ -2,10 +2,21 @@
 
 import logging
 
+from click import HelpFormatter
+
 from .__version__ import __version__
 from .io import read_pdf
+from .plotting import PlotMethods
 
 
+def _write_usage(self, prog, args='', prefix='Usage: '):
+    return self._write_usage('camelot', args, prefix=prefix)
+
+
+# monkey patch click.HelpFormatter
+HelpFormatter._write_usage = HelpFormatter.write_usage
+HelpFormatter.write_usage = _write_usage
+
 # set up logging
 logger = logging.getLogger('camelot')
 
@@ -15,3 +26,6 @@ handler = logging.StreamHandler()
 handler.setFormatter(formatter)
 
 logger.addHandler(handler)
+
+# instantiate plot method
+plot = PlotMethods()

camelot/__main__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+
+__all__ = ('main',)
+
+
+def main():
+    from camelot.cli import cli
+
+    cli()
+
+
+if __name__ == "__main__":
+    main()

camelot/__version__.py

@@ -1,11 +1,23 @@
 # -*- coding: utf-8 -*-
 
-VERSION = (0, 2, 1)
+VERSION = (0, 5, 0)
+PRERELEASE = None # alpha, beta or rc
+REVISION = None
+
+
+def generate_version(version, prerelease=None, revision=None):
+    version_parts = ['.'.join(map(str, version))]
+    if prerelease is not None:
+        version_parts.append('-{}'.format(prerelease))
+    if revision is not None:
+        version_parts.append('.{}'.format(revision))
+    return ''.join(version_parts)
+
 
 __title__ = 'camelot-py'
 __description__ = 'PDF Table Extraction for Humans.'
 __url__ = 'http://camelot-py.readthedocs.io/'
-__version__ = '.'.join(map(str, VERSION))
+__version__ = generate_version(VERSION, prerelease=PRERELEASE, revision=REVISION)
 __author__ = 'Vinayak Mehta'
 __author_email__ = 'vmehta94@gmail.com'
 __license__ = 'MIT License'

camelot/cli.py

@@ -3,9 +3,14 @@
 import logging
 
 import click
+try:
+    import matplotlib.pyplot as plt
+except ImportError:
+    _HAS_MPL = False
+else:
+    _HAS_MPL = True
 
-from . import __version__
-from .io import read_pdf
+from . import __version__, read_pdf, plot
 
 
 logger = logging.getLogger('camelot')
@@ -25,8 +30,10 @@ pass_config = click.make_pass_decorator(Config)
 
 @click.group()
 @click.version_option(version=__version__)
+@click.option('-q', '--quiet', is_flag=False, help='Suppress logs and warnings.')
 @click.option('-p', '--pages', default='1', help='Comma-separated page numbers.'
               ' Example: 1,3,4 or 1,4-end.')
+@click.option('-pw', '--password', help='Password for decryption.')
 @click.option('-o', '--output', help='Output file path.')
 @click.option('-f', '--format',
               type=click.Choice(['csv', 'json', 'excel', 'html']),
@@ -47,7 +54,7 @@ def cli(ctx, *args, **kwargs):
 
 
 @cli.command('lattice')
-@click.option('-T', '--table_area', default=[], multiple=True,
+@click.option('-T', '--table_areas', default=[], multiple=True,
               help='Table areas to process. Example: x1,y1,x2,y2'
               ' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
 @click.option('-back', '--process_background', is_flag=True,
@@ -78,8 +85,8 @@ def cli(ctx, *args, **kwargs):
 @click.option('-I', '--iterations', default=0,
               help='Number of times for erosion/dilation will be applied.')
 @click.option('-plot', '--plot_type',
-              type=click.Choice(['text', 'table', 'contour', 'joint', 'line']),
-              help='Plot geometry found on PDF page, for debugging.')
+              type=click.Choice(['text', 'grid', 'contour', 'joint', 'line']),
+              help='Plot elements found on PDF page for visual debugging.')
 @click.argument('filepath', type=click.Path(exists=True))
 @pass_config
 def lattice(c, *args, **kwargs):
@@ -89,31 +96,39 @@ def lattice(c, *args, **kwargs):
     output = conf.pop('output')
     f = conf.pop('format')
     compress = conf.pop('zip')
+    quiet = conf.pop('quiet')
     plot_type = kwargs.pop('plot_type')
     filepath = kwargs.pop('filepath')
     kwargs.update(conf)
 
-    table_area = list(kwargs['table_area'])
-    kwargs['table_area'] = None if not table_area else table_area
+    table_areas = list(kwargs['table_areas'])
+    kwargs['table_areas'] = None if not table_areas else table_areas
     copy_text = list(kwargs['copy_text'])
     kwargs['copy_text'] = None if not copy_text else copy_text
     kwargs['shift_text'] = list(kwargs['shift_text'])
 
-    tables = read_pdf(filepath, pages=pages, flavor='lattice', **kwargs)
-    click.echo('Found {} tables'.format(tables.n))
     if plot_type is not None:
-        for table in tables:
-            table.plot(plot_type)
+        if not _HAS_MPL:
+            raise ImportError('matplotlib is required for plotting.')
     else:
         if output is None:
             raise click.UsageError('Please specify output file path using --output')
         if f is None:
             raise click.UsageError('Please specify output file format using --format')
+
+    tables = read_pdf(filepath, pages=pages, flavor='lattice',
+                      suppress_stdout=quiet, **kwargs)
+    click.echo('Found {} tables'.format(tables.n))
+    if plot_type is not None:
+        for table in tables:
+            plot(table, kind=plot_type)
+            plt.show()
+    else:
         tables.export(output, f=f, compress=compress)
 
 
 @cli.command('stream')
-@click.option('-T', '--table_area', default=[], multiple=True,
+@click.option('-T', '--table_areas', default=[], multiple=True,
               help='Table areas to process. Example: x1,y1,x2,y2'
               ' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
 @click.option('-C', '--columns', default=[], multiple=True,
@@ -123,8 +138,8 @@ def lattice(c, *args, **kwargs):
 @click.option('-c', '--col_close_tol', default=0, help='Tolerance parameter'
               ' used to combine text horizontally, to generate columns.')
 @click.option('-plot', '--plot_type',
-              type=click.Choice(['text', 'table']),
-              help='Plot geometry found on PDF page for debugging.')
+              type=click.Choice(['text', 'grid', 'contour', 'textedge']),
+              help='Plot elements found on PDF page for visual debugging.')
 @click.argument('filepath', type=click.Path(exists=True))
 @pass_config
 def stream(c, *args, **kwargs):
@@ -134,23 +149,31 @@ def stream(c, *args, **kwargs):
     output = conf.pop('output')
     f = conf.pop('format')
     compress = conf.pop('zip')
+    quiet = conf.pop('quiet')
     plot_type = kwargs.pop('plot_type')
     filepath = kwargs.pop('filepath')
     kwargs.update(conf)
 
-    table_area = list(kwargs['table_area'])
-    kwargs['table_area'] = None if not table_area else table_area
+    table_areas = list(kwargs['table_areas'])
+    kwargs['table_areas'] = None if not table_areas else table_areas
     columns = list(kwargs['columns'])
     kwargs['columns'] = None if not columns else columns
 
-    tables = read_pdf(filepath, pages=pages, flavor='stream', **kwargs)
-    click.echo('Found {} tables'.format(tables.n))
     if plot_type is not None:
-        for table in tables:
-            table.plot(plot_type)
+        if not _HAS_MPL:
+            raise ImportError('matplotlib is required for plotting.')
     else:
         if output is None:
             raise click.UsageError('Please specify output file path using --output')
         if f is None:
             raise click.UsageError('Please specify output file format using --format')
+
+    tables = read_pdf(filepath, pages=pages, flavor='stream',
+                      suppress_stdout=quiet, **kwargs)
+    click.echo('Found {} tables'.format(tables.n))
+    if plot_type is not None:
+        for table in tables:
+            plot(table, kind=plot_type)
+            plt.show()
+    else:
         tables.export(output, f=f, compress=compress)

camelot/core.py

@@ -3,11 +3,208 @@
 import os
 import zipfile
 import tempfile
+from itertools import chain
+from operator import itemgetter
 
 import numpy as np
 import pandas as pd
 
-from .plotting import *
+
+# minimum number of vertical textline intersections for a textedge
+# to be considered valid
+TEXTEDGE_REQUIRED_ELEMENTS = 4
+# y coordinate tolerance for extending textedge
+TEXTEDGE_EXTEND_TOLERANCE = 50
+# padding added to table area on the left, right and bottom
+TABLE_AREA_PADDING = 10
+
+
+class TextEdge(object):
+    """Defines a text edge coordinates relative to a left-bottom
+    origin. (PDF coordinate space)
+
+    Parameters
+    ----------
+    x : float
+        x-coordinate of the text edge.
+    y0 : float
+        y-coordinate of bottommost point.
+    y1 : float
+        y-coordinate of topmost point.
+    align : string, optional (default: 'left')
+        {'left', 'right', 'middle'}
+
+    Attributes
+    ----------
+    intersections: int
+        Number of intersections with horizontal text rows.
+    is_valid: bool
+        A text edge is valid if it intersections with at least
+        TEXTEDGE_REQUIRED_ELEMENTS horizontal text rows.
+
+    """
+    def __init__(self, x, y0, y1, align='left'):
+        self.x = x
+        self.y0 = y0
+        self.y1 = y1
+        self.align = align
+        self.intersections = 0
+        self.is_valid = False
+
+    def __repr__(self):
+        return '<TextEdge x={} y0={} y1={} align={} valid={}>'.format(
+            round(self.x, 2), round(self.y0, 2), round(self.y1, 2), self.align, self.is_valid)
+
+    def update_coords(self, x, y0):
+        """Updates the text edge's x and bottom y coordinates and sets
+        the is_valid attribute.
+        """
+        if np.isclose(self.y0, y0, atol=TEXTEDGE_EXTEND_TOLERANCE):
+            self.x = (self.intersections * self.x + x) / float(self.intersections + 1)
+            self.y0 = y0
+            self.intersections += 1
+            # a textedge is valid only if it extends uninterrupted
+            # over a required number of textlines
+            if self.intersections > TEXTEDGE_REQUIRED_ELEMENTS:
+                self.is_valid = True
+
+
+class TextEdges(object):
+    """Defines a dict of left, right and middle text edges found on
+    the PDF page. The dict has three keys based on the alignments,
+    and each key's value is a list of camelot.core.TextEdge objects.
+    """
+    def __init__(self):
+        self._textedges = {'left': [], 'right': [], 'middle': []}
+
+    @staticmethod
+    def get_x_coord(textline, align):
+        """Returns the x coordinate of a text row based on the
+        specified alignment.
+        """
+        x_left = textline.x0
+        x_right = textline.x1
+        x_middle = x_left + (x_right - x_left) / 2.0
+        x_coord = {'left': x_left, 'middle': x_middle, 'right': x_right}
+        return x_coord[align]
+
+    def find(self, x_coord, align):
+        """Returns the index of an existing text edge using
+        the specified x coordinate and alignment.
+        """
+        for i, te in enumerate(self._textedges[align]):
+            if np.isclose(te.x, x_coord, atol=0.5):
+                return i
+        return None
+
+    def add(self, textline, align):
+        """Adds a new text edge to the current dict.
+        """
+        x = self.get_x_coord(textline, align)
+        y0 = textline.y0
+        y1 = textline.y1
+        te = TextEdge(x, y0, y1, align=align)
+        self._textedges[align].append(te)
+
+    def update(self, textline):
+        """Updates an existing text edge in the current dict.
+        """
+        for align in ['left', 'right', 'middle']:
+            x_coord = self.get_x_coord(textline, align)
+            idx = self.find(x_coord, align)
+            if idx is None:
+                self.add(textline, align)
+            else:
+                self._textedges[align][idx].update_coords(x_coord, textline.y0)
+
+    def generate(self, textlines):
+        """Generates the text edges dict based on horizontal text
+        rows.
+        """
+        for tl in textlines:
+            if len(tl.get_text().strip()) > 1: # TODO: hacky
+                self.update(tl)
+
+    def get_relevant(self):
+        """Returns the list of relevant text edges (all share the same
+        alignment) based on which list intersects horizontal text rows
+        the most.
+        """
+        intersections_sum = {
+            'left': sum(te.intersections for te in self._textedges['left'] if te.is_valid),
+            'right': sum(te.intersections for te in self._textedges['right'] if te.is_valid),
+            'middle': sum(te.intersections for te in self._textedges['middle'] if te.is_valid)
+        }
+
+        # TODO: naive
+        # get vertical textedges that intersect maximum number of
+        # times with horizontal textlines
+        relevant_align = max(intersections_sum.items(), key=itemgetter(1))[0]
+        return self._textedges[relevant_align]
+
+    def get_table_areas(self, textlines, relevant_textedges):
+        """Returns a dict of interesting table areas on the PDF page
+        calculated using relevant text edges.
+        """
+        def pad(area, average_row_height):
+            x0 = area[0] - TABLE_AREA_PADDING
+            y0 = area[1] - TABLE_AREA_PADDING
+            x1 = area[2] + TABLE_AREA_PADDING
+            # add a constant since table headers can be relatively up
+            y1 = area[3] + average_row_height * 5
+            return (x0, y0, x1, y1)
+
+        # sort relevant textedges in reading order
+        relevant_textedges.sort(key=lambda te: (-te.y0, te.x))
+
+        table_areas = {}
+        for te in relevant_textedges:
+            if te.is_valid:
+                if not table_areas:
+                    table_areas[(te.x, te.y0, te.x, te.y1)] = None
+                else:
+                    found = None
+                    for area in table_areas:
+                        # check for overlap
+                        if te.y1 >= area[1] and te.y0 <= area[3]:
+                            found = area
+                            break
+                    if found is None:
+                        table_areas[(te.x, te.y0, te.x, te.y1)] = None
+                    else:
+                        table_areas.pop(found)
+                        updated_area = (
+                            found[0], min(te.y0, found[1]), max(found[2], te.x), max(found[3], te.y1))
+                        table_areas[updated_area] = None
+
+        # extend table areas based on textlines that overlap
+        # vertically. it's possible that these textlines were
+        # eliminated during textedges generation since numbers and
+        # chars/words/sentences are often aligned differently.
+        # drawback: table areas that have paragraphs on their sides
+        # will include the paragraphs too.
+        sum_textline_height = 0
+        for tl in textlines:
+            sum_textline_height += tl.y1 - tl.y0
+            found = None
+            for area in table_areas:
+                # check for overlap
+                if tl.y0 >= area[1] and tl.y1 <= area[3]:
+                    found = area
+                    break
+            if found is not None:
+                table_areas.pop(found)
+                updated_area = (
+                    min(tl.x0, found[0]), min(tl.y0, found[1]), max(found[2], tl.x1), max(found[3], tl.y1))
+                table_areas[updated_area] = None
+        average_textline_height = sum_textline_height / float(len(textlines))
+
+        # add some padding to table areas
+        table_areas_padded = {}
+        for area in table_areas:
+            table_areas_padded[pad(area, average_textline_height)] = None
+
+        return table_areas_padded
 
 
 class Cell(object):
@@ -251,7 +448,7 @@ class Table(object):
                         self.cells[L][J].top = True
                         J += 1
             elif i == []:  # only bottom edge
-                I = len(self.rows) - 1
+                L = len(self.rows) - 1
                 if k:
                     K = k[0]
                     while J < K:
@@ -321,33 +518,6 @@ class Table(object):
                     cell.hspan = True
         return self
 
-    def plot(self, geometry_type):
-        """Plot geometry found on PDF page based on geometry_type
-        specified, useful for debugging and playing with different
-        parameters to get the best output.
-
-        Parameters
-        ----------
-        geometry_type : str
-            The geometry type for which a plot should be generated.
-            Can be 'text', 'table', 'contour', 'joint', 'line'
-
-        """
-        if self.flavor == 'stream' and geometry_type in ['contour', 'joint', 'line']:
-            raise NotImplementedError("{} cannot be plotted with flavor='stream'".format(
-                                       geometry_type))
-
-        if geometry_type == 'text':
-            plot_text(self._text)
-        elif geometry_type == 'table':
-            plot_table(self)
-        elif geometry_type == 'contour':
-            plot_contour(self._image)
-        elif geometry_type == 'joint':
-            plot_joint(self._image)
-        elif geometry_type == 'line':
-            plot_line(self._segments)
-
     def to_csv(self, path, **kwargs):
         """Writes Table to a comma-separated values (csv) file.
 

camelot/handlers.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import os
+import sys
 
 from PyPDF2 import PdfFileReader, PdfFileWriter
 
@@ -21,14 +22,22 @@ class PDFHandler(object):
         Path to PDF file.
     pages : str, optional (default: '1')
         Comma-separated page numbers.
-        Example: 1,3,4 or 1,4-end.
+        Example: '1,3,4' or '1,4-end'.
+    password : str, optional (default: None)
+        Password for decryption.
 
     """
-    def __init__(self, filename, pages='1'):
+    def __init__(self, filename, pages='1', password=None):
         self.filename = filename
-        if not self.filename.endswith('.pdf'):
+        if not filename.lower().endswith('.pdf'):
             raise NotImplementedError("File format not supported")
         self.pages = self._get_pages(self.filename, pages)
+        if password is None:
+            self.password = ''
+        else:
+            self.password = password
+            if sys.version_info[0] < 3:
+                self.password = self.password.encode('ascii')
 
     def _get_pages(self, filename, pages):
         """Converts pages string to list of ints.
@@ -52,6 +61,8 @@ class PDFHandler(object):
             page_numbers.append({'start': 1, 'end': 1})
         else:
             infile = PdfFileReader(open(filename, 'rb'), strict=False)
+            if infile.isEncrypted:
+                infile.decrypt(self.password)
             if pages == 'all':
                 page_numbers.append({'start': 1, 'end': infile.getNumPages()})
             else:
@@ -84,7 +95,7 @@ class PDFHandler(object):
         with open(filename, 'rb') as fileobj:
             infile = PdfFileReader(fileobj, strict=False)
             if infile.isEncrypted:
-                infile.decrypt('')
+                infile.decrypt(self.password)
             fpath = os.path.join(temp, 'page-{0}.pdf'.format(page))
             froot, fext = os.path.splitext(fpath)
             p = infile.getPage(page - 1)
@@ -103,7 +114,7 @@ class PDFHandler(object):
                 os.rename(fpath, fpath_new)
                 infile = PdfFileReader(open(fpath_new, 'rb'), strict=False)
                 if infile.isEncrypted:
-                    infile.decrypt('')
+                    infile.decrypt(self.password)
                 outfile = PdfFileWriter()
                 p = infile.getPage(0)
                 if rotation == 'anticlockwise':
@@ -114,7 +125,7 @@ class PDFHandler(object):
                 with open(fpath, 'wb') as f:
                     outfile.write(f)
 
-    def parse(self, flavor='lattice', **kwargs):
+    def parse(self, flavor='lattice', suppress_stdout=False, **kwargs):
         """Extracts tables by calling parser.get_tables on all single
         page PDFs.
 
@@ -123,6 +134,8 @@ class PDFHandler(object):
         flavor : str (default: 'lattice')
             The parsing method to use ('lattice' or 'stream').
             Lattice is used by default.
+        suppress_stdout : str (default: False)
+            Suppress logs and warnings.
         kwargs : dict
             See camelot.read_pdf kwargs.
 
@@ -130,9 +143,6 @@ class PDFHandler(object):
         -------
         tables : camelot.core.TableList
             List of tables found in PDF.
-        geometry : camelot.core.GeometryList
-            List of geometry objects (contours, lines, joints) found
-            in PDF.
 
         """
         tables = []
@@ -143,6 +153,6 @@ class PDFHandler(object):
                      for p in self.pages]
             parser = Lattice(**kwargs) if flavor == 'lattice' else Stream(**kwargs)
             for p in pages:
-                t = parser.extract_tables(p)
+                t = parser.extract_tables(p, suppress_stdout=suppress_stdout)
                 tables.extend(t)
         return TableList(tables)

camelot/io.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
+import warnings
 
 from .handlers import PDFHandler
 from .utils import validate_input, remove_extra
 
 
-def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
+def read_pdf(filepath, pages='1', password=None, flavor='lattice',
+             suppress_stdout=False, **kwargs):
     """Read PDF and return extracted tables.
 
     Note: kwargs annotated with ^ can only be used with flavor='stream'
@@ -16,11 +18,15 @@ def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
         Path to PDF file.
     pages : str, optional (default: '1')
         Comma-separated page numbers.
-        Example: 1,3,4 or 1,4-end.
+        Example: '1,3,4' or '1,4-end'.
+    password : str, optional (default: None)
+        Password for decryption.
     flavor : str (default: 'lattice')
         The parsing method to use ('lattice' or 'stream').
         Lattice is used by default.
-    table_area : list, optional (default: None)
+    suppress_stdout : bool, optional (default: True)
+        Print all logs and warnings.
+    table_areas : list, optional (default: None)
         List of table area strings of the form x1,y1,x2,y2
         where (x1, y1) -> left-top and (x2, y2) -> right-bottom
         in PDF coordinate space.
@@ -85,8 +91,12 @@ def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
         raise NotImplementedError("Unknown flavor specified."
                                   " Use either 'lattice' or 'stream'")
 
-    validate_input(kwargs, flavor=flavor)
-    p = PDFHandler(filepath, pages)
-    kwargs = remove_extra(kwargs, flavor=flavor)
-    tables = p.parse(flavor=flavor, **kwargs)
-    return tables
+    with warnings.catch_warnings():
+        if suppress_stdout:
+            warnings.simplefilter("ignore")
+
+        validate_input(kwargs, flavor=flavor)
+        p = PDFHandler(filepath, pages=pages, password=password)
+        kwargs = remove_extra(kwargs, flavor=flavor)
+        tables = p.parse(flavor=flavor, suppress_stdout=suppress_stdout, **kwargs)
+        return tables

camelot/parsers/lattice.py

@@ -31,7 +31,7 @@ class Lattice(BaseParser):
 
     Parameters
     ----------
-    table_area : list, optional (default: None)
+    table_areas : list, optional (default: None)
         List of table area strings of the form x1,y1,x2,y2
         where (x1, y1) -> left-top and (x2, y2) -> right-bottom
         in PDF coordinate space.
@@ -79,12 +79,12 @@ class Lattice(BaseParser):
         For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
 
     """
-    def __init__(self, table_area=None, process_background=False,
+    def __init__(self, table_areas=None, process_background=False,
                  line_size_scaling=15, copy_text=None, shift_text=['l', 't'],
                  split_text=False, flag_size=False, line_close_tol=2,
                  joint_close_tol=2, threshold_blocksize=15, threshold_constant=-2,
                  iterations=0, margins=(1.0, 0.5, 0.1), **kwargs):
-        self.table_area = table_area
+        self.table_areas = table_areas
         self.process_background = process_background
         self.line_size_scaling = line_size_scaling
         self.copy_text = copy_text
@@ -210,9 +210,9 @@ class Lattice(BaseParser):
             self.threshold, direction='horizontal',
             line_size_scaling=self.line_size_scaling, iterations=self.iterations)
 
-        if self.table_area is not None:
+        if self.table_areas is not None:
             areas = []
-            for area in self.table_area:
+            for area in self.table_areas:
                 x1, y1, x2, y2 = area.split(",")
                 x1 = float(x1)
                 y1 = float(y1)
@@ -237,10 +237,11 @@ class Lattice(BaseParser):
             tk, self.vertical_segments, self.horizontal_segments)
         t_bbox['horizontal'] = text_in_bbox(tk, self.horizontal_text)
         t_bbox['vertical'] = text_in_bbox(tk, self.vertical_text)
-        self.t_bbox = t_bbox
 
-        for direction in t_bbox:
-            t_bbox[direction].sort(key=lambda x: (-x.y0, x.x0))
+        t_bbox['horizontal'].sort(key=lambda x: (-x.y0, x.x0))
+        t_bbox['vertical'].sort(key=lambda x: (x.x0, -x.y0))
+
+        self.t_bbox = t_bbox
 
         cols, rows = zip(*self.table_bbox[tk])
         cols, rows = list(cols), list(rows)
@@ -274,7 +275,9 @@ class Lattice(BaseParser):
         table = table.set_span()
 
         pos_errors = []
-        for direction in self.t_bbox:
+        # TODO: have a single list in place of two directional ones?
+        # sorted on x-coordinate based on reading order i.e. LTR or RTL
+        for direction in ['vertical', 'horizontal']:
             for t in self.t_bbox[direction]:
                 indices, error = get_table_index(
                     table, t, direction, split_text=self.split_text,
@@ -307,12 +310,14 @@ class Lattice(BaseParser):
         table._text = _text
         table._image = (self.image, self.table_bbox_unscaled)
         table._segments = (self.vertical_segments, self.horizontal_segments)
+        table._textedges = None
 
         return table
 
-    def extract_tables(self, filename):
+    def extract_tables(self, filename, suppress_stdout=False):
         self._generate_layout(filename)
-        logger.info('Processing {}'.format(os.path.basename(self.rootname)))
+        if not suppress_stdout:
+            logger.info('Processing {}'.format(os.path.basename(self.rootname)))
 
         if not self.horizontal_text:
             warnings.warn("No tables found on {}".format(
@@ -328,6 +333,7 @@ class Lattice(BaseParser):
                 self.table_bbox.keys(), key=lambda x: x[1], reverse=True)):
             cols, rows, v_s, h_s = self._generate_columns_and_rows(table_idx, tk)
             table = self._generate_table(table_idx, cols, rows, v_s=v_s, h_s=h_s)
+            table._bbox = tk
             _tables.append(table)
 
         return _tables

camelot/parsers/stream.py

@@ -9,7 +9,7 @@ import numpy as np
 import pandas as pd
 
 from .base import BaseParser
-from ..core import Table
+from ..core import TextEdges, Table
 from ..utils import (text_in_bbox, get_table_index, compute_accuracy,
                      compute_whitespace)
 
@@ -26,7 +26,7 @@ class Stream(BaseParser):
 
     Parameters
     ----------
-    table_area : list, optional (default: None)
+    table_areas : list, optional (default: None)
         List of table area strings of the form x1,y1,x2,y2
         where (x1, y1) -> left-top and (x2, y2) -> right-bottom
         in PDF coordinate space.
@@ -50,10 +50,10 @@ class Stream(BaseParser):
         For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
 
     """
-    def __init__(self, table_area=None, columns=None, split_text=False,
+    def __init__(self, table_areas=None, columns=None, split_text=False,
                  flag_size=False, row_close_tol=2, col_close_tol=0,
                  margins=(1.0, 0.5, 0.1), **kwargs):
-        self.table_area = table_area
+        self.table_areas = table_areas
         self.columns = columns
         self._validate_columns()
         self.split_text = split_text
@@ -116,7 +116,7 @@ class Stream(BaseParser):
                     row_y = t.y0
                 temp.append(t)
         rows.append(sorted(temp, key=lambda t: t.x0))
-        __ = rows.pop(0)  # hacky
+        __ = rows.pop(0)  # TODO: hacky
         return rows
 
     @staticmethod
@@ -241,15 +241,42 @@ class Stream(BaseParser):
         return cols
 
     def _validate_columns(self):
-        if self.table_area is not None and self.columns is not None:
-            if len(self.table_area) != len(self.columns):
-                raise ValueError("Length of table_area and columns"
+        if self.table_areas is not None and self.columns is not None:
+            if len(self.table_areas) != len(self.columns):
+                raise ValueError("Length of table_areas and columns"
                                  " should be equal")
 
+    def _nurminen_table_detection(self, textlines):
+        """A general implementation of the table detection algorithm
+        described by Anssi Nurminen's master's thesis.
+        Link: https://dspace.cc.tut.fi/dpub/bitstream/handle/123456789/21520/Nurminen.pdf?sequence=3
+
+        Assumes that tables are situated relatively far apart
+        vertically.
+        """
+
+        # TODO: add support for arabic text #141
+        # sort textlines in reading order
+        textlines.sort(key=lambda x: (-x.y0, x.x0))
+        textedges = TextEdges()
+        # generate left, middle and right textedges
+        textedges.generate(textlines)
+        # select relevant edges
+        relevant_textedges = textedges.get_relevant()
+        self.textedges.extend(relevant_textedges)
+        # guess table areas using textlines and relevant edges
+        table_bbox = textedges.get_table_areas(textlines, relevant_textedges)
+        # treat whole page as table area if no table areas found
+        if not len(table_bbox):
+            table_bbox = {(0, 0, self.pdf_width, self.pdf_height): None}
+
+        return table_bbox
+
     def _generate_table_bbox(self):
-        if self.table_area is not None:
+        self.textedges = []
+        if self.table_areas is not None:
             table_bbox = {}
-            for area in self.table_area:
+            for area in self.table_areas:
                 x1, y1, x2, y2 = area.split(",")
                 x1 = float(x1)
                 y1 = float(y1)
@@ -257,7 +284,8 @@ class Stream(BaseParser):
                 y2 = float(y2)
                 table_bbox[(x1, y2, x2, y1)] = None
         else:
-            table_bbox = {(0, 0, self.pdf_width, self.pdf_height): None}
+            # find tables based on nurminen's detection algorithm
+            table_bbox = self._nurminen_table_detection(self.horizontal_text)
         self.table_bbox = table_bbox
 
     def _generate_columns_and_rows(self, table_idx, tk):
@@ -265,10 +293,11 @@ class Stream(BaseParser):
         t_bbox = {}
         t_bbox['horizontal'] = text_in_bbox(tk, self.horizontal_text)
         t_bbox['vertical'] = text_in_bbox(tk, self.vertical_text)
-        self.t_bbox = t_bbox
 
-        for direction in self.t_bbox:
-            self.t_bbox[direction].sort(key=lambda x: (-x.y0, x.x0))
+        t_bbox['horizontal'].sort(key=lambda x: (-x.y0, x.x0))
+        t_bbox['vertical'].sort(key=lambda x: (x.x0, -x.y0))
+
+        self.t_bbox = t_bbox
 
         text_x_min, text_y_min, text_x_max, text_y_max = self._text_bbox(self.t_bbox)
         rows_grouped = self._group_rows(self.t_bbox['horizontal'], row_close_tol=self.row_close_tol)
@@ -286,10 +315,21 @@ class Stream(BaseParser):
             cols.append(text_x_max)
             cols = [(cols[i], cols[i + 1]) for i in range(0, len(cols) - 1)]
         else:
+            # calculate mode of the list of number of elements in
+            # each row to guess the number of columns
             ncols = max(set(elements), key=elements.count)
             if ncols == 1:
-                warnings.warn("No tables found on {}".format(
-                    os.path.basename(self.rootname)))
+                # if mode is 1, the page usually contains not tables
+                # but there can be cases where the list can be skewed,
+                # try to remove all 1s from list in this case and
+                # see if the list contains elements, if yes, then use
+                # the mode after removing 1s
+                elements = list(filter(lambda x: x != 1, elements))
+                if len(elements):
+                    ncols = max(set(elements), key=elements.count)
+                else:
+                    warnings.warn("No tables found in table area {}".format(
+                        table_idx + 1))
             cols = [(t.x0, t.x1) for r in rows_grouped if len(r) == ncols for t in r]
             cols = self._merge_columns(sorted(cols), col_close_tol=self.col_close_tol)
             inner_text = []
@@ -311,8 +351,11 @@ class Stream(BaseParser):
     def _generate_table(self, table_idx, cols, rows, **kwargs):
         table = Table(cols, rows)
         table = table.set_all_edges()
+
         pos_errors = []
-        for direction in self.t_bbox:
+        # TODO: have a single list in place of two directional ones?
+        # sorted on x-coordinate based on reading order i.e. LTR or RTL
+        for direction in ['vertical', 'horizontal']:
             for t in self.t_bbox[direction]:
                 indices, error = get_table_index(
                     table, t, direction, split_text=self.split_text,
@@ -341,12 +384,14 @@ class Stream(BaseParser):
         table._text = _text
         table._image = None
         table._segments = None
+        table._textedges = self.textedges
 
         return table
 
-    def extract_tables(self, filename):
+    def extract_tables(self, filename, suppress_stdout=False):
         self._generate_layout(filename)
-        logger.info('Processing {}'.format(os.path.basename(self.rootname)))
+        if not suppress_stdout:
+            logger.info('Processing {}'.format(os.path.basename(self.rootname)))
 
         if not self.horizontal_text:
             warnings.warn("No tables found on {}".format(
@@ -361,6 +406,7 @@ class Stream(BaseParser):
                 self.table_bbox.keys(), key=lambda x: x[1], reverse=True)):
             cols, rows = self._generate_columns_and_rows(table_idx, tk)
             table = self._generate_table(table_idx, cols, rows)
+            table._bbox = tk
             _tables.append(table)
 
         return _tables

camelot/plotting.py

@@ -1,108 +1,244 @@
-import cv2
-import matplotlib.pyplot as plt
-import matplotlib.patches as patches
+# -*- coding: utf-8 -*-
+
+try:
+    import matplotlib.pyplot as plt
+    import matplotlib.patches as patches
+except ImportError:
+    _HAS_MPL = False
+else:
+    _HAS_MPL = True
 
 
-def plot_text(text):
-    """Generates a plot for all text present on the PDF page.
+class PlotMethods(object):
+    def __call__(self, table, kind='text', filename=None):
+        """Plot elements found on PDF page based on kind
+        specified, useful for debugging and playing with different
+        parameters to get the best output.
 
-    Parameters
-    ----------
-    text : list
+        Parameters
+        ----------
+        table: camelot.core.Table
+            A Camelot Table.
+        kind : str, optional (default: 'text')
+            {'text', 'grid', 'contour', 'joint', 'line'}
+            The element type for which a plot should be generated.
+        filepath: str, optional (default: None)
+            Absolute path for saving the generated plot.
 
-    """
-    fig = plt.figure()
-    ax = fig.add_subplot(111, aspect='equal')
-    xs, ys = [], []
-    for t in text:
-        xs.extend([t[0], t[2]])
-        ys.extend([t[1], t[3]])
-        ax.add_patch(
-            patches.Rectangle(
-                (t[0], t[1]),
-                t[2] - t[0],
-                t[3] - t[1]
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
+
+        """
+        if not _HAS_MPL:
+            raise ImportError('matplotlib is required for plotting.')
+
+        if table.flavor == 'lattice' and kind in ['textedge']:
+            raise NotImplementedError("Lattice flavor does not support kind='{}'".format(
+                                      kind))
+        elif table.flavor == 'stream' and kind in ['joint', 'line']:
+            raise NotImplementedError("Stream flavor does not support kind='{}'".format(
+                                      kind))
+
+        plot_method = getattr(self, kind)
+        return plot_method(table)
+
+    def text(self, table):
+        """Generates a plot for all text elements present
+        on the PDF page.
+
+        Parameters
+        ----------
+        table : camelot.core.Table
+
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
+
+        """
+        fig = plt.figure()
+        ax = fig.add_subplot(111, aspect='equal')
+        xs, ys = [], []
+        for t in table._text:
+            xs.extend([t[0], t[2]])
+            ys.extend([t[1], t[3]])
+            ax.add_patch(
+                patches.Rectangle(
+                    (t[0], t[1]),
+                    t[2] - t[0],
+                    t[3] - t[1]
+                )
             )
-        )
-    ax.set_xlim(min(xs) - 10, max(xs) + 10)
-    ax.set_ylim(min(ys) - 10, max(ys) + 10)
-    plt.show()
+        ax.set_xlim(min(xs) - 10, max(xs) + 10)
+        ax.set_ylim(min(ys) - 10, max(ys) + 10)
+        return fig
 
+    def grid(self, table):
+        """Generates a plot for the detected table grids
+        on the PDF page.
 
-def plot_table(table):
-    """Generates a plot for the table.
+        Parameters
+        ----------
+        table : camelot.core.Table
 
-    Parameters
-    ----------
-    table : camelot.core.Table
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
 
-    """
-    for row in table.cells:
-        for cell in row:
-            if cell.left:
-                plt.plot([cell.lb[0], cell.lt[0]],
-                         [cell.lb[1], cell.lt[1]])
-            if cell.right:
-                plt.plot([cell.rb[0], cell.rt[0]],
-                         [cell.rb[1], cell.rt[1]])
-            if cell.top:
-                plt.plot([cell.lt[0], cell.rt[0]],
-                         [cell.lt[1], cell.rt[1]])
-            if cell.bottom:
-                plt.plot([cell.lb[0], cell.rb[0]],
-                         [cell.lb[1], cell.rb[1]])
-    plt.show()
+        """
+        fig = plt.figure()
+        ax = fig.add_subplot(111, aspect='equal')
+        for row in table.cells:
+            for cell in row:
+                if cell.left:
+                    ax.plot([cell.lb[0], cell.lt[0]],
+                            [cell.lb[1], cell.lt[1]])
+                if cell.right:
+                    ax.plot([cell.rb[0], cell.rt[0]],
+                            [cell.rb[1], cell.rt[1]])
+                if cell.top:
+                    ax.plot([cell.lt[0], cell.rt[0]],
+                            [cell.lt[1], cell.rt[1]])
+                if cell.bottom:
+                    ax.plot([cell.lb[0], cell.rb[0]],
+                            [cell.lb[1], cell.rb[1]])
+        return fig
 
+    def contour(self, table):
+        """Generates a plot for all table boundaries present
+        on the PDF page.
 
-def plot_contour(image):
-    """Generates a plot for all table boundaries present on the
-    PDF page.
+        Parameters
+        ----------
+        table : camelot.core.Table
 
-    Parameters
-    ----------
-    image : tuple
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
 
-    """
-    img, table_bbox = image
-    for t in table_bbox.keys():
-        cv2.rectangle(img, (t[0], t[1]),
-                      (t[2], t[3]), (255, 0, 0), 20)
-    plt.imshow(img)
-    plt.show()
+        """
+        try:
+            img, table_bbox = table._image
+            _FOR_LATTICE = True
+        except TypeError:
+            img, table_bbox = (None, {table._bbox: None})
+            _FOR_LATTICE = False
+        fig = plt.figure()
+        ax = fig.add_subplot(111, aspect='equal')
 
+        xs, ys = [], []
+        if not _FOR_LATTICE:
+            for t in table._text:
+                xs.extend([t[0], t[2]])
+                ys.extend([t[1], t[3]])
+                ax.add_patch(
+                    patches.Rectangle(
+                        (t[0], t[1]),
+                        t[2] - t[0],
+                        t[3] - t[1],
+                        color='blue'
+                    )
+                )
 
-def plot_joint(image):
-    """Generates a plot for all line intersections present on the
-    PDF page.
+        for t in table_bbox.keys():
+            ax.add_patch(
+                patches.Rectangle(
+                    (t[0], t[1]),
+                    t[2] - t[0],
+                    t[3] - t[1],
+                    fill=False,
+                    color='red'
+                )
+            )
+            if not _FOR_LATTICE:
+                xs.extend([t[0], t[2]])
+                ys.extend([t[1], t[3]])
+                ax.set_xlim(min(xs) - 10, max(xs) + 10)
+                ax.set_ylim(min(ys) - 10, max(ys) + 10)
 
-    Parameters
-    ----------
-    image : tuple
+        if _FOR_LATTICE:
+            ax.imshow(img)
+        return fig
 
-    """
-    img, table_bbox = image
-    x_coord = []
-    y_coord = []
-    for k in table_bbox.keys():
-        for coord in table_bbox[k]:
-            x_coord.append(coord[0])
-            y_coord.append(coord[1])
-    plt.plot(x_coord, y_coord, 'ro')
-    plt.imshow(img)
-    plt.show()
+    def textedge(self, table):
+        """Generates a plot for relevant textedges.
 
+        Parameters
+        ----------
+        table : camelot.core.Table
 
-def plot_line(segments):
-    """Generates a plot for all line segments present on the PDF page.
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
 
-    Parameters
-    ----------
-    segments : tuple
+        """
+        fig = plt.figure()
+        ax = fig.add_subplot(111, aspect='equal')
+        xs, ys = [], []
+        for t in table._text:
+            xs.extend([t[0], t[2]])
+            ys.extend([t[1], t[3]])
+            ax.add_patch(
+                patches.Rectangle(
+                    (t[0], t[1]),
+                    t[2] - t[0],
+                    t[3] - t[1],
+                    color='blue'
+                )
+            )
+        ax.set_xlim(min(xs) - 10, max(xs) + 10)
+        ax.set_ylim(min(ys) - 10, max(ys) + 10)
 
-    """
-    vertical, horizontal = segments
-    for v in vertical:
-        plt.plot([v[0], v[2]], [v[1], v[3]])
-    for h in horizontal:
-        plt.plot([h[0], h[2]], [h[1], h[3]])
-    plt.show()
+        for te in table._textedges:
+            ax.plot([te.x, te.x],
+                    [te.y0, te.y1])
+
+        return fig
+
+    def joint(self, table):
+        """Generates a plot for all line intersections present
+        on the PDF page.
+
+        Parameters
+        ----------
+        table : camelot.core.Table
+
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
+
+        """
+        img, table_bbox = table._image
+        fig = plt.figure()
+        ax = fig.add_subplot(111, aspect='equal')
+        x_coord = []
+        y_coord = []
+        for k in table_bbox.keys():
+            for coord in table_bbox[k]:
+                x_coord.append(coord[0])
+                y_coord.append(coord[1])
+        ax.plot(x_coord, y_coord, 'ro')
+        ax.imshow(img)
+        return fig
+
+    def line(self, table):
+        """Generates a plot for all line segments present
+        on the PDF page.
+
+        Parameters
+        ----------
+        table : camelot.core.Table
+
+        Returns
+        -------
+        fig : matplotlib.fig.Figure
+
+        """
+        fig = plt.figure()
+        ax = fig.add_subplot(111, aspect='equal')
+        vertical, horizontal = table._segments
+        for v in vertical:
+            ax.plot([v[0], v[2]], [v[1], v[3]])
+        for h in horizontal:
+            ax.plot([h[0], h[2]], [h[1], h[3]])
+        return fig

camelot/utils.py

@@ -344,9 +344,9 @@ def flag_font_size(textline, direction):
                 fchars = [t[0] for t in chars]
                 if ''.join(fchars).strip():
                     flist.append(''.join(fchars))
-        fstring = ''.join(flist).strip('\n')
+        fstring = ''.join(flist)
     else:
-        fstring = ''.join([t.get_text() for t in textline]).strip('\n')
+        fstring = ''.join([t.get_text() for t in textline])
     return fstring
 
 
@@ -419,7 +419,7 @@ def split_textline(table, textline, direction, flag_size=False):
             grouped_chars.append((key[0], key[1], flag_font_size([t[2] for t in chars], direction)))
         else:
             gchars = [t[2].get_text() for t in chars]
-            grouped_chars.append((key[0], key[1], ''.join(gchars).strip('\n')))
+            grouped_chars.append((key[0], key[1], ''.join(gchars)))
     return grouped_chars
 
 
@@ -500,7 +500,7 @@ def get_table_index(table, t, direction, split_text=False, flag_size=False):
         if flag_size:
             return [(r_idx, c_idx, flag_font_size(t._objs, direction))], error
         else:
-            return [(r_idx, c_idx, t.get_text().strip('\n'))], error
+            return [(r_idx, c_idx, t.get_text())], error
 
 
 def compute_accuracy(error_weights):