Add properties to Table

camelot/core.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pandas as pd
 
 
 class Cell(object):
@@ -48,9 +47,15 @@ class Table(object):
         self.rows = rows
         self.cells = [[Cell(c[0], r[1], c[1], r[0])
                        for c in cols] for r in rows]
+        self._df = None
+        self._shape = (0, 0)
+        self._accuracy = 0
+        self._whitespace = 0
+        self._order = None
+        self._page = None
 
     def __repr__(self):
-        pass
+        return '<{} shape={}>'.format(self.__class__.__name__, self._shape)
 
     def set_all_edges(self):
         for r in range(len(self.rows)):
@@ -216,12 +221,61 @@ class Table(object):
 
         return self
 
-    def get_list(self):
+    @property
-        ar = []
+    def data(self):
+        d = []
         for r in range(len(self.rows)):
-            ar.append([self.cells[r][c].get_text().strip()
+            d.append([self.cells[r][c].get_text().strip()
                        for c in range(len(self.cols))])
-        return ar
+        return d
+
+    @property
+    def df(self):
+        return self._df
+
+    @df.setter
+    def df(self, dataframe):
+        self._df = dataframe
+
+    @property
+    def shape(self):
+        return self._shape
+
+    @shape.setter
+    def shape(self, s):
+        self._shape = s
+
+    @property
+    def accuracy(self):
+        return self._accuracy
+
+    @accuracy.setter
+    def accuracy(self, a):
+        self._accuracy = a
+
+    @property
+    def whitespace(self):
+        return self._whitespace
+
+    @whitespace.setter
+    def whitespace(self, w):
+        self._whitespace = w
+
+    @property
+    def order(self):
+        return self._order
+
+    @order.setter
+    def order(self, o):
+        self._order = o
+
+    @property
+    def page(self):
+        return self._page
+
+    @page.setter
+    def page(self, p):
+        self._page = p
 
 
 class TableList(list):
@@ -236,8 +290,8 @@ class TableList(list):
 class Geometry(object):
     def __init__(self):
         self._text = []
-        self._images = []
+        self._images = ()
-        self._segments = []
+        self._segments = ()
         self._tables = []
 
     @property

camelot/handlers.py

@@ -3,6 +3,7 @@ import tempfile
 
 from PyPDF2 import PdfFileReader, PdfFileWriter
 
+from .core import TableList, GeometryList
 from .parsers import Stream, Lattice
 from .utils import get_page_layout, get_text_objects, get_rotation
 
@@ -80,5 +81,5 @@ class PDFHandler(object):
         for p in pages:
             t, g = parser.extract_tables(p)
             tables.extend(t)
-            geometry.extend(g)
+            geometry.append(g)
         return TableList(tables), GeometryList(geometry)

camelot/parsers.py

@@ -9,13 +9,14 @@ import warnings
 import subprocess
 
 import numpy as np
+import pandas as pd
 
 from .core import Table, Geometry
 from .image_processing import (adaptive_threshold, find_lines, find_table_contours,
                                find_table_joints)
 from .utils import (scale_to_pdf, scale_to_image, segments_bbox, text_in_bbox,
-                    merge_close_values, get_table_index, get_score, count_empty,
+                    merge_close_values, get_table_index, compute_accuracy, count_empty,
-                    encode_list, get_text_objects, get_page_layout)
+                    get_text_objects, get_page_layout, encode_)
 
 
 __all__ = ['Stream', 'Lattice']
@@ -31,58 +32,6 @@ copy_reg.pickle(types.MethodType, _reduce_method)
 
 
 class Stream:
-    """Stream looks for spaces between text elements to form a table.
-
-    If you want to give columns, ytol or mtol for each table
-    when specifying multiple table areas, make sure that their length
-    is equal to the length of table_area. Mapping between them is based
-    on index.
-
-    If you don't want to specify columns for the some tables in a pdf
-    page having multiple tables, pass them as empty strings.
-    For example: ['', 'x1,x2,x3,x4', '']
-
-    Parameters
-    ----------
-    table_area : list
-        List of strings of the form x1,y1,x2,y2 where
-        (x1, y1) -> left-top and (x2, y2) -> right-bottom in PDFMiner's
-        coordinate space, denoting table areas to analyze.
-        (optional, default: None)
-
-    columns : list
-        List of strings where each string is comma-separated values of
-        x-coordinates in PDFMiner's coordinate space.
-        (optional, default: None)
-
-    ytol : list
-        List of ints specifying the y-tolerance parameters.
-        (optional, default: [2])
-
-    mtol : list
-        List of ints specifying the m-tolerance parameters.
-        (optional, default: [0])
-
-    margins : tuple
-        PDFMiner margins. (char_margin, line_margin, word_margin)
-        (optional, default: (1.0, 0.5, 0.1))
-
-    split_text : bool
-        Whether or not to split a text line if it spans across
-        different cells.
-        (optional, default: False)
-
-    flag_size : bool
-        Whether or not to highlight a substring using <s></s>
-        if its size is different from rest of the string, useful for
-        super and subscripts.
-        (optional, default: True)
-
-    debug : bool
-        Set to True to generate a matplotlib plot of
-        LTTextLineHorizontals in order to select table_area, columns.
-        (optional, default: False)
-    """
     def __init__(self, table_area=None, columns=None, ytol=[2], mtol=[0],
                  margins=(1.0, 0.5, 0.1), split_text=False, flag_size=True,
                  debug=False):
@@ -99,20 +48,6 @@ class Stream:
 
     @staticmethod
     def _text_bbox(t_bbox):
-        """Returns bounding box for the text present on a page.
-
-        Parameters
-        ----------
-        t_bbox : dict
-            Dict with two keys 'horizontal' and 'vertical' with lists of
-            LTTextLineHorizontals and LTTextLineVerticals respectively.
-
-        Returns
-        -------
-        text_bbox : tuple
-            Tuple of the form (x0, y0, x1, y1) in PDFMiner's coordinate
-            space.
-        """
         xmin = min([t.x0 for direction in t_bbox for t in t_bbox[direction]])
         ymin = min([t.y0 for direction in t_bbox for t in t_bbox[direction]])
         xmax = max([t.x1 for direction in t_bbox for t in t_bbox[direction]])
@@ -122,23 +57,6 @@ class Stream:
 
     @staticmethod
     def _group_rows(text, ytol=2):
-        """Groups PDFMiner text objects into rows using their
-        y-coordinates taking into account some tolerance ytol.
-
-        Parameters
-        ----------
-        text : list
-            List of PDFMiner text objects.
-
-        ytol : int
-            Tolerance parameter.
-            (optional, default: 2)
-
-        Returns
-        -------
-        rows : list
-            Two-dimensional list of text objects grouped into rows.
-        """
         row_y = 0
         rows = []
         temp = []
@@ -158,23 +76,6 @@ class Stream:
 
     @staticmethod
     def _merge_columns(l, mtol=0):
-        """Merges column boundaries if they overlap or lie within some
-        tolerance mtol.
-
-        Parameters
-        ----------
-        l : list
-            List of column coordinate tuples.
-
-        mtol : int
-            TODO
-            (optional, default: 0)
-
-        Returns
-        -------
-        merged : list
-            List of merged column coordinate tuples.
-        """
         merged = []
         for higher in l:
             if not merged:
@@ -203,22 +104,6 @@ class Stream:
 
     @staticmethod
     def _join_rows(rows_grouped, text_y_max, text_y_min):
-        """Makes row coordinates continuous.
-
-        Parameters
-        ----------
-        rows_grouped : list
-            Two-dimensional list of text objects grouped into rows.
-
-        text_y_max : int
-
-        text_y_min : int
-
-        Returns
-        -------
-        rows : list
-            List of continuous row coordinate tuples.
-        """
         row_mids = [sum([(t.y0 + t.y1) / 2 for t in r]) / len(r)
                     if len(r) > 0 else 0 for r in rows_grouped]
         rows = [(row_mids[i] + row_mids[i - 1]) / 2 for i in range(1, len(row_mids))]
@@ -230,25 +115,6 @@ class Stream:
 
     @staticmethod
     def _add_columns(cols, text, ytol):
-        """Adds columns to existing list by taking into account
-        the text that lies outside the current column coordinates.
-
-        Parameters
-        ----------
-        cols : list
-            List of column coordinate tuples.
-
-        text : list
-            List of PDFMiner text objects.
-
-        ytol : int
-            Tolerance parameter.
-
-        Returns
-        -------
-        cols : list
-            Updated list of column coordinate tuples.
-        """
         if text:
             text = Stream._group_rows(text, ytol=ytol)
             elements = [len(r) for r in text]
@@ -259,22 +125,6 @@ class Stream:
 
     @staticmethod
     def _join_columns(cols, text_x_min, text_x_max):
-        """Makes column coordinates continuous.
-
-        Parameters
-        ----------
-        cols : list
-            List of column coordinate tuples.
-
-        text_x_min : int
-
-        text_y_max : int
-
-        Returns
-        -------
-        cols : list
-            Updated list of column coordinate tuples.
-        """
         cols = sorted(cols)
         cols = [(cols[i][0] + cols[i - 1][1]) / 2 for i in range(1, len(cols))]
         cols.insert(0, text_x_min)
@@ -284,17 +134,6 @@ class Stream:
         return cols
 
     def extract_tables(self, pdfname):
-        """Expects a single page pdf as input with rotation corrected.
-
-        Parameters
-        ---------
-        pdfname : string
-            Path to single page pdf file.
-
-        Returns
-        -------
-        page : dict
-        """
         layout, dim = get_page_layout(pdfname, char_margin=self.char_margin,
             line_margin=self.line_margin, word_margin=self.word_margin)
         lttextlh = get_text_objects(layout, ltype="lh")
@@ -314,7 +153,6 @@ class Stream:
             text.extend([(t.x0, t.y0, t.x1, t.y1) for t in lttextlh])
             text.extend([(t.x0, t.y0, t.x1, t.y1) for t in lttextlv])
             g.text = text
-            return [None], [g]
 
         if self.table_area is not None:
             if self.columns is not None:
@@ -343,17 +181,13 @@ class Stream:
         else:
             mtolerance = copy.deepcopy(self.mtol)
 
-        page = {}
+        _tables = []
-        tables = {}
         # sort tables based on y-coord
         for table_no, k in enumerate(sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True)):
             # select elements which lie within table_bbox
-            table_data = {}
             t_bbox = {}
             t_bbox['horizontal'] = text_in_bbox(k, lttextlh)
             t_bbox['vertical'] = text_in_bbox(k, lttextlv)
-            char_bbox = text_in_bbox(k, ltchar)
-            table_data['text_p'] = 100 * (1 - (len(char_bbox) / len(ltchar)))
             for direction in t_bbox:
                 t_bbox[direction].sort(key=lambda x: (-x.y0, x.x0))
             text_x_min, text_y_min, text_x_max, text_y_max = self._text_bbox(t_bbox)
@@ -399,124 +233,38 @@ class Stream:
 
             table = Table(cols, rows)
             table = table.set_all_edges()
-            assignment_errors = []
+            pos_errors = []
-            table_data['split_text'] = []
-            table_data['superscript'] = []
             for direction in t_bbox:
                 for t in t_bbox[direction]:
                     indices, error = get_table_index(
                         table, t, direction, split_text=self.split_text,
                         flag_size=self.flag_size)
-                    assignment_errors.append(error)
+                    if indices[:2] != (-1, -1):
-                    if len(indices) > 1:
+                        pos_errors.append(error)
-                        table_data['split_text'].append(indices)
                         for r_idx, c_idx, text in indices:
-                        if all(s in text for s in ['<s>', '</s>']):
-                            table_data['superscript'].append((r_idx, c_idx, text))
                             table.cells[r_idx][c_idx].add_text(text)
             if guess:
-                score = get_score([[66, assignment_errors], [34, [len_non_mode / len(elements)]]])
+                accuracy = compute_accuracy([[66, pos_errors], [34, [len_non_mode / len(elements)]]])
             else:
-                score = get_score([[100, assignment_errors]])
+                accuracy = compute_accuracy([[100, pos_errors]])
 
-            table_data['score'] = score
+            data = table.data
-            ar = table.get_list()
+            data = encode_(data)
-            ar = encode_list(ar)
+            table.df = pd.DataFrame(data)
-            table_data['data'] = ar
+            table.shape = table.df.shape
-            empty_p, r_nempty_cells, c_nempty_cells = count_empty(ar)
-            table_data['empty_p'] = empty_p
-            table_data['r_nempty_cells'] = r_nempty_cells
-            table_data['c_nempty_cells'] = c_nempty_cells
-            table_data['nrows'] = len(ar)
-            table_data['ncols'] = len(ar[0])
-            tables['table-{0}'.format(table_no + 1)] = table_data
-        page[os.path.basename(bname)] = tables
 
-        return page
+            whitespace, __, __ = count_empty(data)
+            table.accuracy = accuracy
+            table.whitespace = whitespace
+            table.order = table_no + 1
+            table.page = os.path.basename(bname).replace('page-', '')
+
+            _tables.append(table)
+
+        return _tables, g
 
 
 class Lattice:
-    """Lattice looks for lines in the pdf to form a table.
-
-    If you want to give fill and mtol for each table when specifying
-    multiple table areas, make sure that the length of fill and mtol
-    is equal to the length of table_area. Mapping between them is based
-    on index.
-
-    Parameters
-    ----------
-    table_area : list
-        List of strings of the form x1,y1,x2,y2 where
-        (x1, y1) -> left-top and (x2, y2) -> right-bottom in PDFMiner's
-        coordinate space, denoting table areas to analyze.
-        (optional, default: None)
-
-    fill : list
-        List of strings specifying directions to fill spanning cells.
-        {'h', 'v'} to fill spanning cells in horizontal or vertical
-        direction.
-        (optional, default: None)
-
-    mtol : list
-        List of ints specifying m-tolerance parameters.
-        (optional, default: [2])
-
-    jtol : list
-        List of ints specifying j-tolerance parameters.
-        (optional, default: [2])
-
-    blocksize : int
-        Size of a pixel neighborhood that is used to calculate a
-        threshold value for the pixel: 3, 5, 7, and so on.
-        (optional, default: 15)
-
-    threshold_constant : float
-        Constant subtracted from the mean or weighted mean
-        (see the details below). Normally, it is positive but may be
-        zero or negative as well.
-        (optional, default: -2)
-
-    scale : int
-        Used to divide the height/width of a pdf to get a structuring
-        element for image processing.
-        (optional, default: 15)
-
-    iterations : int
-        Number of iterations for dilation.
-        (optional, default: 0)
-
-    invert : bool
-        Whether or not to invert the image. Useful when pdfs have
-        tables with lines in background.
-        (optional, default: False)
-
-    margins : tuple
-        PDFMiner margins. (char_margin, line_margin, word_margin)
-        (optional, default: (1.0, 0.5, 0.1))
-
-    split_text : bool
-        Whether or not to split a text line if it spans across
-        different cells.
-        (optional, default: False)
-
-    flag_size : bool
-        Whether or not to highlight a substring using <s></s>
-        if its size is different from rest of the string, useful for
-        super and subscripts.
-        (optional, default: True)
-
-    shift_text : list
-        {'l', 'r', 't', 'b'}
-        Select one or more from above and pass them as a list to
-        specify where the text in a spanning cell should flow.
-        (optional, default: ['l', 't'])
-
-    debug : string
-        {'contour', 'line', 'joint', 'table'}
-        Set to one of the above values to generate a matplotlib plot
-        of detected contours, lines, joints and the table generated.
-        (optional, default: None)
-    """
     def __init__(self, table_area=None, fill=None, mtol=[2], jtol=[2],
                  blocksize=15, threshold_constant=-2, scale=15, iterations=0,
                  invert=False, margins=(1.0, 0.5, 0.1), split_text=False,
@@ -540,28 +288,6 @@ class Lattice:
 
     @staticmethod
     def _reduce_index(t, idx, shift_text):
-        """Reduces index of a text object if it lies within a spanning
-        cell.
-
-        Parameters
-        ----------
-        table : object
-            camelot.table.Table
-
-        idx : list
-            List of tuples of the form (r_idx, c_idx, text).
-
-        shift_text : list
-            {'l', 'r', 't', 'b'}
-            Select one or more from above and pass them as a list to
-            specify where the text in a spanning cell should flow.
-
-        Returns
-        -------
-        indices : list
-            List of tuples of the form (idx, text) where idx is the reduced
-            index of row/column and text is the an lttextline substring.
-        """
         indices = []
         for r_idx, c_idx, text in idx:
             for d in shift_text:
@@ -586,24 +312,6 @@ class Lattice:
 
 
     def _fill_spanning(t, fill=None):
-        """Fills spanning cells.
-
-        Parameters
-        ----------
-        t : object
-            camelot.table.Table
-
-        fill : list
-            {'h', 'v'}
-            Specify to fill spanning cells in horizontal or vertical
-            direction.
-            (optional, default: None)
-
-        Returns
-        -------
-        t : object
-            camelot.table.Table
-        """
         for f in fill:
             if f == "h":
                 for i in range(len(t.cells)):
@@ -620,17 +328,6 @@ class Lattice:
         return t
 
     def extract_tables(self, pdfname):
-        """Expects a single page pdf as input with rotation corrected.
-
-        Parameters
-        ----------
-        pdfname : string
-            Path to single page pdf file.
-
-        Returns
-        -------
-        page : dict
-        """
         layout, dim = get_page_layout(pdfname, char_margin=self.char_margin,
             line_margin=self.line_margin, word_margin=self.word_margin)
         lttextlh = get_text_objects(layout, ltype="lh")
@@ -700,27 +397,22 @@ class Lattice:
 
         g = Geometry()
         if self.debug:
-            g.images = [(img, table_bbox)]
+            g.images = (img, table_bbox)
 
         table_bbox, v_segments, h_segments = scale_to_pdf(table_bbox, v_segments,
             h_segments, factors_pdf)
 
         if self.debug:
-            g.segments = [(v_segments, h_segments)]
+            g.segments = (v_segments, h_segments)
-            _tables = []
 
-        page = {}
+        _tables = []
-        tables = {}
         # sort tables based on y-coord
         for table_no, k in enumerate(sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True)):
             # select elements which lie within table_bbox
-            table_data = {}
             t_bbox = {}
             v_s, h_s = segments_bbox(k, v_segments, h_segments)
             t_bbox['horizontal'] = text_in_bbox(k, lttextlh)
             t_bbox['vertical'] = text_in_bbox(k, lttextlv)
-            char_bbox = text_in_bbox(k, ltchar)
-            table_data['text_p'] = 100 * (1 - (len(char_bbox) / len(ltchar)))
             for direction in t_bbox:
                 t_bbox[direction].sort(key=lambda x: (-x.y0, x.x0))
             cols, rows = zip(*table_bbox[k])
@@ -745,45 +437,36 @@ class Lattice:
             # set table border edges to True
             table = table.set_border_edges()
 
-            if self.debug:
+            pos_errors = []
-                _tables.append(table)
-
-            assignment_errors = []
-            table_data['split_text'] = []
-            table_data['superscript'] = []
             for direction in ['vertical', 'horizontal']:
                 for t in t_bbox[direction]:
                     indices, error = get_table_index(
                         table, t, direction, split_text=self.split_text,
                         flag_size=self.flag_size)
                     if indices[:2] != (-1, -1):
-                        assignment_errors.append(error)
+                        pos_errors.append(error)
                         indices = self._reduce_index(table, indices, shift_text=self.shift_text)
-                        if len(indices) > 1:
-                            table_data['split_text'].append(indices)
                         for r_idx, c_idx, text in indices:
-                            if all(s in text for s in ['<s>', '</s>']):
-                                table_data['superscript'].append((r_idx, c_idx, text))
                             table.cells[r_idx][c_idx].add_text(text)
-            score = get_score([[100, assignment_errors]])
+            accuracy = compute_accuracy([[100, pos_errors]])
-            table_data['score'] = score
 
             if self.fill is not None:
                 table = self._fill_spanning(table, fill=self.fill)
-            ar = table.get_list()
+
-            ar = encode_list(ar)
+            data = table.data
-            table_data['data'] = ar
+            data = encode_(data)
-            empty_p, r_nempty_cells, c_nempty_cells = count_empty(ar)
+            table.df = pd.DataFrame(data)
-            table_data['empty_p'] = empty_p
+            table.shape = table.df.shape
-            table_data['r_nempty_cells'] = r_nempty_cells
+
-            table_data['c_nempty_cells'] = c_nempty_cells
+            whitespace, __, __ = count_empty(data)
-            table_data['nrows'] = len(ar)
+            table.accuracy = accuracy
-            table_data['ncols'] = len(ar[0])
+            table.whitespace = whitespace
-            tables['table-{0}'.format(table_no + 1)] = table_data
+            table.order = table_no + 1
-        page[os.path.basename(bname)] = tables
+            table.page = os.path.basename(bname).replace('page-', '')
+
+            _tables.append(table)
 
         if self.debug:
             g.tables = _tables
-            return [None], [g]
 
-        return page
+        return _tables, g

camelot/utils.py

@@ -557,7 +557,7 @@ def get_table_index(table, t, direction, split_text=False, flag_size=True):
             return [(r_idx, c_idx, t.get_text().strip('\n'))], error
 
 
-def get_score(error_weights):
+def compute_accuracy(error_weights):
     """Calculates score based on weights assigned to various parameters,
     and their error percentages.
 
@@ -648,7 +648,7 @@ def count_empty(d):
     return empty_p, r_nempty_cells, c_nempty_cells
 
 
-def encode_list(ar):
+def encode_(ar):
     """Encodes list of text.
 
     Parameters