[MRG] Add tests for output formats and parser kwargs (#126)

* Remove unused image processing code

* Add opencv back-compat comment

* Add tests for parser special cases

* Fix lattice table area test

* Add tests for output format

* Add openpyxl dep

camelot/image_processing.py

@@ -7,8 +7,6 @@ from operator import itemgetter
 import cv2
 import numpy as np
 
-from .utils import merge_tuples
-
 
 def adaptive_threshold(imagename, process_background=False, blocksize=15, c=-2):
     """Thresholds an image using OpenCV's adaptiveThreshold.
@@ -102,6 +100,7 @@ def find_lines(threshold, direction='horizontal', line_size_scaling=15, iteratio
         _, contours, _ = cv2.findContours(
             threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
     except ValueError:
+        # for opencv backward compatibility
         contours, _ = cv2.findContours(
             threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
 
@@ -141,6 +140,7 @@ def find_table_contours(vertical, horizontal):
         __, contours, __ = cv2.findContours(
             mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
     except ValueError:
+        # for opencv backward compatibility
         contours, __ = cv2.findContours(
             mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
     contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]
@@ -185,6 +185,7 @@ def find_table_joints(contours, vertical, horizontal):
             __, jc, __ = cv2.findContours(
                 roi, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
         except ValueError:
+            # for opencv backward compatibility
             jc, __ = cv2.findContours(
                 roi, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
         if len(jc) <= 4:  # remove contours with less than 4 joints
@@ -196,80 +197,4 @@ def find_table_joints(contours, vertical, horizontal):
             joint_coords.append((c1, c2))
         tables[(x, y + h, x + w, y)] = joint_coords
 
-    return tables
-
-
-def remove_lines(threshold, line_size_scaling=15):
-    """Removes lines from a thresholded image.
-
-    Parameters
-    ----------
-    threshold : object
-        numpy.ndarray representing the thresholded image.
-    line_size_scaling : int, optional (default: 15)
-        Factor by which the page dimensions will be divided to get
-        smallest length of lines that should be detected.
-
-        The larger this value, smaller the detected lines. Making it
-        too large will lead to text being detected as lines.
-
-    Returns
-    -------
-    threshold : object
-        numpy.ndarray representing the thresholded image
-        with horizontal and vertical lines removed.
-
-    """
-    size = threshold.shape[0] // line_size_scaling
-    vertical_erode_el = cv2.getStructuringElement(cv2.MORPH_RECT, (1, size))
-    horizontal_erode_el = cv2.getStructuringElement(cv2.MORPH_RECT, (size, 1))
-    dilate_el = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
-
-    vertical = cv2.erode(threshold, vertical_erode_el)
-    vertical = cv2.dilate(vertical, dilate_el)
-
-    horizontal = cv2.erode(threshold, horizontal_erode_el)
-    horizontal = cv2.dilate(horizontal, dilate_el)
-
-    threshold = np.bitwise_and(threshold, np.invert(vertical))
-    threshold = np.bitwise_and(threshold, np.invert(horizontal))
-    return threshold
-
-
-def find_cuts(threshold, char_size_scaling=200):
-    """Finds cuts made by text projections on y-axis.
-
-    Parameters
-    ----------
-    threshold : object
-        numpy.ndarray representing the thresholded image.
-    line_size_scaling : int, optional (default: 200)
-        Factor by which the page dimensions will be divided to get
-        smallest length of lines that should be detected.
-
-        The larger this value, smaller the detected lines. Making it
-        too large will lead to text being detected as lines.
-
-    Returns
-    -------
-    y_cuts : list
-        List of cuts on y-axis.
-    """
-    size = threshold.shape[0] // char_size_scaling
-    char_el = cv2.getStructuringElement(cv2.MORPH_RECT, (1, size))
-
-    threshold = cv2.erode(threshold, char_el)
-    threshold = cv2.dilate(threshold, char_el)
-
-    try:
-        __, contours, __ = cv2.findContours(threshold, cv2.RETR_EXTERNAL,
-            cv2.CHAIN_APPROX_SIMPLE)
-    except ValueError:
-        contours, __ = cv2.findContours(threshold, cv2.RETR_EXTERNAL,
-            cv2.CHAIN_APPROX_SIMPLE)
-
-    contours = [cv2.boundingRect(c) for c in contours]
-    y_cuts = [(c[1], c[1] + c[3]) for c in contours]
-    y_cuts = list(merge_tuples(sorted(y_cuts)))
-    y_cuts = [(y_cuts[i][0] + y_cuts[i - 1][1]) // 2 for i in range(1, len(y_cuts))]
-    return sorted(y_cuts, reverse=True)
+    return tables

camelot/utils.py

@@ -640,25 +640,4 @@ def get_text_objects(layout, ltype="char", t=None):
                 t += get_text_objects(obj, ltype=ltype)
     except AttributeError:
         pass
-    return t
-
-
-def merge_tuples(tuples):
-    """Merges a list of overlapping tuples.
-     Parameters
-    ----------
-    tuples : list
-        List of tuples where a tuple is a single axis coordinate pair.
-     Yields
-    ------
-    tuple
-     """
-    merged = list(tuples[0])
-    for s, e in tuples:
-        if s <= merged[1]:
-            merged[1] = max(merged[1], e)
-        else:
-            yield tuple(merged)
-            merged[0] = s
-            merged[1] = e
-    yield tuple(merged)
+    return t