diff --git a/.gitignore b/.gitignore
index e5bdc6b..b1f7323 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,6 @@ __pycache__/
*.py[cod]
*.so
-.camelot/
\ No newline at end of file
+build/
+dist/
+*.egg-info/
diff --git a/README.md b/README.md
index 3e11120..fa6d6a1 100644
--- a/README.md
+++ b/README.md
@@ -4,25 +4,39 @@
Currently, camelot works under Python 2.7.
-The required dependencies include pdfminer, numpy, opencv.
-
-For debugging, matplotlib is required. For runnings tests in the future, nose may be required.
+The required dependencies include numpy, opencv, and imagemagick.
## Install
+Make sure you have the required dependencies installed on your system. If you're working in a virtual environment, copy the `cv2.so` file from your system's site-packages to the virtualenv's site-packages. After that, `cd` into the project directory and issue the following command.
+
+
+camelot parses tables from PDFs!
+
+usage:
+ camelot.py [options] [...]
options:
- -h, --help Show this screen.
- -v, --version Show version.
- -p, --pages <pageno> Comma-separated list of page numbers.
- Example: -p 1,3-6,10 [default: 1]
- -f, --format <format> Output format. (csv,xlsx) [default: csv]
- -l, --log Print log to file.
- -o, --output <directory> Output directory.
+ -h, --help Show this screen.
+ -v, --version Show version.
+ -p, --pages <pageno> Comma-separated list of page numbers.
+ Example: -p 1,3-6,10 [default: 1]
+ -f, --format <format> Output format. (csv,tsv,html,json,xlsx) [default: csv]
+ -l, --log Print log to file.
+ -o, --output <directory> Output directory.
camelot methods:
lattice Looks for lines between data.
diff --git a/camelot.py b/camelot.py
deleted file mode 100755
index 9d72a0f..0000000
--- a/camelot.py
+++ /dev/null
@@ -1,264 +0,0 @@
-#!/usr/bin/env python2
-import os
-import re
-import csv
-import sys
-import glob
-import time
-import shutil
-import logging
-import zipfile
-import tempfile
-from docopt import docopt
-from werkzeug.utils import secure_filename
-from PyPDF2 import PdfFileWriter, PdfFileReader
-
-from lattice import lattice
-from stream import stream
-
-
-doc = """
-camelot parses tables from PDFs!
-
-usage:
- camelot.py [options] [...]
-
-options:
- -h, --help Show this screen.
- -v, --version Show version.
- -p, --pages Comma-separated list of page numbers.
- Example: -p 1,3-6,10 [default: 1]
- -f, --format Output format. (csv,xlsx) [default: csv]
- -l, --log Print log to file.
- -o, --output Output directory.
-
-camelot methods:
- lattice Looks for lines between data.
- stream Looks for spaces between data.
-
-See 'camelot -h' for more information on a specific method.
-"""
-
-lattice_doc = """
-Lattice method looks for lines between data to form a table.
-
-usage:
- camelot.py lattice [options] [--]
-
-options:
- -F, --fill Fill data in horizontal and/or vertical spanning
- cells. Example: -F h, -F v, -F hv
- -s, --scale Scaling factor. Large scaling factor leads to
- smaller lines being detected. [default: 15]
- -j, --jtol Tolerance to account for when comparing joint
- and line coordinates. [default: 2]
- -m, --mtol Tolerance to account for when merging lines
- which are very close. [default: 2]
- -i, --invert Invert pdf image to make sure that lines are
- in foreground.
- -d, --debug Debug by visualizing pdf geometry.
- (contour,line,joint,table) Example: -d table
-"""
-
-stream_doc = """
-Stream method looks for spaces between data to form a table.
-
-usage:
- camelot.py stream [options] [--]
-
-options:
- -n, --ncols Number of columns. [default: 0]
- -c, --columns Comma-separated list of column x-coordinates.
- Example: -c 10.1,20.2,30.3
- -M, --cmargin Char margin. Chars closer than cmargin are
- grouped together to form a word. [default: 2.0]
- -L, --lmargin Line margin. Lines closer than lmargin are
- grouped together to form a textbox. [default: 0.5]
- -W, --wmargin Word margin. Insert blank spaces between chars
- if distance between words is greater than word
- margin. [default: 0.1]
- -d, --debug Debug by visualizing textboxes.
-"""
-
-pno = re.compile(r'\d+')
-
-
-def filesort(filepath):
- filename = os.path.basename(filepath)
- num = pno.findall(filename)
- if len(num) == 2:
- return (int(num[0]), int(num[1]))
- else:
- return (int(num[0]), 0)
-
-
-if __name__ == '__main__':
- start_time = time.time()
- tmpdir = tempfile.mkdtemp()
-
- args = docopt(doc, version='0.1', options_first=True)
- argv = [args['']] + args['']
- if args[''] == 'lattice':
- args.update(docopt(lattice_doc, argv=argv))
- elif args[''] == 'stream':
- args.update(docopt(stream_doc, argv=argv))
-
- if args['--pages']:
- if args['--pages'] == ['all']:
- p = args['--pages']
- else:
- p = []
- for r in args['--pages'].split(','):
- if '-' in r:
- a, b = r.split('-')
- a, b = int(a), int(b)
- p.extend([str(i) for i in range(a, b + 1)])
- else:
- p.extend([str(r)])
- else:
- p = ['1']
- p = sorted(set(p))
-
- fname = os.path.basename(args[''])
- fname = secure_filename(fname)
- fdir = os.path.dirname(args[''])
- froot, fext = os.path.splitext(fname)
- if fext.lower() != '.pdf':
- print "camelot can parse only pdfs right now"
- shutil.rmtree(tmpdir)
- sys.exit()
-
- logfname = os.path.join(tmpdir, froot + '.log')
- logging.basicConfig(filename=logfname, filemode='w', level=logging.DEBUG)
-
- shutil.copy(args[''], os.path.join(tmpdir, fname))
- print "separating pdf into pages"
- print
- if p == ['all']:
- infile = PdfFileReader(open(os.path.join(tmpdir, fname), 'rb'))
- for i in range(infile.getNumPages()):
- p = infile.getPage(i)
- outfile = PdfFileWriter()
- outfile.addPage(p)
- with open(os.path.join(tmpdir, 'pg-%d.pdf' % (i + 1)), 'wb') as f:
- outfile.write(f)
- else:
- for page in p:
- infile = PdfFileReader(open(os.path.join(tmpdir, fname), 'rb'))
- p = infile.getPage(int(page) - 1)
- outfile = PdfFileWriter()
- outfile.addPage(p)
- with open(os.path.join(tmpdir, 'pg-%s.pdf' % page), 'wb') as f:
- outfile.write(f)
-
- glob_pdf = sorted(glob.glob(os.path.join(tmpdir, 'pg-*.pdf')))
- if args[''] == 'lattice':
- print "using the lattice method"
- for g in glob_pdf:
- g_fname = os.path.basename(g)
- print "working on", g_fname
- g_froot, __ = os.path.splitext(g)
- try:
- data = lattice(g, f=args['--fill'], s=int(args['--scale']),
- jtol=int(args['--jtol']), mtol=int(args['--mtol']),
- invert=args['--invert'], debug=args['--debug'])
- if data is None:
- print
- continue
- for k in sorted(data.keys()):
- csvfile = g_froot + '_%s.csv' % k
- with open(csvfile, 'w') as outfile:
- writer = csv.writer(outfile)
- for d in data[k]:
- writer.writerow([c.encode('utf-8') for c in d])
- print "saved as", os.path.basename(csvfile)
- print
- except Exception:
- logging.exception("")
- print "couldn't parse", g_fname, "see log for more info"
- print
- elif args[''] == 'stream':
- print "using the stream method"
- for g in glob_pdf:
- g_fname = os.path.basename(g)
- print "working on", g_fname
- g_froot, __ = os.path.splitext(g)
- try:
- data = stream(g, ncolumns=int(args['--ncols']), columns=args['--columns'],
- char_margin=float(args['--cmargin']),
- line_margin=float(args['--lmargin']),
- word_margin=float(args['--wmargin']),
- debug=args['--debug'])
- if data is None:
- print
- continue
- csvfile = g_froot + '.csv'
- with open(csvfile, 'w') as outfile:
- writer = csv.writer(outfile)
- for d in data:
- writer.writerow([c.encode('utf-8') for c in d])
- print "saved as", os.path.basename(csvfile)
- print
- except Exception:
- logging.exception("")
- print "couldn't parse", g_fname, "see log for more info"
- print
-
- if args['--log']:
- if args['--output']:
- shutil.copy(logfname, args['--output'])
- else:
- shutil.copy(logfname, fdir)
-
- if args['--debug'] not in [None, False]:
- print "See 'camelot -h' for various parameters you can tweak."
- shutil.rmtree(tmpdir)
- sys.exit()
-
- glob_csv = sorted(glob.glob(os.path.join(tmpdir, '*.csv')), key=filesort)
- if args['--format'] == 'csv':
- if len(glob_csv) == 1:
- if args['--output']:
- shutil.copy(glob_csv[0], args['--output'])
- else:
- shutil.copy(glob_csv[0], fdir)
- else:
- zipname = froot + '.zip'
- zippath = os.path.join(tmpdir, zipname)
- print "zipping 'em up"
- with zipfile.ZipFile(zippath, 'a', zipfile.ZIP_DEFLATED) as myzip:
- for g in glob_csv:
- myzip.write(g, os.path.join(froot, os.path.basename(g)))
- if args['--output']:
- shutil.copy(zippath, args['--output'])
- else:
- shutil.copy(zippath, fdir)
- print
- elif args['--format'] == 'xlsx':
- from pyexcel_xlsx import save_data
- from collections import OrderedDict
- data = OrderedDict()
- for c in glob_csv:
- c_fname = os.path.basename(c)
- c_froot, __ = os.path.splitext(c)
- print "adding", c_fname, "to excel file"
- with open(c, 'r') as csvfile:
- reader = csv.reader(csvfile)
- c_froot, __ = os.path.splitext(c_fname)
- data.update({c_froot: [row for row in reader]})
- xlsxname = froot + '.xlsx'
- xlsxpath = os.path.join(tmpdir, xlsxname)
- save_data(xlsxpath, data)
- if args['--output']:
- shutil.copy(xlsxpath, args['--output'])
- else:
- shutil.copy(xlsxpath, fdir)
- print
- print "saved as", xlsxname
-
- print "cleaning up..."
- shutil.rmtree(tmpdir)
-
- print "finished in", time.time() - start_time, "seconds"
- logging.info("Time taken for " + fname + ": " +
- str(time.time() - start_time) + " seconds")
diff --git a/camelot/__init__.py b/camelot/__init__.py
new file mode 100644
index 0000000..9003377
--- /dev/null
+++ b/camelot/__init__.py
@@ -0,0 +1,7 @@
+from .pdf import Pdf
+from .lattice import Lattice
+from .stream import Stream
+
+__version__ = '0.1'
+
+__all__ = ['Pdf', 'Lattice', 'Stream']
diff --git a/cell.py b/camelot/cell.py
similarity index 83%
rename from cell.py
rename to camelot/cell.py
index ee993ae..515878d 100644
--- a/cell.py
+++ b/camelot/cell.py
@@ -37,8 +37,13 @@ class Cell:
spanning_v : bool
"""
+
def __init__(self, x1, y1, x2, y2):
-
+
+ self.x1 = x1
+ self.y1 = y1
+ self.x2 = x2
+ self.y2 = y2
self.lb = (x1, y1)
self.lt = (x1, y2)
self.rb = (x2, y1)
@@ -53,7 +58,7 @@ class Cell:
self.spanning_v = False
def add_text(self, text):
- """Add text to cell object.
+ """Adds text to cell object.
Parameters
----------
@@ -62,8 +67,8 @@ class Cell:
self.text = ''.join([self.text, text])
def get_text(self):
- """Get text from cell object.
-
+ """Returns text from cell object.
+
Returns
-------
text : string
@@ -71,7 +76,7 @@ class Cell:
return self.text
def get_bounded_edges(self):
- """Get number of edges by which a cell is bounded.
+ """Returns number of edges by which a cell is bounded.
Returns
-------
diff --git a/camelot/lattice.py b/camelot/lattice.py
new file mode 100644
index 0000000..00cf28a
--- /dev/null
+++ b/camelot/lattice.py
@@ -0,0 +1,354 @@
+import os
+
+import cv2
+import numpy as np
+
+from .table import Table
+from .utils import (transform, elements_bbox, detect_vertical, merge_close_values,
+ get_row_index, get_column_index, reduce_index, outline,
+ fill_spanning, remove_empty, encode_list)
+
+
+__all__ = ['Lattice']
+
+
+def _morph_transform(imagename, scale=15, invert=False):
+ """Morphological Transformation
+
+ Applies a series of morphological operations on the image
+ to find table contours and line segments.
+ http://answers.opencv.org/question/63847/how-to-extract-tables-from-an-image/
+
+ Empirical result for adaptiveThreshold's blockSize=5 and C=-0.2
+ taken from http://pequan.lip6.fr/~bereziat/pima/2012/seuillage/sezgin04.pdf
+
+ Parameters
+ ----------
+ imagename : Path to image.
+
+ scale : int
+ Scaling factor. Large scaling factor leads to smaller lines
+ being detected. (optional, default: 15)
+
+ invert : bool
+ Invert pdf image to make sure that lines are in foreground.
+ (optional, default: False)
+
+ Returns
+ -------
+ img : ndarray
+
+ tables : dict
+ Dictionary with table bounding box as key and list of
+ joints found in the table as value.
+
+ v_segments : list
+ List of vertical line segments found in the image.
+
+ h_segments : list
+ List of horizontal line segments found in the image.
+ """
+ img = cv2.imread(imagename)
+ gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+ if invert:
+ threshold = cv2.adaptiveThreshold(
+ gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY,
+ 15, -0.2)
+ else:
+ threshold = cv2.adaptiveThreshold(
+ np.invert(gray), 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+ cv2.THRESH_BINARY,
+ 15, -0.2)
+
+ vertical = threshold
+ horizontal = threshold
+
+ verticalsize = vertical.shape[0] / scale
+ horizontalsize = horizontal.shape[1] / scale
+
+ ver = cv2.getStructuringElement(cv2.MORPH_RECT, (1, verticalsize))
+ hor = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontalsize, 1))
+
+ vertical = cv2.erode(vertical, ver, (-1, -1))
+ vertical = cv2.dilate(vertical, ver, (-1, -1))
+
+ horizontal = cv2.erode(horizontal, hor, (-1, -1))
+ horizontal = cv2.dilate(horizontal, hor, (-1, -1))
+
+ mask = vertical + horizontal
+ joints = np.bitwise_and(vertical, horizontal)
+ __, contours, __ = cv2.findContours(
+ mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+ contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]
+
+ tables = {}
+ for c in contours:
+ c_poly = cv2.approxPolyDP(c, 3, True)
+ x, y, w, h = cv2.boundingRect(c_poly)
+ roi = joints[y : y + h, x : x + w]
+ __, jc, __ = cv2.findContours(
+ roi, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
+ if len(jc) <= 4: # remove contours with less than <=4 joints
+ continue
+ joint_coords = []
+ for j in jc:
+ jx, jy, jw, jh = cv2.boundingRect(j)
+ c1, c2 = x + (2 * jx + jw) / 2, y + (2 * jy + jh) / 2
+ joint_coords.append((c1, c2))
+ tables[(x, y + h, x + w, y)] = joint_coords
+
+ v_segments, h_segments = [], []
+ _, vcontours, _ = cv2.findContours(
+ vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+ for vc in vcontours:
+ x, y, w, h = cv2.boundingRect(vc)
+ x1, x2 = x, x + w
+ y1, y2 = y, y + h
+ v_segments.append(((x1 + x2) / 2, y2, (x1 + x2) / 2, y1))
+
+ _, hcontours, _ = cv2.findContours(
+ horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+ for hc in hcontours:
+ x, y, w, h = cv2.boundingRect(hc)
+ x1, x2 = x, x + w
+ y1, y2 = y, y + h
+ h_segments.append((x1, (y1 + y2) / 2, x2, (y1 + y2) / 2))
+
+ return img, tables, v_segments, h_segments
+
+
+class Lattice:
+ """Lattice algorithm
+
+ Makes use of pdf geometry by processing its image, to make a table
+ and fills text objects in table cells.
+
+ Parameters
+ ----------
+ pdfobject : camelot.pdf.Pdf
+
+ fill : None, 'h', 'v', 'hv'
+ Fill data in horizontal and/or vertical spanning
+ cells. (optional)
+
+ scale : int
+ Scaling factor. Large scaling factor leads to smaller lines
+ being detected. (optional, default: 15)
+
+ jtol : int
+ Tolerance to account for when comparing joint and line
+ coordinates. (optional, default: 2)
+
+ mtol : int
+ Tolerance to account for when merging lines which are
+ very close. (optional, default: 2)
+
+ invert : bool
+ Invert pdf image to make sure that lines are in foreground.
+ (optional, default: False)
+
+ debug : 'contour', 'line', 'joint', 'table'
+ Debug by visualizing pdf geometry.
+
+ Attributes
+ ----------
+ tables : dict
+ Dictionary with page number as key and list of tables on that
+ page as value.
+ """
+
+ def __init__(self, pdfobject, fill=None, scale=15, jtol=2, mtol=2,
+ invert=False, debug=None):
+
+ self.pdfobject = pdfobject
+ self.fill = fill
+ self.scale = scale
+ self.jtol = jtol
+ self.mtol = mtol
+ self.invert = invert
+ self.debug = debug
+ self.tables = {}
+ if self.debug is not None:
+ self.debug_images = {}
+ self.debug_segments = {}
+ self.debug_tables = {}
+
+ def get_tables(self):
+ """Returns all tables found in given pdf.
+
+ Returns
+ -------
+ tables : dict
+ Dictionary with page number as key and list of tables on that
+ page as value.
+ """
+ self.pdfobject.split()
+ self.pdfobject.convert()
+ for page in self.pdfobject.extract():
+ p, text, __, width, height = page
+ pkey = 'pg-{0}'.format(p)
+ imagename = os.path.join(
+ self.pdfobject.temp, '{}.png'.format(pkey))
+ pdf_x = width
+ pdf_y = height
+ img, table_bbox, v_segments, h_segments = _morph_transform(
+ imagename, scale=self.scale, invert=self.invert)
+ img_x = img.shape[1]
+ img_y = img.shape[0]
+ scaling_factor_x = pdf_x / float(img_x)
+ scaling_factor_y = pdf_y / float(img_y)
+
+ if self.debug is not None:
+ self.debug_images[pkey] = (img, table_bbox)
+
+ factors = (scaling_factor_x, scaling_factor_y, img_y)
+ table_bbox, v_segments, h_segments = transform(table_bbox, v_segments,
+ h_segments, factors)
+
+ if self.debug is not None:
+ self.debug_segments[pkey] = (v_segments, h_segments)
+
+ if self.debug is not None:
+ debug_page_tables = []
+ page_tables = []
+ # sort tables based on y-coord
+ for k in sorted(table_bbox.keys(), key=lambda x: x[1], reverse=True):
+ # select edges which lie within table_bbox
+ text_bbox, v_s, h_s = elements_bbox(k, text, v_segments,
+ h_segments)
+ rotated = detect_vertical(text_bbox)
+ cols, rows = zip(*table_bbox[k])
+ cols, rows = list(cols), list(rows)
+ cols.extend([k[0], k[2]])
+ rows.extend([k[1], k[3]])
+ # sort horizontal and vertical segments
+ cols = merge_close_values(sorted(cols), mtol=self.mtol)
+ rows = merge_close_values(
+ sorted(rows, reverse=True), mtol=self.mtol)
+ # make grid using x and y coord of shortlisted rows and cols
+ cols = [(cols[i], cols[i + 1])
+ for i in range(0, len(cols) - 1)]
+ rows = [(rows[i], rows[i + 1])
+ for i in range(0, len(rows) - 1)]
+ table = Table(cols, rows)
+ # set table edges to True using ver+hor lines
+ table = table.set_edges(v_s, h_s, jtol=self.jtol)
+ # set spanning cells to True
+ table = table.set_spanning()
+ # set table border edges to True
+ table = outline(table)
+
+ if self.debug is not None:
+ debug_page_tables.append(table)
+
+ # fill text after sorting it
+ if rotated == '':
+ text_bbox.sort(key=lambda x: (-x.y0, x.x0))
+ elif rotated == 'left':
+ text_bbox.sort(key=lambda x: (x.x0, x.y0))
+ elif rotated == 'right':
+ text_bbox.sort(key=lambda x: (-x.x0, -x.y0))
+ for t in text_bbox:
+ r_idx = get_row_index(t, rows)
+ c_idx = get_column_index(t, cols)
+ if None in [r_idx, c_idx]:
+ # couldn't assign LTChar to any cell
+ pass
+ else:
+ r_idx, c_idx = reduce_index(
+ table, rotated, r_idx, c_idx)
+ table.cells[r_idx][c_idx].add_text(
+ t.get_text().strip('\n'))
+
+ if self.fill is not None:
+ table = fill_spanning(table, fill=self.fill)
+ ar = table.get_list()
+ if rotated == 'left':
+ ar = zip(*ar[::-1])
+ elif rotated == 'right':
+ ar = zip(*ar[::1])
+ ar.reverse()
+ ar = remove_empty(ar)
+ ar = [list(o) for o in ar]
+ page_tables.append(encode_list(ar))
+ print pkey # verbose
+ self.tables[pkey] = page_tables
+
+ if self.debug is not None:
+ self.debug_tables[pkey] = debug_page_tables
+
+ if self.pdfobject.clean:
+ self.pdfobject.remove_tempdir()
+
+ if self.debug is not None:
+ return None
+
+ return self.tables
+
+ def plot_geometry(self, geometry):
+ """Plots various pdf geometries that are detected so user can choose
+ tweak scale, jtol, mtol parameters.
+ """
+ import matplotlib.pyplot as plt
+
+ if geometry == 'contour':
+ for pkey in self.debug_images.keys():
+ img, table_bbox = self.debug_images[pkey]
+ for t in table_bbox.keys():
+ cv2.rectangle(img, (t[0], t[1]),
+ (t[2], t[3]), (255, 0, 0), 3)
+ plt.imshow(img)
+ plt.axis('off')
+ plt.show()
+ elif geometry == 'joint':
+ x_coord = []
+ y_coord = []
+ for pkey in self.debug_images.keys():
+ img, table_bbox = self.debug_images[pkey]
+ for k in table_bbox.keys():
+ for coord in table_bbox[k]:
+ x_coord.append(coord[0])
+ y_coord.append(coord[1])
+ max_x, max_y = max(x_coord), max(y_coord)
+ plt.plot(x_coord, y_coord, 'ro')
+ plt.axis([0, max_x + 100, max_y + 100, 0])
+ plt.imshow(img)
+ plt.axis('off')
+ plt.show()
+ elif geometry == 'line':
+ for pkey in self.debug_segments.keys():
+ v_s, h_s = self.debug_segments[pkey]
+ for v in v_s:
+ plt.plot([v[0], v[2]], [v[1], v[3]])
+ for h in h_s:
+ plt.plot([h[0], h[2]], [h[1], h[3]])
+ plt.axis('off')
+ plt.show()
+ elif geometry == 'table':
+ for pkey in self.debug_tables.keys():
+ for table in self.debug_tables[pkey]:
+ for i in range(len(table.cells)):
+ for j in range(len(table.cells[i])):
+ if table.cells[i][j].left:
+ plt.plot([table.cells[i][j].lb[0],
+ table.cells[i][j].lt[0]],
+ [table.cells[i][j].lb[1],
+ table.cells[i][j].lt[1]])
+ if table.cells[i][j].right:
+ plt.plot([table.cells[i][j].rb[0],
+ table.cells[i][j].rt[0]],
+ [table.cells[i][j].rb[1],
+ table.cells[i][j].rt[1]])
+ if table.cells[i][j].top:
+ plt.plot([table.cells[i][j].lt[0],
+ table.cells[i][j].rt[0]],
+ [table.cells[i][j].lt[1],
+ table.cells[i][j].rt[1]])
+ if table.cells[i][j].bottom:
+ plt.plot([table.cells[i][j].lb[0],
+ table.cells[i][j].rb[0]],
+ [table.cells[i][j].lb[1],
+ table.cells[i][j].rb[1]])
+ plt.axis('off')
+ plt.show()
\ No newline at end of file
diff --git a/camelot/pdf.py b/camelot/pdf.py
new file mode 100644
index 0000000..2988e3d
--- /dev/null
+++ b/camelot/pdf.py
@@ -0,0 +1,178 @@
+import os
+import shutil
+import tempfile
+
+from PyPDF2 import PdfFileReader, PdfFileWriter
+from pdfminer.pdfparser import PDFParser
+from pdfminer.pdfdocument import PDFDocument
+from pdfminer.pdfpage import PDFPage
+from pdfminer.pdfpage import PDFTextExtractionNotAllowed
+from pdfminer.pdfinterp import PDFResourceManager
+from pdfminer.pdfinterp import PDFPageInterpreter
+from pdfminer.pdfdevice import PDFDevice
+from pdfminer.converter import PDFPageAggregator
+from pdfminer.layout import LAParams, LTChar, LTTextLineHorizontal
+from wand.image import Image
+
+
+__all__ = ['Pdf']
+
+
+def _parse_page_numbers(pagenos):
+ """Converts list of page ranges to a list of page numbers.
+
+ Parameters
+ ----------
+ pagenos : list
+ List of dicts containing page ranges.
+
+ Returns
+ -------
+ page_numbers : list
+ List of page numbers.
+ """
+ page_numbers = []
+ for p in pagenos:
+ page_numbers.extend(range(p['start'], p['end'] + 1))
+ page_numbers = sorted(set(page_numbers))
+ return page_numbers
+
+
+def _extract_text_objects(layout, LTObject, t=None):
+ """Recursively parses pdf layout to get a list of
+ text objects.
+
+ Parameters
+ ----------
+ layout : object
+ Layout object.
+
+ LTObject : object
+ Text object, either LTChar or LTTextLineHorizontal.
+
+ t : list
+
+ Returns
+ -------
+ t : list
+ List of text objects.
+ """
+ if t is None:
+ t = []
+ try:
+ for obj in layout._objs:
+ if isinstance(obj, LTObject):
+ t.append(obj)
+ else:
+ t += _extract_text_objects(obj, LTObject)
+ except AttributeError:
+ pass
+ return t
+
+
+class Pdf:
+ """Handles all pdf operations which include:
+
+ 1. Split pdf into single page pdfs using given page numbers
+ 2. Convert single page pdfs into images
+ 3. Extract text from single page pdfs
+
+ Parameters
+ ----------
+ pdfname : string
+ Path to pdf.
+
+ pagenos : list
+ List of dicts which specify pdf page ranges.
+
+ char_margin : float
+ Chars closer than char_margin are grouped together to form a
+ word. (optional, default: 2.0)
+
+ line_margin : float
+ Lines closer than line_margin are grouped together to form a
+ textbox. (optional, default: 0.5)
+
+ word_margin : float
+ Insert blank spaces between chars if distance between words
+ is greater than word_margin. (optional, default: 0.1)
+
+ Attributes
+ ----------
+ temp : string
+ Path to temporary directory.
+
+ lattice_objects : dict
+ List of text objects.
+
+ stream_objects : dict
+ List of text objects.
+
+ width : dict
+ List of dicts with width of each pdf page.
+
+ height : dict
+ List of dicts with height of each pdf page.
+ """
+
+ def __init__(self, pdfname, pagenos=[{'start': 1, 'end': 1}],
+ char_margin=2.0, line_margin=0.5, word_margin=0.1,
+ clean=False):
+
+ self.pdfname = pdfname
+ self.pagenos = _parse_page_numbers(pagenos)
+ self.char_margin = char_margin
+ self.line_margin = line_margin
+ self.word_margin = word_margin
+ self.clean = clean
+ self.temp = tempfile.mkdtemp()
+
+ def split(self):
+ """Splits pdf into single page pdfs.
+ """
+ infile = PdfFileReader(open(self.pdfname, 'rb'), strict=False)
+ for p in self.pagenos:
+ page = infile.getPage(p - 1)
+ outfile = PdfFileWriter()
+ outfile.addPage(page)
+ with open(os.path.join(self.temp, 'pg-{0}.pdf'.format(p)), 'wb') as f:
+ outfile.write(f)
+
+ def extract(self):
+ """Extracts text objects, width, height from a pdf.
+ """
+ for p in self.pagenos:
+ pkey = 'pg-{0}'.format(p)
+ pname = os.path.join(self.temp, '{}.pdf'.format(pkey))
+ with open(pname, 'r') as f:
+ parser = PDFParser(f)
+ document = PDFDocument(parser)
+ if not document.is_extractable:
+ raise PDFTextExtractionNotAllowed
+ laparams = LAParams(char_margin=self.char_margin,
+ line_margin=self.line_margin,
+ word_margin=self.word_margin)
+ rsrcmgr = PDFResourceManager()
+ device = PDFPageAggregator(rsrcmgr, laparams=laparams)
+ interpreter = PDFPageInterpreter(rsrcmgr, device)
+ for page in PDFPage.create_pages(document):
+ interpreter.process_page(page)
+ layout = device.get_result()
+ lattice_objects = _extract_text_objects(layout, LTChar)
+ stream_objects = _extract_text_objects(
+ layout, LTTextLineHorizontal)
+ width = layout.bbox[2]
+ height = layout.bbox[3]
+ yield p, lattice_objects, stream_objects, width, height
+
+ def convert(self):
+ """Converts single page pdfs to images.
+ """
+ for p in self.pagenos:
+ pdfname = os.path.join(self.temp, 'pg-{0}.pdf'.format(p))
+ imagename = os.path.join(self.temp, 'pg-{0}.png'.format(p))
+ with Image(filename=pdfname, depth=8, resolution=300) as png:
+ png.save(filename=imagename)
+
+ def remove_tempdir(self):
+ shutil.rmtree(self.temp)
diff --git a/camelot/stream.py b/camelot/stream.py
new file mode 100644
index 0000000..320a6e6
--- /dev/null
+++ b/camelot/stream.py
@@ -0,0 +1,210 @@
+import os
+
+import numpy as np
+
+from .utils import get_column_index, encode_list
+
+
+__all__ = ['Stream']
+
+
+def _group_rows(text, ytol=2):
+ """Groups text objects into rows using ytol.
+
+ Parameters
+ ----------
+ text : list
+ List of text objects.
+
+ ytol : int
+ Tolerance to account for when grouping rows
+ together. (default: 2, optional)
+
+ Returns
+ -------
+ rows : list
+ List of grouped text rows.
+ """
+ row_y = 0
+ rows = []
+ temp = []
+ for t in text:
+ # is checking for upright necessary?
+ # if t.get_text().strip() and all([obj.upright for obj in t._objs if
+ # type(obj) is LTChar]):
+ if t.get_text().strip():
+ if not np.isclose(row_y, t.y0, atol=ytol):
+ row_y = t.y0
+ rows.append(temp)
+ temp = []
+ temp.append(t)
+ return rows
+
+
+def _merge_columns(l):
+ """Merges overlapping columns and returns list with updated
+ columns boundaries.
+
+ Parameters
+ ----------
+ l : list
+ List of column x-coordinates.
+
+ Returns
+ -------
+ merged : list
+ List of merged column x-coordinates.
+ """
+ merged = []
+ for higher in l:
+ if not merged:
+ merged.append(higher)
+ else:
+ lower = merged[-1]
+ if higher[0] <= lower[1]:
+ upper_bound = max(lower[1], higher[1])
+ lower_bound = min(lower[0], higher[0])
+ merged[-1] = (lower_bound, upper_bound)
+ else:
+ merged.append(higher)
+ return merged
+
+
+class Stream:
+ """Stream algorithm
+
+ Groups text objects into rows and guesses number of columns
+ using mode of the number of text objects in each row.
+
+ The number of columns can be passed explicitly or specified by a
+ list of column x-coordinates.
+
+ Parameters
+ ----------
+ pdfobject : camelot.pdf.Pdf
+
+ ncolumns : int
+ Number of columns. (optional, default: 0)
+
+ columns : string
+ Comma-separated list of column x-coordinates.
+ (optional, default: None)
+
+ ytol : int
+ Tolerance to account for when grouping rows
+ together. (optional, default: 2)
+
+ debug : bool
+ Debug by visualizing textboxes. (optional, default: False)
+
+ Attributes
+ ----------
+ tables : dict
+ Dictionary with page number as key and list of tables on that
+ page as value.
+ """
+
+ def __init__(self, pdfobject, ncolumns=0, columns=None, ytol=2,
+ debug=False):
+
+ self.pdfobject = pdfobject
+ self.ncolumns = ncolumns
+ self.columns = columns
+ self.ytol = ytol
+ self.debug = debug
+ self.tables = {}
+ if self.debug:
+ self.debug_text = {}
+
+ def get_tables(self):
+ """Returns all tables found in given pdf.
+
+ Returns
+ -------
+ tables : dict
+ Dictionary with page number as key and list of tables on that
+ page as value.
+ """
+ self.pdfobject.split()
+ for page in self.pdfobject.extract():
+ p, __, text, __, __ = page
+ pkey = 'pg-{0}'.format(p)
+ text.sort(key=lambda x: (-x.y0, x.x0))
+
+ if self.debug:
+ self.debug_text[pkey] = text
+
+ rows = _group_rows(text, ytol=self.ytol)
+ elements = [len(r) for r in rows]
+ # a table can't have just 1 column, can it?
+ elements = filter(lambda x: x != 1, elements)
+
+ guess = False
+ if self.columns:
+ cols = self.columns.split(',')
+ cols = [(float(cols[i]), float(cols[i + 1]))
+ for i in range(0, len(cols) - 1)]
+ else:
+ guess = True
+ ncols = self.ncolumns if self.ncolumns else max(
+ set(elements), key=elements.count)
+ if ncols == 0:
+ # no tables detected
+ continue
+ cols = [(t.x0, t.x1)
+ for r in rows for t in r if len(r) == ncols]
+ cols = _merge_columns(sorted(cols))
+ cols = [(c[0] + c[1]) / 2.0 for c in cols]
+
+ ar = [['' for c in cols] for r in rows]
+ for r_idx, r in enumerate(rows):
+ for t in r:
+ if guess:
+ cog = (t.x0 + t.x1) / 2.0
+ diff = [abs(cog - c) for c in cols]
+ c_idx = diff.index(min(diff))
+ else:
+ c_idx = get_column_index(t, cols)
+ if None in [r_idx, c_idx]: # couldn't assign LTTextLH to any cell
+ continue
+ if ar[r_idx][c_idx]:
+ ar[r_idx][c_idx] = ' '.join(
+ [ar[r_idx][c_idx], t.get_text().strip()])
+ else:
+ ar[r_idx][c_idx] = t.get_text().strip()
+ print pkey # verbose
+ self.tables[pkey] = [encode_list(ar)]
+
+ if self.pdfobject.clean:
+ self.pdfobject.remove_tempdir()
+
+ if self.debug:
+ return None
+
+ return self.tables
+
+ def plot_text(self):
+ """Plots all text objects so user can choose number of columns
+ or columns x-coordinates using the matplotlib interface.
+ """
+ import matplotlib.pyplot as plt
+ import matplotlib.patches as patches
+
+ for pkey in sorted(self.debug_text.keys()):
+ fig = plt.figure()
+ ax = fig.add_subplot(111, aspect='equal')
+ xs, ys = [], []
+ for t in self.debug_text[pkey]:
+ xs.extend([t.x0, t.x1])
+ ys.extend([t.y0, t.y1])
+ ax.add_patch(
+ patches.Rectangle(
+ (t.x0, t.y0),
+ t.x1 - t.x0,
+ t.y1 - t.y0
+ )
+ )
+ ax.set_xlim(min(xs) - 10, max(xs) + 10)
+ ax.set_ylim(min(ys) - 10, max(ys) + 10)
+ plt.axis('off')
+ plt.show()
diff --git a/table.py b/camelot/table.py
similarity index 86%
rename from table.py
rename to camelot/table.py
index d38279c..a1a8c29 100644
--- a/table.py
+++ b/camelot/table.py
@@ -1,14 +1,14 @@
import numpy as np
-from cell import Cell
+from .cell import Cell
class Table:
"""Table
-
+
Parameters
----------
- columns : list
+ cols : list
List of column x-coordinates.
rows : list
@@ -18,22 +18,17 @@ class Table:
----------
cells : list
2-D list of cell objects.
-
- columns : list
- List of column x-coordinates.
-
- rows : list
- List of row y-coordinates.
"""
- def __init__(self, columns, rows):
- self.cells = [[Cell(c[0], r[1], c[1], r[0])
- for c in columns] for r in rows]
- self.columns = columns
+ def __init__(self, cols, rows):
+
+ self.cols = cols
self.rows = rows
+ self.cells = [[Cell(c[0], r[1], c[1], r[0])
+ for c in cols] for r in rows]
def set_edges(self, vertical, horizontal, jtol=2):
- """Set cell edges to True if corresponding line segments
+ """Sets cell edges to True if corresponding line segments
are detected in the pdf image.
Parameters
@@ -47,16 +42,11 @@ class Table:
jtol : int, default: 2, optional
Tolerance to account for when comparing joint and line
coordinates.
-
- Returns
- -------
- self : object
- Returns self.
"""
for v in vertical:
# find closest x coord
# iterate over y coords and find closest points
- i = [i for i, t in enumerate(self.columns)
+ i = [i for i, t in enumerate(self.cols)
if np.isclose(v[0], t[0], atol=jtol)]
j = [j for j, t in enumerate(self.rows)
if np.isclose(v[3], t[0], atol=jtol)]
@@ -78,7 +68,7 @@ class Table:
self.cells[J][I].left = True
J += 1
elif i == []: # only right edge
- I = len(self.columns) - 1
+ I = len(self.cols) - 1
if k:
K = k[0]
while J < K:
@@ -109,9 +99,9 @@ class Table:
# iterate over x coords and find closest points
i = [i for i, t in enumerate(self.rows)
if np.isclose(h[1], t[0], atol=jtol)]
- j = [j for j, t in enumerate(self.columns)
+ j = [j for j, t in enumerate(self.cols)
if np.isclose(h[0], t[0], atol=jtol)]
- k = [k for k, t in enumerate(self.columns)
+ k = [k for k, t in enumerate(self.cols)
if np.isclose(h[2], t[0], atol=jtol)]
if not j:
continue
@@ -124,7 +114,7 @@ class Table:
self.cells[I][J].top = True
J += 1
else:
- K = len(self.columns)
+ K = len(self.cols)
while J < K:
self.cells[I][J].top = True
J += 1
@@ -136,7 +126,7 @@ class Table:
self.cells[I][J].bottom = True
J += 1
else:
- K = len(self.columns)
+ K = len(self.cols)
while J < K:
self.cells[I][J].bottom = True
J += 1
@@ -149,7 +139,7 @@ class Table:
self.cells[I - 1][J].bottom = True
J += 1
else:
- K = len(self.columns)
+ K = len(self.cols)
while J < K:
self.cells[I][J].top = True
self.cells[I - 1][J].bottom = True
@@ -158,13 +148,8 @@ class Table:
return self
def set_spanning(self):
- """Set spanning values of a cell to True if it isn't
+ """Sets spanning values of a cell to True if it isn't
bounded by four edges.
-
- Returns
- -------
- self : object
- Returns self.
"""
for i in range(len(self.cells)):
for j in range(len(self.cells[i])):
@@ -175,7 +160,7 @@ class Table:
elif bound == 3:
if not self.cells[i][j].left:
if (self.cells[i][j].right and
- self.cells[i][j].top and
+ self.cells[i][j].top and
self.cells[i][j].bottom):
self.cells[i][j].spanning_h = True
@@ -209,3 +194,16 @@ class Table:
self.cells[i][j].spanning_h = True
return self
+
+ def get_list(self):
+ """Returns text from all cells as list of lists.
+
+ Returns
+ -------
+ ar : list
+ """
+ ar = []
+ for i in range(len(self.cells)):
+ ar.append([self.cells[i][j].get_text().strip()
+ for j in range(len(self.cells[i]))])
+ return ar
diff --git a/utils.py b/camelot/utils.py
similarity index 54%
rename from utils.py
rename to camelot/utils.py
index 89d037c..8ef0db0 100644
--- a/utils.py
+++ b/camelot/utils.py
@@ -2,7 +2,7 @@ import numpy as np
def translate(x1, x2):
- """Translate coordinate x2 by x1.
+ """Translates x2 by x1.
Parameters
----------
@@ -19,7 +19,7 @@ def translate(x1, x2):
def scale(x, s):
- """Scale coordinate x by scaling factor s.
+ """Scales x by scaling factor s.
Parameters
----------
@@ -36,7 +36,7 @@ def scale(x, s):
def rotate(x1, y1, x2, y2, angle):
- """Rotate point x2, y2 about point x1, y1 by angle.
+ """Rotates point x2, y2 about point x1, y1 by angle.
Parameters
----------
@@ -68,15 +68,126 @@ def rotate(x1, y1, x2, y2, angle):
return xnew, ynew
+def transform(tables, v_segments, h_segments, factors):
+ """Translates and scales OpenCV coordinates to PDFMiner coordinate
+ space.
+
+ Parameters
+ ----------
+ tables : dict
+
+ v_segments : list
+
+ h_segments : list
+
+ factors : tuple
+
+ Returns
+ -------
+ tables_new : dict
+
+ v_segments_new : dict
+
+ h_segments_new : dict
+ """
+ scaling_factor_x, scaling_factor_y, img_y = factors
+ tables_new = {}
+ for k in tables.keys():
+ x1, y1, x2, y2 = k
+ x1 = scale(x1, scaling_factor_x)
+ y1 = scale(abs(translate(-img_y, y1)), scaling_factor_y)
+ x2 = scale(x2, scaling_factor_x)
+ y2 = scale(abs(translate(-img_y, y2)), scaling_factor_y)
+ j_x, j_y = zip(*tables[k])
+ j_x = [scale(j, scaling_factor_x) for j in j_x]
+ j_y = [scale(abs(translate(-img_y, j)), scaling_factor_y) for j in j_y]
+ joints = zip(j_x, j_y)
+ tables_new[(x1, y1, x2, y2)] = joints
+
+ v_segments_new = []
+ for v in v_segments:
+ x1, x2 = scale(v[0], scaling_factor_x), scale(v[2], scaling_factor_x)
+ y1, y2 = scale(abs(translate(-img_y, v[1])), scaling_factor_y), scale(
+ abs(translate(-img_y, v[3])), scaling_factor_y)
+ v_segments_new.append((x1, y1, x2, y2))
+
+ h_segments_new = []
+ for h in h_segments:
+ x1, x2 = scale(h[0], scaling_factor_x), scale(h[2], scaling_factor_x)
+ y1, y2 = scale(abs(translate(-img_y, h[1])), scaling_factor_y), scale(
+ abs(translate(-img_y, h[3])), scaling_factor_y)
+ h_segments_new.append((x1, y1, x2, y2))
+
+ return tables_new, v_segments_new, h_segments_new
+
+
+def detect_vertical(text):
+ """Detects if text in table is vertical or not and returns
+ its orientation.
+
+ Parameters
+ ----------
+ text : list
+
+ Returns
+ -------
+ rotated : string
+ """
+ num_v = [t for t in text if (not t.upright) and t.get_text().strip()]
+ num_h = [t for t in text if t.upright and t.get_text().strip()]
+ vger = len(num_v) / float(len(num_v) + len(num_h))
+ rotated = ''
+ if vger > 0.8:
+ clockwise = sum(t.matrix[1] < 0 and t.matrix[2] > 0 for t in text)
+ anticlockwise = sum(t.matrix[1] > 0 and t.matrix[2] < 0 for t in text)
+ rotated = 'left' if clockwise < anticlockwise else 'right'
+ return rotated
+
+
+def elements_bbox(bbox, text, v_segments, h_segments):
+ """Returns all text objects and line segments present inside a
+ table's bounding box.
+
+ Parameters
+ ----------
+ bbox : tuple
+
+ text : list
+
+ v_segments : list
+
+ h_segments : list
+
+ Returns
+ -------
+ text_bbox : list
+
+ v_s : list
+
+ h_s : list
+ """
+ lb = (bbox[0], bbox[1])
+ rt = (bbox[2], bbox[3])
+ text_bbox = [t for t in text if lb[0] - 2 <= (t.x0 + t.x1) / 2.0
+ <= rt[0] + 2 and lb[1] - 2 <= (t.y0 + t.y1) / 2.0
+ <= rt[1] + 2]
+ v_s = [v for v in v_segments if v[1] > lb[1] - 2 and
+ v[3] < rt[1] + 2 and lb[0] - 2 <= v[0] <= rt[0] + 2]
+ h_s = [h for h in h_segments if h[0] > lb[0] - 2 and
+ h[2] < rt[0] + 2 and lb[1] - 2 <= h[1] <= rt[1] + 2]
+ return text_bbox, v_s, h_s
+
+
def remove_close_values(ar, mtol=2):
- """Remove values which are within a tolerance of mtol of another value
+ """Removes values which are within a tolerance of mtol of another value
present in list.
Parameters
----------
ar : list
- mtol : int, default: 2, optional
+ mtol : int
+ (optional, default: 2)
Returns
-------
@@ -96,14 +207,15 @@ def remove_close_values(ar, mtol=2):
def merge_close_values(ar, mtol=2):
- """Merge values which are within a tolerance of mtol by calculating
+ """Merges values which are within a tolerance of mtol by calculating
a moving mean.
Parameters
----------
ar : list
- mtol : int, default: 2, optional
+ mtol : int
+ (optional, default: 2)
Returns
-------
@@ -123,8 +235,8 @@ def merge_close_values(ar, mtol=2):
return ret
-def get_row_idx(t, rows):
- """Get index of the row in which the given object falls by
+def get_row_index(t, rows):
+ """Gets index of the row in which the given object falls by
comparing their co-ordinates.
Parameters
@@ -142,8 +254,8 @@ def get_row_idx(t, rows):
return r
-def get_column_idx(t, columns):
- """Get index of the column in which the given object falls by
+def get_column_index(t, columns):
+ """Gets index of the column in which the given object falls by
comparing their co-ordinates.
Parameters
@@ -162,8 +274,8 @@ def get_column_idx(t, columns):
def reduce_index(t, rotated, r_idx, c_idx):
- """Shift a text object if it lies within a spanning cell taking
- in account table rotation.
+ """Reduces index of a text object if it lies within a spanning
+ cell taking in account table rotation.
Parameters
----------
@@ -206,7 +318,7 @@ def reduce_index(t, rotated, r_idx, c_idx):
def outline(t):
- """Light up table boundary.
+ """Sets table border edges to True.
Parameters
----------
@@ -225,32 +337,33 @@ def outline(t):
return t
-def fill(t, f=None):
- """Fill spanning cells.
+def fill_spanning(t, fill=None):
+ """Fills spanning cells.
Parameters
----------
t : object
- f : string, default: None, optional
+ f : string
+ (optional, default: None)
Returns
-------
t : object
"""
- if f == "h":
+ if fill == "h":
for i in range(len(t.cells)):
for j in range(len(t.cells[i])):
if t.cells[i][j].get_text().strip() == '':
if t.cells[i][j].spanning_h:
t.cells[i][j].add_text(t.cells[i][j - 1].get_text())
- elif f == "v":
+ elif fill == "v":
for i in range(len(t.cells)):
for j in range(len(t.cells[i])):
if t.cells[i][j].get_text().strip() == '':
if t.cells[i][j].spanning_v:
t.cells[i][j].add_text(t.cells[i - 1][j].get_text())
- elif f == "hv":
+ elif fill == "hv":
for i in range(len(t.cells)):
for j in range(len(t.cells[i])):
if t.cells[i][j].get_text().strip() == '':
@@ -262,7 +375,7 @@ def fill(t, f=None):
def remove_empty(d):
- """Remove empty rows and columns.
+ """Removes empty rows and columns from list of lists.
Parameters
----------
@@ -279,3 +392,8 @@ def remove_empty(d):
d = [list(row) for row in d if any(row)]
d = zip(*d)
return d
+
+
+def encode_list(ar):
+ ar = [[r.encode('utf-8') for r in row] for row in ar]
+ return ar
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..2e38f75
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,225 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+ @echo "Please use \`make ' where is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " applehelp to make an Apple Help Book"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " epub3 to make an epub3"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " texinfo to make Texinfo files"
+ @echo " info to make Texinfo files and run them through makeinfo"
+ @echo " gettext to make PO message catalogs"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " xml to make Docutils-native XML files"
+ @echo " pseudoxml to make pseudoxml-XML files for display purposes"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+ @echo " coverage to run coverage check of the documentation (if enabled)"
+ @echo " dummy to check syntax errors of document sources"
+
+.PHONY: clean
+clean:
+ rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: singlehtml
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+.PHONY: pickle
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+.PHONY: json
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+.PHONY: htmlhelp
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+.PHONY: qthelp
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/camelot.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/camelot.qhc"
+
+.PHONY: applehelp
+applehelp:
+ $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+ @echo
+ @echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+ @echo "N.B. You won't be able to view it unless you put it in" \
+ "~/Library/Documentation/Help or install it in your application" \
+ "bundle."
+
+.PHONY: devhelp
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/camelot"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/camelot"
+ @echo "# devhelp"
+
+.PHONY: epub
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+.PHONY: epub3
+epub3:
+ $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
+ @echo
+ @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
+
+.PHONY: latex
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+.PHONY: latexpdf
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: latexpdfja
+latexpdfja:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through platex and dvipdfmx..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: text
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+.PHONY: man
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+.PHONY: texinfo
+texinfo:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo
+ @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+ @echo "Run \`make' in that directory to run these through makeinfo" \
+ "(use \`make info' here to do that automatically)."
+
+.PHONY: info
+info:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo "Running Texinfo files through makeinfo..."
+ make -C $(BUILDDIR)/texinfo info
+ @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+.PHONY: gettext
+gettext:
+ $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+ @echo
+ @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+.PHONY: changes
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+ $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+ @echo "Testing of coverage in the sources finished, look at the " \
+ "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: xml
+xml:
+ $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+ @echo
+ @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+.PHONY: pseudoxml
+pseudoxml:
+ $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+ @echo
+ @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+.PHONY: dummy
+dummy:
+ $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
+ @echo
+ @echo "Build finished. Dummy builder generates no files."
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..ca51963
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,352 @@
+# -*- coding: utf-8 -*-
+#
+# camelot documentation build configuration file, created by
+# sphinx-quickstart on Tue Jul 19 13:44:18 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.napoleon',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.todo',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.viewcode',
+ 'sphinx.ext.githubpages',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'camelot'
+copyright = u'2016, SocialCops'
+author = u'Vinayak Mehta'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+
+import camelot
+# The short X.Y version.
+version = camelot.__version__
+# The full version, including alpha/beta/rc tags.
+release = camelot.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#
+# today = ''
+#
+# Else, today_fmt is used as the format for a strftime call.
+#
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.
+# " v documentation" by default.
+#
+# html_title = u'camelot v0.1'
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#
+# html_logo = None
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#
+# html_extra_path = []
+
+# If not None, a 'Last updated on:' timestamp is inserted at every page
+# bottom, using the given strftime format.
+# The empty string is equivalent to '%b %d, %Y'.
+#
+# html_last_updated_fmt = None
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+#
+# html_domain_indices = True
+
+# If false, no index is generated.
+#
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
+#
+# html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# 'ja' uses this config value.
+# 'zh' user can custom change `jieba` dictionary path.
+#
+# html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#
+# html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'camelotdoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'camelot.tex', u'camelot Documentation',
+ u'Vinayak Mehta', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+#
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#
+# latex_appendices = []
+
+# It false, will not define \strong, \code, itleref, \crossref ... but only
+# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
+# packages.
+#
+# latex_keep_old_macro_names = True
+
+# If false, no module index is generated.
+#
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'camelot', u'camelot Documentation',
+ [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'camelot', u'camelot Documentation',
+ author, 'camelot', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+#
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#
+# texinfo_no_detailmenu = False
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/2': None}
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..633973d
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,22 @@
+.. camelot documentation master file, created by
+ sphinx-quickstart on Tue Jul 19 13:44:18 2016.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+camelot: Parse tables from PDFs!
+================================
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000..99887b0
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,281 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+ set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^` where ^ is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. epub3 to make an epub3
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. texinfo to make Texinfo files
+ echo. gettext to make PO message catalogs
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. xml to make Docutils-native XML files
+ echo. pseudoxml to make pseudoxml-XML files for display purposes
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ echo. coverage to run coverage check of the documentation if enabled
+ echo. dummy to check syntax errors of document sources
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+
+REM Check if sphinx-build is available and fallback to Python version if any
+%SPHINXBUILD% 1>NUL 2>NUL
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.http://sphinx-doc.org/
+ exit /b 1
+)
+
+:sphinx_ok
+
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\camelot.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\camelot.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "epub3" (
+ %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub3 file is in %BUILDDIR%/epub3.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdf" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdfja" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf-ja
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "texinfo" (
+ %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+ goto end
+)
+
+if "%1" == "gettext" (
+ %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+if "%1" == "coverage" (
+ %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of coverage in the sources finished, look at the ^
+results in %BUILDDIR%/coverage/python.txt.
+ goto end
+)
+
+if "%1" == "xml" (
+ %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The XML files are in %BUILDDIR%/xml.
+ goto end
+)
+
+if "%1" == "pseudoxml" (
+ %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+ goto end
+)
+
+if "%1" == "dummy" (
+ %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. Dummy builder generates no files.
+ goto end
+)
+
+:end
diff --git a/lattice.py b/lattice.py
deleted file mode 100644
index 0966b63..0000000
--- a/lattice.py
+++ /dev/null
@@ -1,313 +0,0 @@
-import os
-import cv2
-import glob
-import numpy as np
-from wand.image import Image
-
-from table import Table
-from pdf import get_pdf_info
-from utils import (translate, scale, merge_close_values, get_row_idx,
- get_column_idx, reduce_index, outline, fill, remove_empty)
-
-
-def morph_transform(img, s=15, invert=False):
- """Morphological Transformation
-
- Applies a series of morphological operations on the image
- to find table contours and line segments.
- http://answers.opencv.org/question/63847/how-to-extract-tables-from-an-image/
-
- Empirical result for adaptiveThreshold's blockSize=5 and C=-0.2
- taken from http://pequan.lip6.fr/~bereziat/pima/2012/seuillage/sezgin04.pdf
-
- Parameters
- ----------
- img : ndarray
-
- s : int, default: 15, optional
- Scaling factor. Large scaling factor leads to smaller lines
- being detected.
-
- invert : bool, default: False, optional
- Invert pdf image to make sure that lines are in foreground.
-
- Returns
- -------
- tables : dict
- Dictionary with table bounding box as key and list of
- joints found in the table as value.
-
- v_segments : list
- List of vertical line segments found in the image.
-
- h_segments : list
- List of horizontal line segments found in the image.
- """
- img_x, img_y = img.shape[1], img.shape[0]
- gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
-
- if invert:
- threshold = cv2.adaptiveThreshold(
- gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 15, -0.2)
- else:
- threshold = cv2.adaptiveThreshold(np.invert(
- gray), 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 15, -0.2)
- vertical = threshold
- horizontal = threshold
-
- scale = s
- verticalsize = vertical.shape[0] / scale
- horizontalsize = horizontal.shape[1] / scale
-
- ver = cv2.getStructuringElement(cv2.MORPH_RECT, (1, verticalsize))
- hor = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontalsize, 1))
-
- vertical = cv2.erode(vertical, ver, (-1, -1))
- vertical = cv2.dilate(vertical, ver, (-1, -1))
-
- horizontal = cv2.erode(horizontal, hor, (-1, -1))
- horizontal = cv2.dilate(horizontal, hor, (-1, -1))
-
- mask = vertical + horizontal
- joints = np.bitwise_and(vertical, horizontal)
- __, contours, __ = cv2.findContours(
- mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
- contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]
-
- tables = {}
- for c in contours:
- c_poly = cv2.approxPolyDP(c, 3, True)
- x, y, w, h = cv2.boundingRect(c_poly)
- # find number of non-zero values in joints using what boundingRect
- # returns
- roi = joints[y : y + h, x : x + w]
- __, jc, __ = cv2.findContours(
- roi, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
- if len(jc) <= 4: # remove contours with less than <=4 joints
- continue
- joint_coords = []
- for j in jc:
- jx, jy, jw, jh = cv2.boundingRect(j)
- c1, c2 = x + (2 * jx + jw) / 2, y + (2 * jy + jh) / 2
- joint_coords.append((c1, c2))
- tables[(x, y + h, x + w, y)] = joint_coords
-
- v_segments, h_segments = [], []
- _, vcontours, _ = cv2.findContours(
- vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
- for vc in vcontours:
- x, y, w, h = cv2.boundingRect(vc)
- x1, x2 = x, x + w
- y1, y2 = y, y + h
- v_segments.append(((x1 + x2) / 2, y2, (x1 + x2) / 2, y1))
-
- _, hcontours, _ = cv2.findContours(
- horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
- for hc in hcontours:
- x, y, w, h = cv2.boundingRect(hc)
- x1, x2 = x, x + w
- y1, y2 = y, y + h
- h_segments.append((x1, (y1 + y2) / 2, x2, (y1 + y2) / 2))
-
- return tables, v_segments, h_segments
-
-
-def lattice(filepath, f=None, s=15, jtol=2, mtol=2, invert=False, debug=None):
- """Lattice algorithm
-
- Makes table using pdf geometry information returned by
- morph_transform and fills data returned by PDFMiner in table cells.
-
- Parameters
- ----------
- filepath : string
-
- f : string, default: None, optional
- Fill data in horizontal and/or vertical spanning
- cells. ('h', 'v', 'hv')
-
- s : int, default: 15, optional
- Scaling factor. Large scaling factor leads to smaller lines
- being detected.
-
- jtol : int, default: 2, optional
- Tolerance to account for when comparing joint and line
- coordinates.
-
- mtol : int, default: 2, optional
- Tolerance to account for when merging lines which are
- very close.
-
- invert : bool, default: False, optional
- Invert pdf image to make sure that lines are in foreground.
-
- debug : string
- Debug by visualizing pdf geometry.
- ('contour', 'line', 'joint', 'table')
- Returns
- -------
- output : dict
- Dictionary with table number as key and list of data as value.
- """
- if debug:
- import matplotlib.pyplot as plt
- fileroot, __ = os.path.splitext(filepath)
- imagename = fileroot + '.png'
- with Image(filename=filepath, depth=8, resolution=300) as png:
- png.save(filename=imagename)
- img = cv2.imread(imagename)
- img_x, img_y = img.shape[1], img.shape[0]
- text, pdf_x, pdf_y = get_pdf_info(filepath, method='lattice')
- scaling_factor_x = pdf_x / float(img_x)
- scaling_factor_y = pdf_y / float(img_y)
- tables, v_segments, h_segments = morph_transform(img, s=s, invert=invert)
-
- if debug == "contour":
- for t in tables.keys():
- cv2.rectangle(img, (t[0], t[1]), (t[2], t[3]), (255, 0, 0), 3)
- plt.imshow(img)
- plt.show()
- return None
- if debug == "joint":
- x_coord = []
- y_coord = []
- for k in tables.keys():
- for coord in tables[k]:
- x_coord.append(coord[0])
- y_coord.append(coord[1])
- max_x, max_y = max(x_coord), max(y_coord)
- plt.plot(x_coord, y_coord, 'ro')
- plt.axis([0, max_x + 100, max_y + 100, 0])
- plt.imshow(img)
- plt.show()
- return None
-
- # detect if vertical
- num_v = [t for t in text if (not t.upright) and t.get_text().strip()]
- num_h = [t for t in text if t.upright and t.get_text().strip()]
- vger = len(num_v) / float(len(num_v) + len(num_h))
- rotated = ''
- if vger > 0.8:
- clockwise = sum(t.matrix[1] < 0 and t.matrix[2] > 0 for t in text)
- anticlockwise = sum(t.matrix[1] > 0 and t.matrix[2] < 0 for t in text)
- rotated = 'left' if clockwise < anticlockwise else 'right'
-
- tables_new = {}
- for k in tables.keys():
- x1, y1, x2, y2 = k
- x1 = scale(x1, scaling_factor_x)
- y1 = scale(abs(translate(-img_y, y1)), scaling_factor_y)
- x2 = scale(x2, scaling_factor_x)
- y2 = scale(abs(translate(-img_y, y2)), scaling_factor_y)
- j_x, j_y = zip(*tables[k])
- j_x = [scale(j, scaling_factor_x) for j in j_x]
- j_y = [scale(abs(translate(-img_y, j)), scaling_factor_y) for j in j_y]
- joints = zip(j_x, j_y)
- tables_new[(x1, y1, x2, y2)] = joints
-
- v_segments_new = []
- for v in v_segments:
- x1, x2 = scale(v[0], scaling_factor_x), scale(v[2], scaling_factor_x)
- y1, y2 = scale(abs(translate(-img_y, v[1])), scaling_factor_y), scale(
- abs(translate(-img_y, v[3])), scaling_factor_y)
- v_segments_new.append((x1, y1, x2, y2))
-
- h_segments_new = []
- for h in h_segments:
- x1, x2 = scale(h[0], scaling_factor_x), scale(h[2], scaling_factor_x)
- y1, y2 = scale(abs(translate(-img_y, h[1])), scaling_factor_y), scale(
- abs(translate(-img_y, h[3])), scaling_factor_y)
- h_segments_new.append((x1, y1, x2, y2))
-
- num_tables = 1
- output = {}
- # sort tables based on y-coord
- for k in sorted(tables_new.keys(), key=lambda x: x[1], reverse=True):
- # find rows and columns that lie in table
- lb = (k[0], k[1])
- rt = (k[2], k[3])
- v_s = [v for v in v_segments_new if v[1] > lb[1] - 2 and v[3]
- < rt[1] + 2 and lb[0] - 2 <= v[0] <= rt[0] + 2]
- h_s = [h for h in h_segments_new if h[0] > lb[0] - 2 and h[2]
- < rt[0] + 2 and lb[1] - 2 <= h[1] <= rt[1] + 2]
-
- if debug == "line":
- for v in v_s:
- plt.plot([v[0], v[2]], [v[1], v[3]])
- for h in h_s:
- plt.plot([h[0], h[2]], [h[1], h[3]])
-
- columns, rows = zip(*tables_new[k])
- columns, rows = list(columns), list(rows)
- columns.extend([lb[0], rt[0]])
- rows.extend([lb[1], rt[1]])
- # sort horizontal and vertical segments
- columns = merge_close_values(sorted(columns), mtol=mtol)
- rows = merge_close_values(sorted(rows, reverse=True), mtol=mtol)
- # make grid using x and y coord of shortlisted rows and columns
- columns = [(columns[i], columns[i + 1])
- for i in range(0, len(columns) - 1)]
- rows = [(rows[i], rows[i + 1]) for i in range(0, len(rows) - 1)]
-
- table = Table(columns, rows)
- # light up cell edges
- table = table.set_edges(v_s, h_s, jtol=jtol)
- # table set span method
- table = table.set_spanning()
- # light up table border
- table = outline(table)
-
- if debug == "table":
- for i in range(len(table.cells)):
- for j in range(len(table.cells[i])):
- if table.cells[i][j].left:
- plt.plot([table.cells[i][j].lb[0], table.cells[i][j].lt[0]],
- [table.cells[i][j].lb[1], table.cells[i][j].lt[1]])
- if table.cells[i][j].right:
- plt.plot([table.cells[i][j].rb[0], table.cells[i][j].rt[0]],
- [table.cells[i][j].rb[1], table.cells[i][j].rt[1]])
- if table.cells[i][j].top:
- plt.plot([table.cells[i][j].lt[0], table.cells[i][j].rt[0]],
- [table.cells[i][j].lt[1], table.cells[i][j].rt[1]])
- if table.cells[i][j].bottom:
- plt.plot([table.cells[i][j].lb[0], table.cells[i][j].rb[0]],
- [table.cells[i][j].lb[1], table.cells[i][j].rb[1]])
-
- # fill text after sorting it
- if not rotated:
- text.sort(key=lambda x: (-x.y0, x.x0))
- elif rotated == 'left':
- text.sort(key=lambda x: (x.x0, x.y0))
- elif rotated == 'right':
- text.sort(key=lambda x: (-x.x0, -x.y0))
-
- for t in text:
- r_idx = get_row_idx(t, rows)
- c_idx = get_column_idx(t, columns)
- if None in [r_idx, c_idx]:
- pass
- else:
- r_idx, c_idx = reduce_index(table, rotated, r_idx, c_idx)
- table.cells[r_idx][c_idx].add_text(t.get_text().strip('\n'))
-
- if f is not None:
- table = fill(table, f=f)
-
- data = []
- for i in range(len(table.cells)):
- data.append([table.cells[i][j].get_text().strip().encode('utf-8')
- for j in range(len(table.cells[i]))])
- if rotated == 'left':
- data = zip(*data[::-1])
- elif rotated == 'right':
- data = zip(*data[::1])
- data.reverse()
- data = remove_empty(data)
- output['table_%d' % num_tables] = data
- num_tables += 1
-
- if debug in ['line', 'table']:
- plt.show()
- return None
-
- return output
\ No newline at end of file
diff --git a/pdf.py b/pdf.py
deleted file mode 100644
index b39f185..0000000
--- a/pdf.py
+++ /dev/null
@@ -1,111 +0,0 @@
-from pdfminer.pdfparser import PDFParser
-from pdfminer.pdfdocument import PDFDocument
-from pdfminer.pdfpage import PDFPage
-from pdfminer.pdfpage import PDFTextExtractionNotAllowed
-from pdfminer.pdfinterp import PDFResourceManager
-from pdfminer.pdfinterp import PDFPageInterpreter
-from pdfminer.pdfdevice import PDFDevice
-from pdfminer.converter import PDFPageAggregator
-from pdfminer.layout import LAParams, LTChar, LTTextLineHorizontal
-
-
-def parse_text_stream(layout, t=None):
- """Recursively parse pdf layout to get a list of
- LTTextHorizontal objects.
-
- Parameters
- ----------
- layout : object
-
- t : list
-
- Returns
- -------
- t : list
- """
- if t is None:
- t = []
- try:
- for obj in layout._objs:
- if isinstance(obj, LTTextLineHorizontal):
- t.append(obj)
- else:
- t += parse_text_stream(obj)
- except AttributeError:
- pass
- return t
-
-
-def parse_text_lattice(layout, t=None):
- """Recursively parse pdf layout to get a list of
- LTChar objects.
-
- Parameters
- ----------
- layout : object
-
- t : list
-
- Returns
- -------
- t : list
- """
- if t is None:
- t = []
- try:
- for obj in layout._objs:
- if isinstance(obj, LTChar):
- t.append(obj)
- else:
- t += parse_text_lattice(obj)
- except AttributeError:
- pass
- return t
-
-
-def get_pdf_info(pdfname, method=None, char_margin=2.0, line_margin=0.5,
- word_margin=0.1):
- """Get list of text objects along with pdf width and height.
-
- Parameters
- ----------
- pdfname : string
-
- method : string
-
- char_margin : float
-
- line_margin : float
-
- word_margin : float
-
- Returns
- -------
- text : list
-
- pdf_x : int
-
- pdf_y : int
- """
- if not method:
- return None
- with open(pdfname, 'r') as f:
- parser = PDFParser(f)
- document = PDFDocument(parser)
- if not document.is_extractable:
- raise PDFTextExtractionNotAllowed
- laparams = LAParams(char_margin=char_margin,
- line_margin=line_margin,
- word_margin=word_margin)
- rsrcmgr = PDFResourceManager()
- device = PDFPageAggregator(rsrcmgr, laparams=laparams)
- interpreter = PDFPageInterpreter(rsrcmgr, device)
- for page in PDFPage.create_pages(document):
- interpreter.process_page(page)
- layout = device.get_result()
- if method == 'stream':
- text = parse_text_stream(layout)
- elif method == 'lattice':
- text = parse_text_lattice(layout)
- pdf_x, pdf_y = layout.bbox[2], layout.bbox[3]
- return text, pdf_x, pdf_y
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..5744ce0
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,8 @@
+docopt
+matplotlib
+nose
+pdfminer
+pyexcel-xlsx
+PyPDF2
+Sphinx
+Wand
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..1d1d4ec
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,80 @@
+from pkg_resources import parse_version
+
+import camelot
+
+NAME = 'camelot'
+VERSION = camelot.__version__
+DESCRIPTION = 'camelot parses tables from PDFs!'
+with open('README.md') as f:
+ LONG_DESCRIPTION = f.read()
+URL = 'https://github.com/socialcopsdev/camelot'
+AUTHOR = 'Vinayak Mehta'
+AUTHOR_EMAIL = 'vinayak@socialcops.com'
+LICENSE = 'BSD License'
+
+opencv_min_version = '2.4.8'
+
+
+def get_opencv_status():
+ """
+ Returns a dictionary containing a boolean specifying whether OpenCV
+ is up-to-date, along with the version string (empty string if
+ not installed).
+ """
+ opencv_status = {}
+ try:
+ import cv2
+ opencv_version = cv2.__version__
+ opencv_status['up_to_date'] = parse_version(
+ opencv_version) >= parse_version(opencv_min_version)
+ opencv_status['version'] = opencv_version
+ except ImportError:
+ opencv_status['up_to_date'] = False
+ opencv_status['version'] = ""
+ return opencv_status
+
+
+def setup_package():
+ reqs = []
+ with open('requirements.txt', 'r') as f:
+ for line in f:
+ reqs.append(line.strip())
+
+ metadata = dict(name=NAME,
+ version=VERSION,
+ description=DESCRIPTION,
+ long_description=LONG_DESCRIPTION,
+ url=URL,
+ author=AUTHOR,
+ author_email=AUTHOR_EMAIL,
+ license=LICENSE,
+ keywords='parse scrape pdf table',
+ packages=['camelot'],
+ install_requires=reqs,
+ scripts=['tools/camelot'])
+
+ try:
+ from setuptools import setup
+ except:
+ from distutils.core import setup
+
+ opencv_status = get_opencv_status()
+ opencv_req_str = "camelot requires OpenCV >= {0}.\n".format(opencv_min_version)
+ instructions = ("Installation instructions are available in the README at "
+ "https://github.com/socialcopsdev/camelot")
+
+ if opencv_status['up_to_date'] is False:
+ if opencv_status['version']:
+ raise ImportError("Your installation of OpenCV "
+ "{0} is out-of-date.\n{1}{2}"
+ .format(opencv_status['version'],
+ opencv_req_str, instructions))
+ else:
+ raise ImportError("OpenCV is not installed.\n{0}{1}"
+ .format(opencv_req_str, instructions))
+
+ setup(**metadata)
+
+
+if __name__ == '__main__':
+ setup_package()
diff --git a/stream.py b/stream.py
deleted file mode 100644
index 16f0429..0000000
--- a/stream.py
+++ /dev/null
@@ -1,143 +0,0 @@
-import os
-import numpy as np
-
-from pdf import get_pdf_info
-
-
-def overlap(l):
- """Groups overlapping columns and returns list with updated
- columns boundaries.
-
- Parameters
- ----------
- l : list
- List of column x-coordinates.
-
- Returns
- -------
- merged : list
- List of merged column x-coordinates.
- """
- merged = []
- for higher in l:
- if not merged:
- merged.append(higher)
- else:
- lower = merged[-1]
- if higher[0] <= lower[1]:
- upper_bound = max(lower[1], higher[1])
- lower_bound = min(lower[0], higher[0])
- merged[-1] = (lower_bound, upper_bound)
- else:
- merged.append(higher)
- return merged
-
-
-def stream(filepath, ncolumns=0, columns=None, char_margin=2.0,
- line_margin=0.5, word_margin=0.1, debug=False):
- """Stream algorithm
-
- Groups data returned by PDFMiner into rows and finds mode of the
- number of elements in each row to guess number of columns.
-
- Parameters
- ----------
- filepath : string
-
- ncolumns : int, default: 0, optional
- Number of columns.
-
- columns : string, default: None, optional
- Comma-separated list of column x-coordinates.
-
- char_margin : float, default: 2.0, optional
- Char margin. Chars closer than cmargin are grouped together
- to form a word.
-
- line_margin : float, default: 0.5, optional
- Line margin. Lines closer than lmargin are grouped together
- to form a textbox.
-
- word_margin : float, default: 0.1, optional
- Word margin. Insert blank spaces between chars if distance
- between words is greater than word margin.
-
- debug : bool, default: False, optional
- Debug by visualizing textboxes.
-
- Returns
- -------
- output : list
- """
- filename = os.path.basename(filepath)
- text, __, __ = get_pdf_info(filepath, method='stream', char_margin=char_margin,
- line_margin=line_margin, word_margin=word_margin)
- text.sort(key=lambda x: (-x.y0, x.x0))
- y_last = 0
- data = []
- temp = []
- elements = []
- for t in text:
- # is checking for upright necessary?
- # if t.get_text().strip() and all([obj.upright for obj in t._objs if
- # type(obj) is LTChar]):
- if t.get_text().strip():
- if not np.isclose(y_last, t.y0, atol=2):
- y_last = t.y0
- elements.append(len(temp))
- data.append(temp)
- temp = []
- temp.append(t)
-
- if debug:
- import matplotlib.pyplot as plt
- import matplotlib.patches as patches
-
- fig = plt.figure()
- ax = fig.add_subplot(111, aspect='equal')
- xs, ys = [], []
- for d in data:
- for t in d:
- xs.extend([t.x0, t.x1])
- ys.extend([t.y0, t.y1])
- ax.add_patch(
- patches.Rectangle(
- (t.x0, t.y0),
- t.x1 - t.x0,
- t.y1 - t.y0
- )
- )
- ax.set_xlim(min(xs) - 10, max(xs) + 10)
- ax.set_ylim(min(ys) - 10, max(ys) + 10)
- plt.show()
- return None
-
- if columns:
- columns = columns.split(',')
- cols = [(float(columns[i]), float(columns[i + 1]))
- for i in range(0, len(columns) - 1)]
- cols = [(c[0] + c[1]) / 2.0 for c in cols]
- else:
- # a table can't have just 1 column, can it?
- elements = filter(lambda x: x != 1, elements)
- mode = ncolumns if ncolumns else max(set(elements), key=elements.count)
- cols = [(t.x0, t.x1) for d in data for t in d if len(d) == mode]
- cols = overlap(sorted(cols))
- cols = [(c[0] + c[1]) / 2.0 for c in cols]
-
- output = [['' for c in cols] for d in data]
- for row, d in enumerate(data):
- for t in d:
- cog = (t.x0 + t.x1) / 2.0
- diff = [(i, abs(cog - c)) for i, c in enumerate(cols)]
- if diff:
- idx = min(diff, key=lambda x: x[1])
- else:
- print "couldn't find a table on this page"
- return None
- if output[row][idx[0]]:
- output[row][idx[0]] += ' ' + t.get_text().strip()
- else:
- output[row][idx[0]] = t.get_text().strip()
-
- return output
\ No newline at end of file
diff --git a/tests/assam.pdf b/tests/assam.pdf
new file mode 100644
index 0000000..92c08af
--- /dev/null
+++ b/tests/assam.pdf
@@ -0,0 +1,4881 @@
+%PDF-1.3
+%âãÏÓ
+5 0 obj [278 333 474 556 556 889 722 238 333 333 389 584 278 333 278 278 556 556 556 556 556 556 556 556 556 556 333 333 584 584 584 611 975 722 722 722 722 667 611 778 722 278 556 722 611 833 722 778 667 778 722 667 611 722 667 944 667 667 611 333 278 333 584 556 333 556 611 556 611 556 333 611 611 278 278 556 278 889 611 611 611 611 389 556 333 611 556 778 556 556 500 389 280 389 584 0 556 0 278 556 500 1000 556 556 333 1000 667 333 1000 0 611 0 0 278 278 500 500 350 556 1000 333 1000 556 333 944 0 500 667 278 333 556 556 556 556 280 556 333 737 370 556 584 333 737 552 400 549 333 333 333 576 556 278 333 333 365 556 834 834 834 611 722 722 722 722 722 722 1000 722 667 667 667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 722 722 722 667 667 611 556 556 556 556 556 556 889 556 556 556 556 556 278 278 278 278 611 611 611 611 611 611 611 549 611 611 611 611 611 556 611 556]
+endobj
+7 0 obj
+<<
+/Length3 532
+/Length2 32560
+/Length1 4614
+/Filter [/ASCII85Decode]
+/Length 48393
+>>
+stream
+,p>`rDKJj'E+LaU0eP.@+@1$^@;JY;/4+8kA07;N@r,Rj+E2IF%:jU#+>Yhs06),QFCeul
+C`m\>F:Dc'E$-qo+>7VTD.7'sF(o9)+E2IF%:jU#+>Ynu04]!;Cgh?,E-673A9Di61H-F0E+O'"
+ASuT4E-673A9Di61H6L1@;TspEcYZ,A0>i6F:Dc'E$-qs+>7_WDffQ3Bl7R"AKYr7F:Dc'E$-tk
++>7\BEb0-)AS-$,E-673A9Di61bg++E+*cuDK9H(BQPA9F`\('F`8H[1*A5YF*(u6Ble&4E-673
+A9Di61c$7-E,9Z<+E2IF%:jU#+>bnt04Sp;D.+Q+F`\('F`8H[2'=PcH#RG.DBNn@F:Dc'E$-tq
++>7\FEbTH&+E2IF%:jU#+>c#"06:r7>FF`JUDF`\('F`8H\
+1E\>_Bm+&1E-673A9Di62)HF/F(okAE-673A9Di62)QL0F(KN+DBNn@F:Dc'E$."r+>7;?B4uB0
+E-673A9Di62)cX2DJ=!"+E2IF%:jU#+>l,$04Sp:Df-\:F`\('F`8H\3?TtrASl-kDes?0+E2IF
+%:jU#+>tnr05P3@F!,FEF:Dc'E$.%n+>7;GF^])/E-673A9Di62DQC.B6%QlFCfJ8E-673A9Di6
+2DZI/EHPi6FD5Z2+E2IF%:jU#+>u&!04ArFE-673A9Di62DlU15p1,hF:Dc'E$.%s+>6&(E-673
+A9Di62E)a36Qg>jF:Dc'E$.%u+>6,*E-673A9Di62E;m573HPlF:Dc'E$.(n+>62,E-673A9Di6
+2_cF.7j)bnF:Dc'E$.(p+>68.E-673A9Di62_uR08K_tpF:Dc'E$.(r+>6>0E-673A9Di62`2^2
+9-A1rF:Dc'E$.(t+>6D2E-673A9Di62`Dj49d"CtF:Dc'E$.)!+>6J4E-673A9Di62`W!6:EXV!
+F:Dc'E$.+o+>6P6E-673A9Di63&)O/;'9h#F:Dc'E$.+q+>6V8E-673A9Di63&;[1;]p%%F:Dc'
+E$.+s+>6\:E-673A9Di63&Mg36b6h>E-673A9Di63&r*7=Wh[+F:Dc'E$..p+>6n@E-673A9Di63ADX0@WGmeCLqU(AS-$,E-673
+A9Di63AM^1@UWb^F)59+BHV8:F:Dc'E$..s+>72E@:O(aFE2).BQPA9F`\('F`8H`1a"G[F(8ou
+@qfsnF_r7?F`\('F`8H`2'=PpDId=!F(9-/AKYr7F:Dc'E$./!+>7AJ@71HE-673A9Di63B838@j#`5F:Dc'E$-kh0H`#Z+E2IF%:jU#+>GQ)+>7:K
+E-673A9Di60eb@*04l>GF`\('F`8HX0Jah)B-;/9F:Dc'E$-kh1a"Gb+E2IF%:jU#+>GQ-+>7FO
+E-673A9Di60ebL.05;VKF`\('F`8HX0K1+-CERS=F:Dc'E$-kh3$9kj+E2IF%:jU#+>GQ1+>7RS
+E-673A9Di60ek@)05_nOF`\('F`8HX0eje(D]j"AF:Dc'E$-ki1*A5h+E2IF%:jU#+>GT,+>7^W
+E-673A9Di60ekL-06/1SF`\('F`8HX0f:(,F!,FEF:Dc'E$-ki2BXYp+E2IF%:jU#+>GT0+>7j[
+E-673A9Di60ekX106SIWF`\('F`8HX0f^@0G9CjIF:Dc'E$-kj0H`#n+E2IF%:jU#+>GW++>8!_
+E-673A9Di60etL,07"a[F`\('F`8HX1,C%+@WGmeASbpiFGZ++>7_WDffQ3Bl7R"@UX=h+E2IF%:jU#+>GZ,+>7>CDfTQ0+E2IF%:jU#+>GZ-
++>7_WDffQ$@VfL^F(HJ5F`\('F`8HX1G^.,ASc0sE-#`7+E2IF%:jU#+>GZ/+>786B4kdr+E2IF
+%:jU#+>GZ0+>786B4kdrA7B?tE-673A9Di60f(^104S^:@s)X#Ch7i6E-673A9Di60f(a205tKC
+FD,T8F(&rs+E2IF%:jU#+>GZ3+>6Z'@<-:(+E2IF%:jU#+>GZ4+>7AMBl%j)DJ*d%AS-$,E-673
+A9Di60f1R,02G"ZE-673A9Di60f1X.03VjoEc5hG]4+>7_W
+DffQ$@Vg(!B4uB0E-673A9Di60f1m504K'?Ch7]2E-673A9Di60f:X-04f$3@<6*)E-673A9Di6
+0f:[.04f!2@<6*)E-673A9Di60f:^/06CoEA7Zm&F`\('F`8HX2)?@.FE1f#ASkjrCERS=F:Dc'
+E$-km1a"Gm@psCrDBNn@F:Dc'E$-km2'=PbF_PZ5Bl7R"EbT0"FGc.+>6)q@<+Xf
++E2IF%:jU#+>Gc/+>7;N@r,RjA8c[0+E2IF%:jU#+>Gc0+>759DKI"?F`\('F`8HX2DZI/F*(u6
+Ch[d"+E2IF%:jU#+>Gc2+>75IEcP`/@sK29F`\('F`8HX2DlU1H"Ck9E-673A9Di60fCp404Js?
+CLqBm@<*K0F`\('F`8HX2E)a3F(Jj'Bl@l3E-673A9Di60fD!604\d.Eb0<+F!,FEF:Dc'E$-kn
+3?TtbDfBuBBkM+$+E2IF%:jU#+>Gf/+>7YRA7fCiBl7X&AKYr7F:Dc'E$-ko0d&,^F_PZ.ASl@-
+Ch74#+E2IF%:jU#+>Gf1+>7PLB5(jaCi405+E2IF%:jU#+>Gf2+>7eJFD,r=BOu'(E-673A9Di6
+0fLp3061W:BleB-Eb/c(E-673A9Di60fLs405Y--Ec5hW/F`S[EF`\('F`8HX2`Mp5FE_YEF`:o4BlA#7E-673A9Di60fM*806ClJ
+AS#q*E+O'(DfQt>F`\('F`8HX3%uI.@:OG%AKYr7F:Dc'E$-kp0d&,dFWbXGF:Dc'E$-kp1*A5h
+@<,dhEa`p#+E2IF%:jU#+>Gi3+>7\FEbTH&@qBP"ATD?m+E2IF%:jU#+>Gi4+>759A8,jr@3BN3
+F:Dc'E$-kp2'=PjDImm3E+O'(DfQt>F`\('F`8HX3&Vm4DfTB*@<5q$Ch[cu+E2IF%:jU#+>Gi7
++>7AMBl%TsD/XN:BkM+$+E2IF%:jU#+>Gi8+>7YNAT;j$Ecbl5+E2IF%:jU#+>Gi9+>7YNAS>Lg
+Afu&8F:Dc'E$-kq0H`#jBQ@ZqEHP]1FCfK6+E2IF%:jU#+>Gl2+>7_WATMs-Df0*(GA\OBF`\('
+F`8HX3AM^16"b4HG%De8F`\('F`8HX3AVd26"+8GFCcS6F`\('F`8HX3A_j36"=VL@s)X#Ch7i6
+E-673A9Di60f_*600s\oCh.*"E-673A9Di60f_-700s,_ATD@'BlbD9F`\('F`8HX3B&'66#p[W
+B-;/9F:Dc'E$-kq3$9k?73HPlF:Dc'E$-kq3?TtB@qB1bCi!N&E-673A9Di61,(C)01BMo@PW,+>6/o
+Bk;PW0+>6;sBk;PW2+>6K3Bl%PZ*+>6N'Eaa-&+E2IF%:jU#+>PZ++>6N!@s)m)+E2IF%:jU#
++>PZ,+>6N#Bl[^+D.[TqGT^sJF:Dc'E$-nj1E\>HFD5Q%AKYr7F:Dc'E$-nj1a"GIA8,UqATMR,
++E2IF%:jU#+>PZ/+>7SSCijB1Cj@.BF`\('F`8HY0fC.-:NL>\F(c\8F`\('F`8HY0fL4.\
+G%De8F`\('F`8HY0fU:/`ATDWrDId3mF!,FEF:Dc'E$-nk1a"G[B6%F(AKYr7F:Dc'E$-nk2'=P\@:OG%AKYr7F:Dc'
+E$-nk2BXY]@qfsnF_t`,AU#>:F`\('F`8HY1,g=/@P`,+>7/7+E2IF%:jU#+>P`-+>757ARoLkCgeH*
+F`\('F`8HY1GU(+AS6$lG%De8F`\('F`8HY1G^.,ART(kFCcS6F`\('F`8HY1Gg4-ARfFp@s)X#
+Ch7i6E-673A9Di61,Cd104e[.ATD@'BlbD9F`\('F`8HY1H$@/BkMHpG%De8F`\('F`8HY1H-F0
+BjkLoFCcS6F`\('F`8HY1H6L1Bk(jt@s)X#Ch7i6E-673A9Di61,Cp5054s2ATD@'BlbD9F`\('
+F`8HY1b^%*ATVT.E-673A9Di61,L^.05blGCh.*"E-673A9Di61,La/05kKD@Pc2+>7YTBl%Pc3+>7YDBk;Pf0+>7kJBk;db
+0H`%l0RY'aART+fDJXS@A7]?^@s)g4ASuU#Bk)6-ASu#f@s)g4ASuU%Bl%?'AS$*t@h#.adlb17
+\51!eeMt+BCg[ri/-:S[XYBou(rYbkE?rM\r(7GFKXD/2M)N]="Z0'T_c2BcH;12qob($\/`rVY
+MHhsSL5jpB*1KGE831CCr>Y3Lf=f-43SPAM!WsN)!sDWDF`D<(ZBFK\Gj1E=\Me/^7=7!lQHWD9
+lDp'dcmKkMe>;\KS:-CaVN=[NlAK>IhPc[2/Z-r]oN\E!PhZ/;4
+"qI'd\Y,b;nUYhN(Z+AD(iC&Aem`p-ieBu"35!*$_7!6(
+:bQpZ,TVVf*(b5@"+OL,P!WkrdDt_B%=#oYkV0Vhr[o>aC*D4)HVC%@?PGOA6SVn@:NDI?!:NYO
+qQ"\]\19=\#p$3[$-B,H07nJ+)We&XcsEf2E0-'-&(hl&BTT#!MI#$7-oIMOD"k[`L-?cHbYDSB
+%SAfKa[#R?TDYs_1Qu7S9bfJ3V-aUR!q>JDkR!i,&(>j>!V>%]Ej#02C_(t+Xf9pcf:a@tqauuG
+DeVL)J;4YFE(i,">]f/=)5D=YVc]I>R)
+OQ:4D66rFTDqU&Em=b(7=5Q&nBltl:AdgUH![NDNbi8cPS(%1qhC65"rJ/eh!5eLQtp,51:9MZ"]?*+W0KtSZg*kC&PZrDgZcRuYr
+fe=U]bq#B6D(]Cb?q?qs:#sl90;_!Z8'IkN(W:6/7YfsM"'VFKL:D3LTiq2i+J'ZMT3',\glo=V
+?4`8NgO,$F%X?L@F?,AS$A`QI5[D886hHY3?.4!26!BZo6mqFDd\S*T:6*n'%Y0K_o