Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 36006cadc5 | |||
| db3f8c6897 | |||
| c0e9235164 | |||
| 79db6e3d1b | |||
| 29f22ad1a6 | |||
| e8af4c2c1c | |||
| 220d6ad29c |
@@ -4,6 +4,15 @@ Release History
|
||||
master
|
||||
------
|
||||
|
||||
0.3.1 (2018-11-02)
|
||||
------------------
|
||||
|
||||
**Improvements**
|
||||
|
||||
* Matplotlib is now an optional requirement. [#190](https://github.com/socialcopsdev/camelot/pull/190) by Vinayak Mehta.
|
||||
* You can install it using `$ pip install camelot-py[plot]`.
|
||||
* [#127](https://github.com/socialcopsdev/camelot/issues/127) Add tests for plotting. Coverage is now at 87%! [#179](https://github.com/socialcopsdev/camelot/pull/179) by [Suyash Behera](https://github.com/Suyash458).
|
||||
|
||||
0.3.0 (2018-10-28)
|
||||
------------------
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ install:
|
||||
pip install ".[dev]"
|
||||
|
||||
test:
|
||||
pytest --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot tests
|
||||
pytest --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot --mpl tests
|
||||
|
||||
docs:
|
||||
cd docs && make html
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
[](https://travis-ci.org/socialcopsdev/camelot) [](https://camelot-py.readthedocs.io/en/master/)
|
||||
[](https://codecov.io/github/socialcopsdev/camelot?branch=master)
|
||||
[](https://pypi.org/project/camelot-py/) [](https://pypi.org/project/camelot-py/) [](https://pypi.org/project/camelot-py/)
|
||||
[](https://pypi.org/project/camelot-py/) [](https://pypi.org/project/camelot-py/) [](https://pypi.org/project/camelot-py/) [](https://gitter.im/camelot-dev/Lobby)
|
||||
|
||||
**Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
|
||||
|
||||
@@ -61,18 +61,10 @@ See [comparison with other PDF table extraction libraries and tools](https://git
|
||||
|
||||
### Using conda
|
||||
|
||||
The easiest way to install Camelot is to install it with [conda](https://conda.io/docs/), which is the package manager that the [Anaconda](http://docs.continuum.io/anaconda/) distribution is built upon.
|
||||
|
||||
First, let's add the [conda-forge](https://conda-forge.org/) channel to conda's config:
|
||||
The easiest way to install Camelot is to install it with [conda](https://conda.io/docs/), which is a package manager and environment management system for the [Anaconda](http://docs.continuum.io/anaconda/) distribution.
|
||||
|
||||
<pre>
|
||||
$ conda config --add channels conda-forge
|
||||
</pre>
|
||||
|
||||
Now, you can simply use conda to install Camelot:
|
||||
|
||||
<pre>
|
||||
$ conda install -c camelot-dev camelot-py
|
||||
$ conda install -c conda-forge camelot-py
|
||||
</pre>
|
||||
|
||||
### Using pip
|
||||
@@ -80,7 +72,7 @@ $ conda install -c camelot-dev camelot-py
|
||||
After [installing the dependencies](https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip) ([tk](https://packages.ubuntu.com/trusty/python-tk) and [ghostscript](https://www.ghostscript.com/)), you can simply use pip to install Camelot:
|
||||
|
||||
<pre>
|
||||
$ pip install camelot-py[all]
|
||||
$ pip install camelot-py[cv]
|
||||
</pre>
|
||||
|
||||
### From the source code
|
||||
@@ -95,7 +87,7 @@ and install Camelot using pip:
|
||||
|
||||
<pre>
|
||||
$ cd camelot
|
||||
$ pip install ".[all]"
|
||||
$ pip install ".[cv]"
|
||||
</pre>
|
||||
|
||||
## Documentation
|
||||
|
||||
@@ -6,6 +6,7 @@ from click import HelpFormatter
|
||||
|
||||
from .__version__ import __version__
|
||||
from .io import read_pdf
|
||||
from .plotting import PlotMethods
|
||||
|
||||
|
||||
def _write_usage(self, prog, args='', prefix='Usage: '):
|
||||
@@ -25,3 +26,6 @@ handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(handler)
|
||||
|
||||
# instantiate plot method
|
||||
plot = PlotMethods()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
VERSION = (0, 3, 0)
|
||||
VERSION = (0, 3, 1)
|
||||
|
||||
__title__ = 'camelot-py'
|
||||
__description__ = 'PDF Table Extraction for Humans.'
|
||||
|
||||
@@ -3,9 +3,14 @@
|
||||
import logging
|
||||
|
||||
import click
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
_HAS_MPL = False
|
||||
else:
|
||||
_HAS_MPL = True
|
||||
|
||||
from . import __version__
|
||||
from .io import read_pdf
|
||||
from . import __version__, read_pdf, plot
|
||||
|
||||
|
||||
logger = logging.getLogger('camelot')
|
||||
@@ -80,8 +85,8 @@ def cli(ctx, *args, **kwargs):
|
||||
@click.option('-I', '--iterations', default=0,
|
||||
help='Number of times for erosion/dilation will be applied.')
|
||||
@click.option('-plot', '--plot_type',
|
||||
type=click.Choice(['text', 'table', 'contour', 'joint', 'line']),
|
||||
help='Plot geometry found on PDF page, for debugging.')
|
||||
type=click.Choice(['text', 'grid', 'contour', 'joint', 'line']),
|
||||
help='Plot elements found on PDF page for visual debugging.')
|
||||
@click.argument('filepath', type=click.Path(exists=True))
|
||||
@pass_config
|
||||
def lattice(c, *args, **kwargs):
|
||||
@@ -102,17 +107,23 @@ def lattice(c, *args, **kwargs):
|
||||
kwargs['copy_text'] = None if not copy_text else copy_text
|
||||
kwargs['shift_text'] = list(kwargs['shift_text'])
|
||||
|
||||
tables = read_pdf(filepath, pages=pages, flavor='lattice',
|
||||
suppress_warnings=suppress_warnings, **kwargs)
|
||||
click.echo('Found {} tables'.format(tables.n))
|
||||
if plot_type is not None:
|
||||
for table in tables:
|
||||
table.plot(plot_type)
|
||||
if not _HAS_MPL:
|
||||
raise ImportError('matplotlib is required for plotting.')
|
||||
else:
|
||||
if output is None:
|
||||
raise click.UsageError('Please specify output file path using --output')
|
||||
if f is None:
|
||||
raise click.UsageError('Please specify output file format using --format')
|
||||
|
||||
tables = read_pdf(filepath, pages=pages, flavor='lattice',
|
||||
suppress_warnings=suppress_warnings, **kwargs)
|
||||
click.echo('Found {} tables'.format(tables.n))
|
||||
if plot_type is not None:
|
||||
for table in tables:
|
||||
plot(table, kind=plot_type)
|
||||
plt.show()
|
||||
else:
|
||||
tables.export(output, f=f, compress=compress)
|
||||
|
||||
|
||||
@@ -127,8 +138,8 @@ def lattice(c, *args, **kwargs):
|
||||
@click.option('-c', '--col_close_tol', default=0, help='Tolerance parameter'
|
||||
' used to combine text horizontally, to generate columns.')
|
||||
@click.option('-plot', '--plot_type',
|
||||
type=click.Choice(['text', 'table']),
|
||||
help='Plot geometry found on PDF page for debugging.')
|
||||
type=click.Choice(['text', 'grid']),
|
||||
help='Plot elements found on PDF page for visual debugging.')
|
||||
@click.argument('filepath', type=click.Path(exists=True))
|
||||
@pass_config
|
||||
def stream(c, *args, **kwargs):
|
||||
@@ -148,15 +159,21 @@ def stream(c, *args, **kwargs):
|
||||
columns = list(kwargs['columns'])
|
||||
kwargs['columns'] = None if not columns else columns
|
||||
|
||||
tables = read_pdf(filepath, pages=pages, flavor='stream',
|
||||
suppress_warnings=suppress_warnings, **kwargs)
|
||||
click.echo('Found {} tables'.format(tables.n))
|
||||
if plot_type is not None:
|
||||
for table in tables:
|
||||
table.plot(plot_type)
|
||||
if not _HAS_MPL:
|
||||
raise ImportError('matplotlib is required for plotting.')
|
||||
else:
|
||||
if output is None:
|
||||
raise click.UsageError('Please specify output file path using --output')
|
||||
if f is None:
|
||||
raise click.UsageError('Please specify output file format using --format')
|
||||
|
||||
tables = read_pdf(filepath, pages=pages, flavor='stream',
|
||||
suppress_warnings=suppress_warnings, **kwargs)
|
||||
click.echo('Found {} tables'.format(tables.n))
|
||||
if plot_type is not None:
|
||||
for table in tables:
|
||||
plot(table, kind=plot_type)
|
||||
plt.show()
|
||||
else:
|
||||
tables.export(output, f=f, compress=compress)
|
||||
|
||||
@@ -7,8 +7,6 @@ import tempfile
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from .plotting import *
|
||||
|
||||
|
||||
class Cell(object):
|
||||
"""Defines a cell in a table with coordinates relative to a
|
||||
@@ -321,33 +319,6 @@ class Table(object):
|
||||
cell.hspan = True
|
||||
return self
|
||||
|
||||
def plot(self, geometry_type):
|
||||
"""Plot geometry found on PDF page based on geometry_type
|
||||
specified, useful for debugging and playing with different
|
||||
parameters to get the best output.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
geometry_type : str
|
||||
The geometry type for which a plot should be generated.
|
||||
Can be 'text', 'table', 'contour', 'joint', 'line'
|
||||
|
||||
"""
|
||||
if self.flavor == 'stream' and geometry_type in ['contour', 'joint', 'line']:
|
||||
raise NotImplementedError("{} cannot be plotted with flavor='stream'".format(
|
||||
geometry_type))
|
||||
|
||||
if geometry_type == 'text':
|
||||
plot_text(self._text)
|
||||
elif geometry_type == 'table':
|
||||
plot_table(self)
|
||||
elif geometry_type == 'contour':
|
||||
plot_contour(self._image)
|
||||
elif geometry_type == 'joint':
|
||||
plot_joint(self._image)
|
||||
elif geometry_type == 'line':
|
||||
plot_line(self._segments)
|
||||
|
||||
def to_csv(self, path, **kwargs):
|
||||
"""Writes Table to a comma-separated values (csv) file.
|
||||
|
||||
|
||||
@@ -141,9 +141,6 @@ class PDFHandler(object):
|
||||
-------
|
||||
tables : camelot.core.TableList
|
||||
List of tables found in PDF.
|
||||
geometry : camelot.core.GeometryList
|
||||
List of geometry objects (contours, lines, joints) found
|
||||
in PDF.
|
||||
|
||||
"""
|
||||
tables = []
|
||||
|
||||
@@ -1,108 +1,179 @@
|
||||
import cv2
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as patches
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as patches
|
||||
except ImportError:
|
||||
_HAS_MPL = False
|
||||
else:
|
||||
_HAS_MPL = True
|
||||
|
||||
|
||||
def plot_text(text):
|
||||
"""Generates a plot for all text present on the PDF page.
|
||||
class PlotMethods(object):
|
||||
def __call__(self, table, kind='text', filename=None):
|
||||
"""Plot elements found on PDF page based on kind
|
||||
specified, useful for debugging and playing with different
|
||||
parameters to get the best output.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : list
|
||||
Parameters
|
||||
----------
|
||||
table: camelot.core.Table
|
||||
A Camelot Table.
|
||||
kind : str, optional (default: 'text')
|
||||
{'text', 'grid', 'contour', 'joint', 'line'}
|
||||
The element type for which a plot should be generated.
|
||||
filepath: str, optional (default: None)
|
||||
Absolute path for saving the generated plot.
|
||||
|
||||
"""
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
xs, ys = [], []
|
||||
for t in text:
|
||||
xs.extend([t[0], t[2]])
|
||||
ys.extend([t[1], t[3]])
|
||||
ax.add_patch(
|
||||
patches.Rectangle(
|
||||
(t[0], t[1]),
|
||||
t[2] - t[0],
|
||||
t[3] - t[1]
|
||||
Returns
|
||||
-------
|
||||
fig : matplotlib.fig.Figure
|
||||
|
||||
"""
|
||||
if not _HAS_MPL:
|
||||
raise ImportError('matplotlib is required for plotting.')
|
||||
|
||||
if table.flavor == 'stream' and kind in ['contour', 'joint', 'line']:
|
||||
raise NotImplementedError("Stream flavor does not support kind='{}'".format(
|
||||
kind))
|
||||
|
||||
plot_method = getattr(self, kind)
|
||||
return plot_method(table)
|
||||
|
||||
def text(self, table):
|
||||
"""Generates a plot for all text elements present
|
||||
on the PDF page.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table : camelot.core.Table
|
||||
|
||||
Returns
|
||||
-------
|
||||
fig : matplotlib.fig.Figure
|
||||
|
||||
"""
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
xs, ys = [], []
|
||||
for t in table._text:
|
||||
xs.extend([t[0], t[2]])
|
||||
ys.extend([t[1], t[3]])
|
||||
ax.add_patch(
|
||||
patches.Rectangle(
|
||||
(t[0], t[1]),
|
||||
t[2] - t[0],
|
||||
t[3] - t[1]
|
||||
)
|
||||
)
|
||||
)
|
||||
ax.set_xlim(min(xs) - 10, max(xs) + 10)
|
||||
ax.set_ylim(min(ys) - 10, max(ys) + 10)
|
||||
plt.show()
|
||||
ax.set_xlim(min(xs) - 10, max(xs) + 10)
|
||||
ax.set_ylim(min(ys) - 10, max(ys) + 10)
|
||||
return fig
|
||||
|
||||
def grid(self, table):
|
||||
"""Generates a plot for the detected table grids
|
||||
on the PDF page.
|
||||
|
||||
def plot_table(table):
|
||||
"""Generates a plot for the table.
|
||||
Parameters
|
||||
----------
|
||||
table : camelot.core.Table
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table : camelot.core.Table
|
||||
Returns
|
||||
-------
|
||||
fig : matplotlib.fig.Figure
|
||||
|
||||
"""
|
||||
for row in table.cells:
|
||||
for cell in row:
|
||||
if cell.left:
|
||||
plt.plot([cell.lb[0], cell.lt[0]],
|
||||
[cell.lb[1], cell.lt[1]])
|
||||
if cell.right:
|
||||
plt.plot([cell.rb[0], cell.rt[0]],
|
||||
[cell.rb[1], cell.rt[1]])
|
||||
if cell.top:
|
||||
plt.plot([cell.lt[0], cell.rt[0]],
|
||||
[cell.lt[1], cell.rt[1]])
|
||||
if cell.bottom:
|
||||
plt.plot([cell.lb[0], cell.rb[0]],
|
||||
[cell.lb[1], cell.rb[1]])
|
||||
plt.show()
|
||||
"""
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
for row in table.cells:
|
||||
for cell in row:
|
||||
if cell.left:
|
||||
ax.plot([cell.lb[0], cell.lt[0]],
|
||||
[cell.lb[1], cell.lt[1]])
|
||||
if cell.right:
|
||||
ax.plot([cell.rb[0], cell.rt[0]],
|
||||
[cell.rb[1], cell.rt[1]])
|
||||
if cell.top:
|
||||
ax.plot([cell.lt[0], cell.rt[0]],
|
||||
[cell.lt[1], cell.rt[1]])
|
||||
if cell.bottom:
|
||||
ax.plot([cell.lb[0], cell.rb[0]],
|
||||
[cell.lb[1], cell.rb[1]])
|
||||
return fig
|
||||
|
||||
def contour(self, table):
|
||||
"""Generates a plot for all table boundaries present
|
||||
on the PDF page.
|
||||
|
||||
def plot_contour(image):
|
||||
"""Generates a plot for all table boundaries present on the
|
||||
PDF page.
|
||||
Parameters
|
||||
----------
|
||||
table : camelot.core.Table
|
||||
|
||||
Parameters
|
||||
----------
|
||||
image : tuple
|
||||
Returns
|
||||
-------
|
||||
fig : matplotlib.fig.Figure
|
||||
|
||||
"""
|
||||
img, table_bbox = image
|
||||
for t in table_bbox.keys():
|
||||
cv2.rectangle(img, (t[0], t[1]),
|
||||
(t[2], t[3]), (255, 0, 0), 20)
|
||||
plt.imshow(img)
|
||||
plt.show()
|
||||
"""
|
||||
img, table_bbox = table._image
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
for t in table_bbox.keys():
|
||||
ax.add_patch(
|
||||
patches.Rectangle(
|
||||
(t[0], t[1]),
|
||||
t[2] - t[0],
|
||||
t[3] - t[1],
|
||||
fill=None,
|
||||
edgecolor='red'
|
||||
)
|
||||
)
|
||||
ax.imshow(img)
|
||||
return fig
|
||||
|
||||
def joint(self, table):
|
||||
"""Generates a plot for all line intersections present
|
||||
on the PDF page.
|
||||
|
||||
def plot_joint(image):
|
||||
"""Generates a plot for all line intersections present on the
|
||||
PDF page.
|
||||
Parameters
|
||||
----------
|
||||
table : camelot.core.Table
|
||||
|
||||
Parameters
|
||||
----------
|
||||
image : tuple
|
||||
Returns
|
||||
-------
|
||||
fig : matplotlib.fig.Figure
|
||||
|
||||
"""
|
||||
img, table_bbox = image
|
||||
x_coord = []
|
||||
y_coord = []
|
||||
for k in table_bbox.keys():
|
||||
for coord in table_bbox[k]:
|
||||
x_coord.append(coord[0])
|
||||
y_coord.append(coord[1])
|
||||
plt.plot(x_coord, y_coord, 'ro')
|
||||
plt.imshow(img)
|
||||
plt.show()
|
||||
"""
|
||||
img, table_bbox = table._image
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
x_coord = []
|
||||
y_coord = []
|
||||
for k in table_bbox.keys():
|
||||
for coord in table_bbox[k]:
|
||||
x_coord.append(coord[0])
|
||||
y_coord.append(coord[1])
|
||||
ax.plot(x_coord, y_coord, 'ro')
|
||||
ax.imshow(img)
|
||||
return fig
|
||||
|
||||
def line(self, table):
|
||||
"""Generates a plot for all line segments present
|
||||
on the PDF page.
|
||||
|
||||
def plot_line(segments):
|
||||
"""Generates a plot for all line segments present on the PDF page.
|
||||
Parameters
|
||||
----------
|
||||
table : camelot.core.Table
|
||||
|
||||
Parameters
|
||||
----------
|
||||
segments : tuple
|
||||
Returns
|
||||
-------
|
||||
fig : matplotlib.fig.Figure
|
||||
|
||||
"""
|
||||
vertical, horizontal = segments
|
||||
for v in vertical:
|
||||
plt.plot([v[0], v[2]], [v[1], v[3]])
|
||||
for h in horizontal:
|
||||
plt.plot([h[0], h[2]], [h[1], h[3]])
|
||||
plt.show()
|
||||
"""
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(111, aspect='equal')
|
||||
vertical, horizontal = table._segments
|
||||
for v in vertical:
|
||||
ax.plot([v[0], v[2]], [v[1], v[3]])
|
||||
for h in horizontal:
|
||||
ax.plot([h[0], h[2]], [h[1], h[3]])
|
||||
return fig
|
||||
|
||||
|
Before Width: | Height: | Size: 20 KiB After Width: | Height: | Size: 20 KiB |
|
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 24 KiB |
|
Before Width: | Height: | Size: 8.1 KiB After Width: | Height: | Size: 8.1 KiB |
|
Before Width: | Height: | Size: 8.8 KiB After Width: | Height: | Size: 8.8 KiB |
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 64 KiB |
@@ -27,6 +27,9 @@ Release v\ |version|. (:ref:`Installation <install>`)
|
||||
.. image:: https://img.shields.io/pypi/pyversions/camelot-py.svg
|
||||
:target: https://pypi.org/project/camelot-py/
|
||||
|
||||
.. image:: https://badges.gitter.im/camelot-dev/Lobby.png
|
||||
:target: https://gitter.im/camelot-dev/Lobby
|
||||
|
||||
**Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
|
||||
|
||||
.. note:: You can also check out `Excalibur`_, which is a web interface for Camelot!
|
||||
|
||||
@@ -27,22 +27,24 @@ To process background lines, you can pass ``process_background=True``.
|
||||
.. csv-table::
|
||||
:file: ../_static/csv/background_lines.csv
|
||||
|
||||
Plot geometry
|
||||
-------------
|
||||
Visual debugging
|
||||
----------------
|
||||
|
||||
You can use a :class:`table <camelot.core.Table>` object's :meth:`plot() <camelot.core.TableList.plot>` method to plot various geometries that were detected by Camelot while processing the PDF page. This can help you select table areas, column separators and debug bad table outputs, by tweaking different configuration parameters.
|
||||
.. note:: Visual debugging using ``plot()`` requires `matplotlib <https://matplotlib.org/>`_ which is an optional dependency. You can install it using ``$ pip install camelot-py[plot]``.
|
||||
|
||||
The following geometries are available for plotting. You can pass them to the :meth:`plot() <camelot.core.TableList.plot>` method, which will then generate a `matplotlib <https://matplotlib.org/>`_ plot for the passed geometry.
|
||||
You can use the :class:`plot() <camelot.plotting.PlotMethods>` method to generate a `matplotlib <https://matplotlib.org/>`_ plot of various elements that were detected on the PDF page while processing it. This can help you select table areas, column separators and debug bad table outputs, by tweaking different configuration parameters.
|
||||
|
||||
You can specify the type of element you want to plot using the ``kind`` keyword argument. The generated plot can be saved to a file by passing a ``filename`` keyword argument. The following plot types are supported:
|
||||
|
||||
- 'text'
|
||||
- 'table'
|
||||
- 'grid'
|
||||
- 'contour'
|
||||
- 'line'
|
||||
- 'joint'
|
||||
|
||||
.. note:: The last three geometries can only be used with :ref:`Lattice <lattice>`, i.e. when ``flavor='lattice'``.
|
||||
.. note:: The last three plot types can only be used with :ref:`Lattice <lattice>`, i.e. when ``flavor='lattice'``.
|
||||
|
||||
Let's generate a plot for each geometry using this `PDF <../_static/pdf/foo.pdf>`__ as an example. First, let's get all the tables out.
|
||||
Let's generate a plot for each type using this `PDF <../_static/pdf/foo.pdf>`__ as an example. First, let's get all the tables out.
|
||||
|
||||
::
|
||||
|
||||
@@ -50,8 +52,6 @@ Let's generate a plot for each geometry using this `PDF <../_static/pdf/foo.pdf>
|
||||
>>> tables
|
||||
<TableList n=1>
|
||||
|
||||
.. _geometry_text:
|
||||
|
||||
text
|
||||
^^^^
|
||||
|
||||
@@ -59,9 +59,10 @@ Let's plot all the text present on the table's PDF page.
|
||||
|
||||
::
|
||||
|
||||
>>> tables[0].plot('text')
|
||||
>>> camelot.plot(tables[0], kind='text')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/geometry_text.png
|
||||
.. figure:: ../_static/png/plot_text.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
@@ -72,18 +73,17 @@ This, as we shall later see, is very helpful with :ref:`Stream <stream>` for not
|
||||
|
||||
.. note:: The *x-y* coordinates shown above change as you move your mouse cursor on the image, which can help you note coordinates.
|
||||
|
||||
.. _geometry_table:
|
||||
|
||||
table
|
||||
^^^^^
|
||||
|
||||
Let's plot the table (to see if it was detected correctly or not). This geometry type, along with contour, line and joint is useful for debugging and improving the extraction output, in case the table wasn't detected correctly. (More on that later.)
|
||||
Let's plot the table (to see if it was detected correctly or not). This plot type, along with contour, line and joint is useful for debugging and improving the extraction output, in case the table wasn't detected correctly. (More on that later.)
|
||||
|
||||
::
|
||||
|
||||
>>> tables[0].plot('table')
|
||||
>>> camelot.plot(tables[0], kind='table')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/geometry_table.png
|
||||
.. figure:: ../_static/png/plot_table.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
@@ -92,8 +92,6 @@ Let's plot the table (to see if it was detected correctly or not). This geometry
|
||||
|
||||
The table is perfect!
|
||||
|
||||
.. _geometry_contour:
|
||||
|
||||
contour
|
||||
^^^^^^^
|
||||
|
||||
@@ -101,17 +99,16 @@ Now, let's plot all table boundaries present on the table's PDF page.
|
||||
|
||||
::
|
||||
|
||||
>>> tables[0].plot('contour')
|
||||
>>> camelot.plot(tables[0], kind='contour')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/geometry_contour.png
|
||||
.. figure:: ../_static/png/plot_contour.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
:alt: A plot of all contours on a PDF page
|
||||
:align: left
|
||||
|
||||
.. _geometry_line:
|
||||
|
||||
line
|
||||
^^^^
|
||||
|
||||
@@ -119,17 +116,16 @@ Cool, let's plot all line segments present on the table's PDF page.
|
||||
|
||||
::
|
||||
|
||||
>>> tables[0].plot('line')
|
||||
>>> camelot.plot(tables[0], kind='line')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/geometry_line.png
|
||||
.. figure:: ../_static/png/plot_line.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
:alt: A plot of all lines on a PDF page
|
||||
:align: left
|
||||
|
||||
.. _geometry_joint:
|
||||
|
||||
joint
|
||||
^^^^^
|
||||
|
||||
@@ -137,9 +133,10 @@ Finally, let's plot all line intersections present on the table's PDF page.
|
||||
|
||||
::
|
||||
|
||||
>>> tables[0].plot('joint')
|
||||
>>> camelot.plot(tables[0], kind='joint')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/geometry_joint.png
|
||||
.. figure:: ../_static/png/plot_joint.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
@@ -149,7 +146,7 @@ Finally, let's plot all line intersections present on the table's PDF page.
|
||||
Specify table areas
|
||||
-------------------
|
||||
|
||||
Since :ref:`Stream <stream>` treats the whole page as a table, `for now`_, it's useful to specify table boundaries in cases such as `these <../_static/pdf/table_areas.pdf>`__. You can :ref:`plot the text <geometry_text>` on this page and note the top left and bottom right coordinates of the table.
|
||||
Since :ref:`Stream <stream>` treats the whole page as a table, `for now`_, it's useful to specify table boundaries in cases such as `these <../_static/pdf/table_areas.pdf>`__. You can plot the text on this page and note the top left and bottom right coordinates of the table.
|
||||
|
||||
Table areas that you want Camelot to analyze can be passed as a list of comma-separated strings to :meth:`read_pdf() <camelot.read_pdf>`, using the ``table_areas`` keyword argument.
|
||||
|
||||
@@ -166,7 +163,7 @@ Table areas that you want Camelot to analyze can be passed as a list of comma-se
|
||||
Specify column separators
|
||||
-------------------------
|
||||
|
||||
In cases like `these <../_static/pdf/column_separators.pdf>`__, where the text is very close to each other, it is possible that Camelot may guess the column separators' coordinates incorrectly. To correct this, you can explicitly specify the *x* coordinate for each column separator by :ref:`plotting the text <geometry_text>` on the page.
|
||||
In cases like `these <../_static/pdf/column_separators.pdf>`__, where the text is very close to each other, it is possible that Camelot may guess the column separators' coordinates incorrectly. To correct this, you can explicitly specify the *x* coordinate for each column separator by plotting the text on the page.
|
||||
|
||||
You can pass the column separators as a list of comma-separated strings to :meth:`read_pdf() <camelot.read_pdf>`, using the ``columns`` keyword argument.
|
||||
|
||||
@@ -174,7 +171,7 @@ In case you passed a single column separators string list, and no table area is
|
||||
|
||||
For example, if you have specified two table areas, ``table_areas=['12,23,43,54', '20,33,55,67']``, and only want to specify column separators for the first table, you can pass an empty string for the second table in the column separators' list like this, ``columns=['10,120,200,400', '']``.
|
||||
|
||||
Let's get back to the *x* coordinates we got from :ref:`plotting text <geometry_text>` that exists on this `PDF <../_static/pdf/column_separators.pdf>`__, and get the table out!
|
||||
Let's get back to the *x* coordinates we got from plotting the text that exists on this `PDF <../_static/pdf/column_separators.pdf>`__, and get the table out!
|
||||
|
||||
::
|
||||
|
||||
@@ -282,23 +279,25 @@ Here's a `PDF <../_static/pdf/short_lines.pdf>`__ where small lines separating t
|
||||
:alt: A PDF table with short lines
|
||||
:align: left
|
||||
|
||||
Let's :ref:`plot the table <geometry_table>` for this PDF.
|
||||
Let's plot the table for this PDF.
|
||||
|
||||
::
|
||||
|
||||
>>> tables = camelot.read_pdf('short_lines.pdf')
|
||||
>>> tables[0].plot('table')
|
||||
>>> camelot.plot(tables[0], kind='table')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/short_lines_1.png
|
||||
:alt: A plot of the PDF table with short lines
|
||||
:align: left
|
||||
|
||||
Clearly, the smaller lines separating the headers, couldn't be detected. Let's try with ``line_size_scaling=40``, and `plot the table <geometry_table>`_ again.
|
||||
Clearly, the smaller lines separating the headers, couldn't be detected. Let's try with ``line_size_scaling=40``, and plot the table again.
|
||||
|
||||
::
|
||||
|
||||
>>> tables = camelot.read_pdf('short_lines.pdf', line_size_scaling=40)
|
||||
>>> tables[0].plot('table')
|
||||
>>> camelot.plot(tables[0], kind='table')
|
||||
>>> plt.show()
|
||||
|
||||
.. figure:: ../_static/png/short_lines_2.png
|
||||
:alt: An improved plot of the PDF table with short lines
|
||||
|
||||
@@ -9,28 +9,28 @@ You can print the help for the interface by typing ``camelot --help`` in your fa
|
||||
|
||||
::
|
||||
|
||||
Usage: camelot [OPTIONS] COMMAND [ARGS]...
|
||||
Usage: camelot [OPTIONS] COMMAND [ARGS]...
|
||||
|
||||
Camelot: PDF Table Extraction for Humans
|
||||
Camelot: PDF Table Extraction for Humans
|
||||
|
||||
Options:
|
||||
--version Show the version and exit.
|
||||
-p, --pages TEXT Comma-separated page numbers. Example: 1,3,4
|
||||
or 1,4-end.
|
||||
-pw, --password TEXT Password for decryption.
|
||||
-o, --output TEXT Output file path.
|
||||
-f, --format [csv|json|excel|html]
|
||||
Output file format.
|
||||
-z, --zip Create ZIP archive.
|
||||
-split, --split_text Split text that spans across multiple cells.
|
||||
-flag, --flag_size Flag text based on font size. Useful to
|
||||
detect super/subscripts.
|
||||
-M, --margins <FLOAT FLOAT FLOAT>...
|
||||
PDFMiner char_margin, line_margin and
|
||||
word_margin.
|
||||
-q, --quiet Suppress warnings.
|
||||
--help Show this message and exit.
|
||||
Options:
|
||||
--version Show the version and exit.
|
||||
-p, --pages TEXT Comma-separated page numbers. Example: 1,3,4
|
||||
or 1,4-end.
|
||||
-pw, --password TEXT Password for decryption.
|
||||
-o, --output TEXT Output file path.
|
||||
-f, --format [csv|json|excel|html]
|
||||
Output file format.
|
||||
-z, --zip Create ZIP archive.
|
||||
-split, --split_text Split text that spans across multiple cells.
|
||||
-flag, --flag_size Flag text based on font size. Useful to
|
||||
detect super/subscripts.
|
||||
-M, --margins <FLOAT FLOAT FLOAT>...
|
||||
PDFMiner char_margin, line_margin and
|
||||
word_margin.
|
||||
-q, --quiet Suppress warnings.
|
||||
--help Show this message and exit.
|
||||
|
||||
Commands:
|
||||
lattice Use lines between text to parse the table.
|
||||
stream Use spaces between text to parse the table.
|
||||
Commands:
|
||||
lattice Use lines between text to parse the table.
|
||||
stream Use spaces between text to parse the table.
|
||||
|
||||
@@ -39,7 +39,7 @@ Let's see how Lattice processes the second page of `this PDF`_, step-by-step.
|
||||
|
||||
1. Line segments are detected.
|
||||
|
||||
.. image:: ../_static/png/geometry_line.png
|
||||
.. image:: ../_static/png/plot_line.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
@@ -49,7 +49,7 @@ Let's see how Lattice processes the second page of `this PDF`_, step-by-step.
|
||||
|
||||
.. _and: https://en.wikipedia.org/wiki/Logical_conjunction
|
||||
|
||||
.. image:: ../_static/png/geometry_joint.png
|
||||
.. image:: ../_static/png/plot_joint.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
@@ -59,7 +59,7 @@ Let's see how Lattice processes the second page of `this PDF`_, step-by-step.
|
||||
|
||||
.. _or: https://en.wikipedia.org/wiki/Logical_disjunction
|
||||
|
||||
.. image:: ../_static/png/geometry_contour.png
|
||||
.. image:: ../_static/png/plot_contour.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
@@ -75,7 +75,7 @@ Let's see how Lattice processes the second page of `this PDF`_, step-by-step.
|
||||
|
||||
5. Spanning cells are detected using the line segments and line intersections.
|
||||
|
||||
.. image:: ../_static/png/geometry_table.png
|
||||
.. image:: ../_static/png/plot_table.png
|
||||
:height: 674
|
||||
:width: 1366
|
||||
:scale: 50%
|
||||
|
||||
@@ -8,15 +8,10 @@ This part of the documentation covers how to install Camelot.
|
||||
Using conda
|
||||
-----------
|
||||
|
||||
The easiest way to install Camelot is to install it with `conda`_, which is the package manager that the `Anaconda`_ distribution is built upon.
|
||||
The easiest way to install Camelot is to install it with `conda`_, which is a package manager and environment management system for the `Anaconda`_ distribution.
|
||||
::
|
||||
|
||||
First, let's add the `conda-forge`_ channel to conda's config::
|
||||
|
||||
$ conda config --add channels conda-forge
|
||||
|
||||
Now, you can simply use conda to install Camelot::
|
||||
|
||||
$ conda install -c camelot-dev camelot-py
|
||||
$ conda install -c conda-forge camelot-py
|
||||
|
||||
.. note:: Camelot is available for Python 2.7, 3.5 and 3.6 on Linux, macOS and Windows. For Windows, you will need to install ghostscript which you can get from their `downloads page`_.
|
||||
|
||||
@@ -100,7 +95,7 @@ If you have ghostscript, you should see the ghostscript version and copyright in
|
||||
|
||||
Finally, you can use pip to install Camelot::
|
||||
|
||||
$ pip install camelot-py[all]
|
||||
$ pip install camelot-py[cv]
|
||||
|
||||
From the source code
|
||||
--------------------
|
||||
@@ -116,6 +111,6 @@ After `installing the dependencies`_, you can install from the source by:
|
||||
::
|
||||
|
||||
$ cd camelot
|
||||
$ pip install ".[all]"
|
||||
$ pip install ".[cv]"
|
||||
|
||||
.. _installing the dependencies: https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip
|
||||
@@ -2,5 +2,5 @@
|
||||
test=pytest
|
||||
|
||||
[tool:pytest]
|
||||
addopts = --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot tests
|
||||
addopts = --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot --mpl tests
|
||||
python_files = tests/test_*.py
|
||||
@@ -15,7 +15,6 @@ with open('README.md', 'r') as f:
|
||||
|
||||
requires = [
|
||||
'click>=6.7',
|
||||
'matplotlib>=2.2.3',
|
||||
'numpy>=1.13.3',
|
||||
'openpyxl>=2.5.8',
|
||||
'pandas>=0.23.4',
|
||||
@@ -23,17 +22,24 @@ requires = [
|
||||
'PyPDF2>=1.26.0'
|
||||
]
|
||||
|
||||
all_requires = [
|
||||
cv_requires = [
|
||||
'opencv-python>=3.4.2.17'
|
||||
]
|
||||
|
||||
plot_requires = [
|
||||
'matplotlib>=2.2.3',
|
||||
]
|
||||
|
||||
dev_requires = [
|
||||
'codecov>=2.0.15',
|
||||
'pytest>=3.8.0',
|
||||
'pytest-cov>=2.6.0',
|
||||
'pytest-mpl>=0.10',
|
||||
'pytest-runner>=4.2',
|
||||
'Sphinx>=1.7.9'
|
||||
]
|
||||
|
||||
all_requires = cv_requires + plot_requires
|
||||
dev_requires = dev_requires + all_requires
|
||||
|
||||
|
||||
@@ -51,7 +57,9 @@ def setup_package():
|
||||
install_requires=requires,
|
||||
extras_require={
|
||||
'all': all_requires,
|
||||
'dev': dev_requires
|
||||
'cv': cv_requires,
|
||||
'dev': dev_requires,
|
||||
'plot': plot_requires
|
||||
},
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
import matplotlib
|
||||
matplotlib.use('agg')
|
||||
|
||||
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 8.2 KiB |
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 6.6 KiB |
|
After Width: | Height: | Size: 8.8 KiB |
@@ -0,0 +1,51 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import camelot
|
||||
|
||||
|
||||
testdir = os.path.dirname(os.path.abspath(__file__))
|
||||
testdir = os.path.join(testdir, "files")
|
||||
|
||||
|
||||
@pytest.mark.mpl_image_compare(
|
||||
baseline_dir="files/baseline_plots", remove_text=True)
|
||||
def test_text_plot():
|
||||
filename = os.path.join(testdir, "foo.pdf")
|
||||
tables = camelot.read_pdf(filename)
|
||||
return camelot.plot(tables[0], kind='text')
|
||||
|
||||
|
||||
@pytest.mark.mpl_image_compare(
|
||||
baseline_dir="files/baseline_plots", remove_text=True)
|
||||
def test_grid_plot():
|
||||
filename = os.path.join(testdir, "foo.pdf")
|
||||
tables = camelot.read_pdf(filename)
|
||||
return camelot.plot(tables[0], kind='grid')
|
||||
|
||||
|
||||
@pytest.mark.mpl_image_compare(
|
||||
baseline_dir="files/baseline_plots", remove_text=True)
|
||||
def test_contour_plot():
|
||||
filename = os.path.join(testdir, "foo.pdf")
|
||||
tables = camelot.read_pdf(filename)
|
||||
return camelot.plot(tables[0], kind='contour')
|
||||
|
||||
|
||||
@pytest.mark.mpl_image_compare(
|
||||
baseline_dir="files/baseline_plots", remove_text=True)
|
||||
def test_line_plot():
|
||||
filename = os.path.join(testdir, "foo.pdf")
|
||||
tables = camelot.read_pdf(filename)
|
||||
return camelot.plot(tables[0], kind='line')
|
||||
|
||||
|
||||
@pytest.mark.mpl_image_compare(
|
||||
baseline_dir="files/baseline_plots", remove_text=True)
|
||||
def test_joint_plot():
|
||||
filename = os.path.join(testdir, "foo.pdf")
|
||||
tables = camelot.read_pdf(filename)
|
||||
return camelot.plot(tables[0], kind='joint')
|
||||