Compare commits
35 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f73062c1c4 | |||
| 9cffe0adbe | |||
| 429640feea | |||
| 4366313484 | |||
| 2830ed9418 | |||
| 39cf65ffef | |||
| 32df09ad1c | |||
| 8205e0e9ab | |||
| a78ef7f841 | |||
| f734af3a0b | |||
| 61963aabb6 | |||
| 60c1270745 | |||
| 72481bc1b5 | |||
| c5c85a2dc8 | |||
| 9c6ec49652 | |||
| 2a60d1fd54 | |||
| 2022a8abc9 | |||
| 3def4a5aea | |||
| 1d064adc3e | |||
| 7baea06bca | |||
| 5645ef5b62 | |||
| 7a3b76cb76 | |||
| 970f906435 | |||
| 297888b18c | |||
| 9362175a82 | |||
| 9e6474e5a6 | |||
| 1ba0cfc7bc | |||
| 8d38907832 | |||
| ac2d40aa44 | |||
| c33bf9c168 | |||
| d628e9b5df | |||
| 750f955f9c | |||
| 898646b73b | |||
| 45e7f7570e | |||
| 296be21d9d |
@@ -0,0 +1,10 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
|
||||
[*.py]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
+29
-17
@@ -1,20 +1,32 @@
|
||||
sudo: false
|
||||
sudo: true
|
||||
language: python
|
||||
cache: pip
|
||||
python:
|
||||
- "2.7"
|
||||
- "3.5"
|
||||
- "3.6"
|
||||
matrix:
|
||||
include:
|
||||
- python: 3.7
|
||||
dist: xenial
|
||||
sudo: true
|
||||
before_install:
|
||||
- sudo apt-get install python-tk python3-tk ghostscript
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
install:
|
||||
- pip install ".[dev]"
|
||||
script:
|
||||
- pytest --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot tests
|
||||
after_success:
|
||||
- codecov --verbose
|
||||
- make install
|
||||
jobs:
|
||||
include:
|
||||
- stage: test
|
||||
script:
|
||||
- make test
|
||||
python: '2.7'
|
||||
- stage: test
|
||||
script:
|
||||
- make test
|
||||
python: '3.5'
|
||||
- stage: test
|
||||
script:
|
||||
- make test
|
||||
python: '3.6'
|
||||
- stage: test
|
||||
script:
|
||||
- make test
|
||||
python: '3.7'
|
||||
dist: xenial
|
||||
- stage: coverage
|
||||
python: '3.6'
|
||||
script:
|
||||
- make test
|
||||
- codecov --verbose
|
||||
|
||||
+39
-2
@@ -1,8 +1,45 @@
|
||||
Release History
|
||||
===============
|
||||
|
||||
master
|
||||
------
|
||||
|
||||
0.3.0 (2018-10-28)
|
||||
------------------
|
||||
|
||||
**Improvements**
|
||||
|
||||
* [#162](https://github.com/socialcopsdev/camelot/issues/162) Add password keyword argument. [#180](https://github.com/socialcopsdev/camelot/pull/180) by [rbares](https://github.com/rbares).
|
||||
* An encrypted PDF can now be decrypted by passing `password='<PASSWORD>'` to `read_pdf` or `--password <PASSWORD>` to the command-line interface. (Limited encryption algorithm support from PyPDF2.)
|
||||
* [#139](https://github.com/socialcopsdev/camelot/issues/139) Add suppress_warnings keyword argument. [#155](https://github.com/socialcopsdev/camelot/pull/155) by [Jonathan Lloyd](https://github.com/jonathanlloyd).
|
||||
* Warnings raised by Camelot can now be suppressed by passing `suppress_warnings=True` to `read_pdf` or `--quiet` to the command-line interface.
|
||||
* [#154](https://github.com/socialcopsdev/camelot/issues/154) The CLI can now be run using `python -m`. Try `python -m camelot --help`. [#159](https://github.com/socialcopsdev/camelot/pull/159) by [Parth P Panchal](https://github.com/pqrth).
|
||||
* [#165](https://github.com/socialcopsdev/camelot/issues/165) Rename `table_area` to `table_areas`. [#171](https://github.com/socialcopsdev/camelot/pull/171) by [Parth P Panchal](https://github.com/pqrth).
|
||||
|
||||
**Bugfixes**
|
||||
|
||||
* Raise error if the ghostscript executable is not on the PATH variable. [#166](https://github.com/socialcopsdev/camelot/pull/166) by Vinayak Mehta.
|
||||
* Convert filename to lowercase to check for PDF extension. [#169](https://github.com/socialcopsdev/camelot/pull/169) by [Vinicius Mesel](https://github.com/vmesel).
|
||||
|
||||
**Files**
|
||||
|
||||
* [#114](https://github.com/socialcopsdev/camelot/issues/114) Add Makefile and make codecov run only once. [#132](https://github.com/socialcopsdev/camelot/pull/132) by [Vaibhav Mule](https://github.com/vaibhavmule).
|
||||
* Add .editorconfig. [#151](https://github.com/socialcopsdev/camelot/pull/151) by [KOLANICH](https://github.com/KOLANICH).
|
||||
* Downgrade numpy version from 1.15.2 to 1.13.3.
|
||||
* Add requirements.txt for readthedocs.
|
||||
|
||||
**Documentation**
|
||||
|
||||
* Add "Using conda" section to installation instructions.
|
||||
* Add readthedocs badge.
|
||||
|
||||
0.2.3 (2018-10-08)
|
||||
------------------
|
||||
|
||||
* Remove hard dependencies on requirements versions.
|
||||
|
||||
0.2.2 (2018-10-08)
|
||||
-----------------
|
||||
------------------
|
||||
|
||||
**Bugfixes**
|
||||
|
||||
@@ -58,4 +95,4 @@ Release History
|
||||
0.1.0 (2018-09-24)
|
||||
------------------
|
||||
|
||||
* Birth!
|
||||
* Rebirth!
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
.PHONY: docs
|
||||
INSTALL :=
|
||||
UNAME_S := $(shell uname -s)
|
||||
ifeq ($(UNAME_S),Linux)
|
||||
INSTALL := @sudo apt install python-tk python3-tk ghostscript
|
||||
else ifeq ($(UNAME_S),Darwin)
|
||||
INSTALL := @brew install tcl-tk ghostscript
|
||||
else
|
||||
INSTALL := @echo "Please install tk and ghostscript"
|
||||
endif
|
||||
|
||||
install:
|
||||
$(INSTALL)
|
||||
pip install --upgrade pip
|
||||
pip install ".[dev]"
|
||||
|
||||
test:
|
||||
pytest --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot tests
|
||||
|
||||
docs:
|
||||
cd docs && make html
|
||||
@echo "\033[95m\n\nBuild successful! View the docs homepage at docs/_build/html/index.html.\n\033[0m"
|
||||
|
||||
publish:
|
||||
pip install twine
|
||||
python setup.py sdist
|
||||
twine upload dist/*
|
||||
rm -fr build dist .egg camelot_py.egg-info
|
||||
@@ -4,11 +4,14 @@
|
||||
|
||||
# Camelot: PDF Table Extraction for Humans
|
||||
|
||||
[](https://travis-ci.org/socialcopsdev/camelot) [](https://codecov.io/github/socialcopsdev/camelot?branch=master)
|
||||
[](https://travis-ci.org/socialcopsdev/camelot) [](https://camelot-py.readthedocs.io/en/master/)
|
||||
[](https://codecov.io/github/socialcopsdev/camelot?branch=master)
|
||||
[](https://pypi.org/project/camelot-py/) [](https://pypi.org/project/camelot-py/) [](https://pypi.org/project/camelot-py/)
|
||||
|
||||
**Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
|
||||
|
||||
**Note:** You can also check out [Excalibur](https://github.com/camelot-dev/excalibur), which is a web interface for Camelot!
|
||||
|
||||
---
|
||||
|
||||
**Here's how you can extract tables from PDF files.** Check out the PDF used in this example [here](https://github.com/socialcopsdev/camelot/blob/master/docs/_static/pdf/foo.pdf).
|
||||
@@ -41,7 +44,7 @@
|
||||
| 2032_2 | 0.17 | 57.8 | 21.7% | 0.3% | 2.7% | 1.2% |
|
||||
| 4171_1 | 0.07 | 173.9 | 58.1% | 1.6% | 2.1% | 0.5% |
|
||||
|
||||
There's a [command-line interface](https://camelot-py.readthedocs.io/en/latest/user/cli.html) too!
|
||||
There's a [command-line interface](https://camelot-py.readthedocs.io/en/master/user/cli.html) too!
|
||||
|
||||
**Note:** Camelot only works with text-based PDFs and not scanned documents. (As Tabula [explains](https://github.com/tabulapdf/tabula#why-tabula), "If you can click and drag to select text in your table in a PDF viewer, then your PDF is text-based".)
|
||||
|
||||
@@ -56,15 +59,33 @@ See [comparison with other PDF table extraction libraries and tools](https://git
|
||||
|
||||
## Installation
|
||||
|
||||
After [installing the dependencies](https://camelot-py.readthedocs.io/en/latest/user/install.html) ([tk](https://packages.ubuntu.com/trusty/python-tk) and [ghostscript](https://www.ghostscript.com/)), you can simply use pip to install Camelot:
|
||||
### Using conda
|
||||
|
||||
The easiest way to install Camelot is to install it with [conda](https://conda.io/docs/), which is the package manager that the [Anaconda](http://docs.continuum.io/anaconda/) distribution is built upon.
|
||||
|
||||
First, let's add the [conda-forge](https://conda-forge.org/) channel to conda's config:
|
||||
|
||||
<pre>
|
||||
$ conda config --add channels conda-forge
|
||||
</pre>
|
||||
|
||||
Now, you can simply use conda to install Camelot:
|
||||
|
||||
<pre>
|
||||
$ conda install -c camelot-dev camelot-py
|
||||
</pre>
|
||||
|
||||
### Using pip
|
||||
|
||||
After [installing the dependencies](https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip) ([tk](https://packages.ubuntu.com/trusty/python-tk) and [ghostscript](https://www.ghostscript.com/)), you can simply use pip to install Camelot:
|
||||
|
||||
<pre>
|
||||
$ pip install camelot-py[all]
|
||||
</pre>
|
||||
|
||||
### Alternatively
|
||||
### From the source code
|
||||
|
||||
After [installing the dependencies](https://camelot-py.readthedocs.io/en/latest/user/install.html), clone the repo using:
|
||||
After [installing the dependencies](https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip), clone the repo using:
|
||||
|
||||
<pre>
|
||||
$ git clone https://www.github.com/socialcopsdev/camelot
|
||||
@@ -77,15 +98,13 @@ $ cd camelot
|
||||
$ pip install ".[all]"
|
||||
</pre>
|
||||
|
||||
**Note:** Use a [virtualenv](https://virtualenv.pypa.io/en/stable/) if you don't want to affect your global Python installation.
|
||||
|
||||
## Documentation
|
||||
|
||||
Great documentation is available at [http://camelot-py.readthedocs.io/](http://camelot-py.readthedocs.io/).
|
||||
|
||||
## Development
|
||||
|
||||
The [Contributor's Guide](https://camelot-py.readthedocs.io/en/latest/dev/contributing.html) has detailed information about contributing code, documentation, tests and more. We've included some basic information in this README.
|
||||
The [Contributor's Guide](https://camelot-py.readthedocs.io/en/master/dev/contributing.html) has detailed information about contributing code, documentation, tests and more. We've included some basic information in this README.
|
||||
|
||||
### Source code
|
||||
|
||||
|
||||
@@ -2,10 +2,20 @@
|
||||
|
||||
import logging
|
||||
|
||||
from click import HelpFormatter
|
||||
|
||||
from .__version__ import __version__
|
||||
from .io import read_pdf
|
||||
|
||||
|
||||
def _write_usage(self, prog, args='', prefix='Usage: '):
|
||||
return self._write_usage('camelot', args, prefix=prefix)
|
||||
|
||||
|
||||
# monkey patch click.HelpFormatter
|
||||
HelpFormatter._write_usage = HelpFormatter.write_usage
|
||||
HelpFormatter.write_usage = _write_usage
|
||||
|
||||
# set up logging
|
||||
logger = logging.getLogger('camelot')
|
||||
|
||||
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
|
||||
__all__ = ('main',)
|
||||
|
||||
|
||||
def main():
|
||||
from camelot.cli import cli
|
||||
|
||||
cli()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
VERSION = (0, 2, 2)
|
||||
VERSION = (0, 3, 0)
|
||||
|
||||
__title__ = 'camelot-py'
|
||||
__description__ = 'PDF Table Extraction for Humans.'
|
||||
|
||||
+14
-8
@@ -27,6 +27,7 @@ pass_config = click.make_pass_decorator(Config)
|
||||
@click.version_option(version=__version__)
|
||||
@click.option('-p', '--pages', default='1', help='Comma-separated page numbers.'
|
||||
' Example: 1,3,4 or 1,4-end.')
|
||||
@click.option('-pw', '--password', help='Password for decryption.')
|
||||
@click.option('-o', '--output', help='Output file path.')
|
||||
@click.option('-f', '--format',
|
||||
type=click.Choice(['csv', 'json', 'excel', 'html']),
|
||||
@@ -38,6 +39,7 @@ pass_config = click.make_pass_decorator(Config)
|
||||
' font size. Useful to detect super/subscripts.')
|
||||
@click.option('-M', '--margins', nargs=3, default=(1.0, 0.5, 0.1),
|
||||
help='PDFMiner char_margin, line_margin and word_margin.')
|
||||
@click.option('-q', '--quiet', is_flag=True, help='Suppress warnings.')
|
||||
@click.pass_context
|
||||
def cli(ctx, *args, **kwargs):
|
||||
"""Camelot: PDF Table Extraction for Humans"""
|
||||
@@ -47,7 +49,7 @@ def cli(ctx, *args, **kwargs):
|
||||
|
||||
|
||||
@cli.command('lattice')
|
||||
@click.option('-T', '--table_area', default=[], multiple=True,
|
||||
@click.option('-T', '--table_areas', default=[], multiple=True,
|
||||
help='Table areas to process. Example: x1,y1,x2,y2'
|
||||
' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
|
||||
@click.option('-back', '--process_background', is_flag=True,
|
||||
@@ -89,17 +91,19 @@ def lattice(c, *args, **kwargs):
|
||||
output = conf.pop('output')
|
||||
f = conf.pop('format')
|
||||
compress = conf.pop('zip')
|
||||
suppress_warnings = conf.pop('quiet')
|
||||
plot_type = kwargs.pop('plot_type')
|
||||
filepath = kwargs.pop('filepath')
|
||||
kwargs.update(conf)
|
||||
|
||||
table_area = list(kwargs['table_area'])
|
||||
kwargs['table_area'] = None if not table_area else table_area
|
||||
table_areas = list(kwargs['table_areas'])
|
||||
kwargs['table_areas'] = None if not table_areas else table_areas
|
||||
copy_text = list(kwargs['copy_text'])
|
||||
kwargs['copy_text'] = None if not copy_text else copy_text
|
||||
kwargs['shift_text'] = list(kwargs['shift_text'])
|
||||
|
||||
tables = read_pdf(filepath, pages=pages, flavor='lattice', **kwargs)
|
||||
tables = read_pdf(filepath, pages=pages, flavor='lattice',
|
||||
suppress_warnings=suppress_warnings, **kwargs)
|
||||
click.echo('Found {} tables'.format(tables.n))
|
||||
if plot_type is not None:
|
||||
for table in tables:
|
||||
@@ -113,7 +117,7 @@ def lattice(c, *args, **kwargs):
|
||||
|
||||
|
||||
@cli.command('stream')
|
||||
@click.option('-T', '--table_area', default=[], multiple=True,
|
||||
@click.option('-T', '--table_areas', default=[], multiple=True,
|
||||
help='Table areas to process. Example: x1,y1,x2,y2'
|
||||
' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
|
||||
@click.option('-C', '--columns', default=[], multiple=True,
|
||||
@@ -134,16 +138,18 @@ def stream(c, *args, **kwargs):
|
||||
output = conf.pop('output')
|
||||
f = conf.pop('format')
|
||||
compress = conf.pop('zip')
|
||||
suppress_warnings = conf.pop('quiet')
|
||||
plot_type = kwargs.pop('plot_type')
|
||||
filepath = kwargs.pop('filepath')
|
||||
kwargs.update(conf)
|
||||
|
||||
table_area = list(kwargs['table_area'])
|
||||
kwargs['table_area'] = None if not table_area else table_area
|
||||
table_areas = list(kwargs['table_areas'])
|
||||
kwargs['table_areas'] = None if not table_areas else table_areas
|
||||
columns = list(kwargs['columns'])
|
||||
kwargs['columns'] = None if not columns else columns
|
||||
|
||||
tables = read_pdf(filepath, pages=pages, flavor='stream', **kwargs)
|
||||
tables = read_pdf(filepath, pages=pages, flavor='stream',
|
||||
suppress_warnings=suppress_warnings, **kwargs)
|
||||
click.echo('Found {} tables'.format(tables.n))
|
||||
if plot_type is not None:
|
||||
for table in tables:
|
||||
|
||||
+16
-5
@@ -1,6 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PyPDF2 import PdfFileReader, PdfFileWriter
|
||||
|
||||
@@ -21,14 +22,22 @@ class PDFHandler(object):
|
||||
Path to PDF file.
|
||||
pages : str, optional (default: '1')
|
||||
Comma-separated page numbers.
|
||||
Example: 1,3,4 or 1,4-end.
|
||||
Example: '1,3,4' or '1,4-end'.
|
||||
password : str, optional (default: None)
|
||||
Password for decryption.
|
||||
|
||||
"""
|
||||
def __init__(self, filename, pages='1'):
|
||||
def __init__(self, filename, pages='1', password=None):
|
||||
self.filename = filename
|
||||
if not self.filename.endswith('.pdf'):
|
||||
if not filename.lower().endswith('.pdf'):
|
||||
raise NotImplementedError("File format not supported")
|
||||
self.pages = self._get_pages(self.filename, pages)
|
||||
if password is None:
|
||||
self.password = ''
|
||||
else:
|
||||
self.password = password
|
||||
if sys.version_info[0] < 3:
|
||||
self.password = self.password.encode('ascii')
|
||||
|
||||
def _get_pages(self, filename, pages):
|
||||
"""Converts pages string to list of ints.
|
||||
@@ -52,6 +61,8 @@ class PDFHandler(object):
|
||||
page_numbers.append({'start': 1, 'end': 1})
|
||||
else:
|
||||
infile = PdfFileReader(open(filename, 'rb'), strict=False)
|
||||
if infile.isEncrypted:
|
||||
infile.decrypt(self.password)
|
||||
if pages == 'all':
|
||||
page_numbers.append({'start': 1, 'end': infile.getNumPages()})
|
||||
else:
|
||||
@@ -84,7 +95,7 @@ class PDFHandler(object):
|
||||
with open(filename, 'rb') as fileobj:
|
||||
infile = PdfFileReader(fileobj, strict=False)
|
||||
if infile.isEncrypted:
|
||||
infile.decrypt('')
|
||||
infile.decrypt(self.password)
|
||||
fpath = os.path.join(temp, 'page-{0}.pdf'.format(page))
|
||||
froot, fext = os.path.splitext(fpath)
|
||||
p = infile.getPage(page - 1)
|
||||
@@ -103,7 +114,7 @@ class PDFHandler(object):
|
||||
os.rename(fpath, fpath_new)
|
||||
infile = PdfFileReader(open(fpath_new, 'rb'), strict=False)
|
||||
if infile.isEncrypted:
|
||||
infile.decrypt('')
|
||||
infile.decrypt(self.password)
|
||||
outfile = PdfFileWriter()
|
||||
p = infile.getPage(0)
|
||||
if rotation == 'anticlockwise':
|
||||
|
||||
+18
-8
@@ -1,10 +1,12 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import warnings
|
||||
|
||||
from .handlers import PDFHandler
|
||||
from .utils import validate_input, remove_extra
|
||||
|
||||
|
||||
def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
|
||||
def read_pdf(filepath, pages='1', password=None, flavor='lattice',
|
||||
suppress_warnings=False, **kwargs):
|
||||
"""Read PDF and return extracted tables.
|
||||
|
||||
Note: kwargs annotated with ^ can only be used with flavor='stream'
|
||||
@@ -16,11 +18,15 @@ def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
|
||||
Path to PDF file.
|
||||
pages : str, optional (default: '1')
|
||||
Comma-separated page numbers.
|
||||
Example: 1,3,4 or 1,4-end.
|
||||
Example: '1,3,4' or '1,4-end'.
|
||||
password : str, optional (default: None)
|
||||
Password for decryption.
|
||||
flavor : str (default: 'lattice')
|
||||
The parsing method to use ('lattice' or 'stream').
|
||||
Lattice is used by default.
|
||||
table_area : list, optional (default: None)
|
||||
suppress_warnings : bool, optional (default: False)
|
||||
Prevent warnings from being emitted by Camelot.
|
||||
table_areas : list, optional (default: None)
|
||||
List of table area strings of the form x1,y1,x2,y2
|
||||
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
|
||||
in PDF coordinate space.
|
||||
@@ -85,8 +91,12 @@ def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
|
||||
raise NotImplementedError("Unknown flavor specified."
|
||||
" Use either 'lattice' or 'stream'")
|
||||
|
||||
validate_input(kwargs, flavor=flavor)
|
||||
p = PDFHandler(filepath, pages)
|
||||
kwargs = remove_extra(kwargs, flavor=flavor)
|
||||
tables = p.parse(flavor=flavor, **kwargs)
|
||||
return tables
|
||||
with warnings.catch_warnings():
|
||||
if suppress_warnings:
|
||||
warnings.simplefilter("ignore")
|
||||
|
||||
validate_input(kwargs, flavor=flavor)
|
||||
p = PDFHandler(filepath, pages=pages, password=password)
|
||||
kwargs = remove_extra(kwargs, flavor=flavor)
|
||||
tables = p.parse(flavor=flavor, **kwargs)
|
||||
return tables
|
||||
|
||||
+39
-21
@@ -28,7 +28,7 @@ class Lattice(BaseParser):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table_area : list, optional (default: None)
|
||||
table_areas : list, optional (default: None)
|
||||
List of table area strings of the form x1,y1,x2,y2
|
||||
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
|
||||
in PDF coordinate space.
|
||||
@@ -76,12 +76,12 @@ class Lattice(BaseParser):
|
||||
For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
|
||||
|
||||
"""
|
||||
def __init__(self, table_area=None, process_background=False,
|
||||
def __init__(self, table_areas=None, process_background=False,
|
||||
line_size_scaling=15, copy_text=None, shift_text=['l', 't'],
|
||||
split_text=False, flag_size=False, line_close_tol=2,
|
||||
joint_close_tol=2, threshold_blocksize=15, threshold_constant=-2,
|
||||
iterations=0, margins=(1.0, 0.5, 0.1), **kwargs):
|
||||
self.table_area = table_area
|
||||
self.table_areas = table_areas
|
||||
self.process_background = process_background
|
||||
self.line_size_scaling = line_size_scaling
|
||||
self.copy_text = copy_text
|
||||
@@ -174,15 +174,39 @@ class Lattice(BaseParser):
|
||||
return t
|
||||
|
||||
def _generate_image(self):
|
||||
# TODO: hacky, get rid of ghostscript #96
|
||||
def get_platform():
|
||||
# TODO: get rid of ghostscript #96
|
||||
def get_executable():
|
||||
import platform
|
||||
from distutils.spawn import find_executable
|
||||
|
||||
info = {
|
||||
'system': platform.system().lower(),
|
||||
'machine': platform.machine().lower()
|
||||
}
|
||||
return info
|
||||
class GhostscriptNotFound(Exception): pass
|
||||
|
||||
gs = None
|
||||
system = platform.system().lower()
|
||||
try:
|
||||
if system == 'windows':
|
||||
if find_executable('gswin32c.exe'):
|
||||
gs = 'gswin32c.exe'
|
||||
elif find_executable('gswin64c.exe'):
|
||||
gs = 'gswin64c.exe'
|
||||
else:
|
||||
raise ValueError
|
||||
else:
|
||||
if find_executable('gs'):
|
||||
gs = 'gs'
|
||||
elif find_executable('gsc'):
|
||||
gs = 'gsc'
|
||||
else:
|
||||
raise ValueError
|
||||
if 'ghostscript' not in subprocess.check_output(
|
||||
[gs, '-version']).decode('utf-8').lower():
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise GhostscriptNotFound(
|
||||
'Please make sure that Ghostscript is installed'
|
||||
' and available on the PATH environment variable')
|
||||
|
||||
return gs
|
||||
|
||||
self.imagename = ''.join([self.rootname, '.png'])
|
||||
gs_call = [
|
||||
@@ -193,15 +217,9 @@ class Lattice(BaseParser):
|
||||
'-r600',
|
||||
self.filename
|
||||
]
|
||||
info = get_platform()
|
||||
if info['system'] == 'windows':
|
||||
bit = info['machine'][-2:]
|
||||
gs_call.insert(0, 'gswin{}c.exe'.format(bit))
|
||||
else:
|
||||
if 'ghostscript' in subprocess.check_output(['gs', '-version']).decode('utf-8').lower():
|
||||
gs_call.insert(0, 'gs')
|
||||
else:
|
||||
gs_call.insert(0, "gsc")
|
||||
gs = get_executable()
|
||||
gs_call.insert(0, gs)
|
||||
|
||||
subprocess.call(
|
||||
gs_call, stdout=open(os.devnull, 'w'),
|
||||
stderr=subprocess.STDOUT)
|
||||
@@ -226,9 +244,9 @@ class Lattice(BaseParser):
|
||||
self.threshold, direction='horizontal',
|
||||
line_size_scaling=self.line_size_scaling, iterations=self.iterations)
|
||||
|
||||
if self.table_area is not None:
|
||||
if self.table_areas is not None:
|
||||
areas = []
|
||||
for area in self.table_area:
|
||||
for area in self.table_areas:
|
||||
x1, y1, x2, y2 = area.split(",")
|
||||
x1 = float(x1)
|
||||
y1 = float(y1)
|
||||
|
||||
@@ -26,7 +26,7 @@ class Stream(BaseParser):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table_area : list, optional (default: None)
|
||||
table_areas : list, optional (default: None)
|
||||
List of table area strings of the form x1,y1,x2,y2
|
||||
where (x1, y1) -> left-top and (x2, y2) -> right-bottom
|
||||
in PDF coordinate space.
|
||||
@@ -50,10 +50,10 @@ class Stream(BaseParser):
|
||||
For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
|
||||
|
||||
"""
|
||||
def __init__(self, table_area=None, columns=None, split_text=False,
|
||||
def __init__(self, table_areas=None, columns=None, split_text=False,
|
||||
flag_size=False, row_close_tol=2, col_close_tol=0,
|
||||
margins=(1.0, 0.5, 0.1), **kwargs):
|
||||
self.table_area = table_area
|
||||
self.table_areas = table_areas
|
||||
self.columns = columns
|
||||
self._validate_columns()
|
||||
self.split_text = split_text
|
||||
@@ -241,15 +241,15 @@ class Stream(BaseParser):
|
||||
return cols
|
||||
|
||||
def _validate_columns(self):
|
||||
if self.table_area is not None and self.columns is not None:
|
||||
if len(self.table_area) != len(self.columns):
|
||||
raise ValueError("Length of table_area and columns"
|
||||
if self.table_areas is not None and self.columns is not None:
|
||||
if len(self.table_areas) != len(self.columns):
|
||||
raise ValueError("Length of table_areas and columns"
|
||||
" should be equal")
|
||||
|
||||
def _generate_table_bbox(self):
|
||||
if self.table_area is not None:
|
||||
if self.table_areas is not None:
|
||||
table_bbox = {}
|
||||
for area in self.table_area:
|
||||
for area in self.table_areas:
|
||||
x1, y1, x2, y2 = area.split(",")
|
||||
x1 = float(x1)
|
||||
y1 = float(y1)
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'Camelot'
|
||||
copyright = u'2018, Peeply Private Ltd (Singapore)'
|
||||
copyright = u'2018, <a href="https://socialcops.com" target="_blank">SocialCops</a>'
|
||||
author = u'Vinayak Mehta'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
|
||||
@@ -11,6 +11,10 @@ Release v\ |version|. (:ref:`Installation <install>`)
|
||||
.. image:: https://travis-ci.org/socialcopsdev/camelot.svg?branch=master
|
||||
:target: https://travis-ci.org/socialcopsdev/camelot
|
||||
|
||||
.. image:: https://readthedocs.org/projects/camelot-py/badge/?version=master
|
||||
:target: https://camelot-py.readthedocs.io/en/master/
|
||||
:alt: Documentation Status
|
||||
|
||||
.. image:: https://codecov.io/github/socialcopsdev/camelot/badge.svg?branch=master&service=github
|
||||
:target: https://codecov.io/github/socialcopsdev/camelot?branch=master
|
||||
|
||||
@@ -25,6 +29,10 @@ Release v\ |version|. (:ref:`Installation <install>`)
|
||||
|
||||
**Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
|
||||
|
||||
.. note:: You can also check out `Excalibur`_, which is a web interface for Camelot!
|
||||
|
||||
.. _Excalibur: https://github.com/camelot-dev/excalibur
|
||||
|
||||
----
|
||||
|
||||
**Here's how you can extract tables from PDF files.** Check out the PDF used in this example `here`_.
|
||||
|
||||
@@ -149,7 +149,7 @@ Finally, let's plot all line intersections present on the table's PDF page.
|
||||
Specify table areas
|
||||
-------------------
|
||||
|
||||
Since :ref:`Stream <stream>` treats the whole page as a table, `for now`_, it's useful to specify table boundaries in cases such as `these <../_static/pdf/table_areas.pdf>`__. You can :ref:`plot the text <geometry_text>` on this page and note the left-top and right-bottom coordinates of the table.
|
||||
Since :ref:`Stream <stream>` treats the whole page as a table, `for now`_, it's useful to specify table boundaries in cases such as `these <../_static/pdf/table_areas.pdf>`__. You can :ref:`plot the text <geometry_text>` on this page and note the top left and bottom right coordinates of the table.
|
||||
|
||||
Table areas that you want Camelot to analyze can be passed as a list of comma-separated strings to :meth:`read_pdf() <camelot.read_pdf>`, using the ``table_areas`` keyword argument.
|
||||
|
||||
@@ -416,4 +416,4 @@ We don't need anything else. Now, let's pass ``copy_text=['v']`` to copy text in
|
||||
"3","Odisha","Kalahandi","iii. Food Poisoning","42","0","02/01/14","03/01/14","Under control","..."
|
||||
"4","West Bengal","West Medinipur","iv. Acute Diarrhoeal Disease","145","0","04/01/14","05/01/14","Under control","..."
|
||||
"4","West Bengal","Birbhum","v. Food Poisoning","199","0","31/12/13","31/12/13","Under control","..."
|
||||
"4","West Bengal","Howrah","vi. Viral Hepatitis A &E","85","0","26/12/13","27/12/13","Under surveillance","..."
|
||||
"4","West Bengal","Howrah","vi. Viral Hepatitis A &E","85","0","26/12/13","27/12/13","Under surveillance","..."
|
||||
|
||||
+21
-19
@@ -9,26 +9,28 @@ You can print the help for the interface by typing ``camelot --help`` in your fa
|
||||
|
||||
::
|
||||
|
||||
Usage: camelot [OPTIONS] COMMAND [ARGS]...
|
||||
Usage: camelot [OPTIONS] COMMAND [ARGS]...
|
||||
|
||||
Camelot: PDF Table Extraction for Humans
|
||||
|
||||
Options:
|
||||
--version Show the version and exit.
|
||||
-p, --pages TEXT Comma-separated page numbers. Example: 1,3,4
|
||||
or 1,4-end.
|
||||
-o, --output TEXT Output file path.
|
||||
-f, --format [csv|json|excel|html]
|
||||
Output file format.
|
||||
-z, --zip Create ZIP archive.
|
||||
-split, --split_text Split text that spans across multiple cells.
|
||||
-flag, --flag_size Flag text based on font size. Useful to
|
||||
detect super/subscripts.
|
||||
-M, --margins <FLOAT FLOAT FLOAT>...
|
||||
PDFMiner char_margin, line_margin and
|
||||
word_margin.
|
||||
--help Show this message and exit.
|
||||
Options:
|
||||
--version Show the version and exit.
|
||||
-p, --pages TEXT Comma-separated page numbers. Example: 1,3,4
|
||||
or 1,4-end.
|
||||
-pw, --password TEXT Password for decryption.
|
||||
-o, --output TEXT Output file path.
|
||||
-f, --format [csv|json|excel|html]
|
||||
Output file format.
|
||||
-z, --zip Create ZIP archive.
|
||||
-split, --split_text Split text that spans across multiple cells.
|
||||
-flag, --flag_size Flag text based on font size. Useful to
|
||||
detect super/subscripts.
|
||||
-M, --margins <FLOAT FLOAT FLOAT>...
|
||||
PDFMiner char_margin, line_margin and
|
||||
word_margin.
|
||||
-q, --quiet Suppress warnings.
|
||||
--help Show this message and exit.
|
||||
|
||||
Commands:
|
||||
lattice Use lines between text to parse the table.
|
||||
stream Use spaces between text to parse the table.
|
||||
Commands:
|
||||
lattice Use lines between text to parse the table.
|
||||
stream Use spaces between text to parse the table.
|
||||
|
||||
+33
-12
@@ -3,14 +3,36 @@
|
||||
Installation of Camelot
|
||||
=======================
|
||||
|
||||
This part of the documentation covers how to install Camelot. First, you'll need to install the dependencies, which include `Tkinter`_ and `ghostscript`_.
|
||||
This part of the documentation covers how to install Camelot.
|
||||
|
||||
Using conda
|
||||
-----------
|
||||
|
||||
The easiest way to install Camelot is to install it with `conda`_, which is the package manager that the `Anaconda`_ distribution is built upon.
|
||||
|
||||
First, let's add the `conda-forge`_ channel to conda's config::
|
||||
|
||||
$ conda config --add channels conda-forge
|
||||
|
||||
Now, you can simply use conda to install Camelot::
|
||||
|
||||
$ conda install -c camelot-dev camelot-py
|
||||
|
||||
.. note:: Camelot is available for Python 2.7, 3.5 and 3.6 on Linux, macOS and Windows. For Windows, you will need to install ghostscript which you can get from their `downloads page`_.
|
||||
|
||||
.. _conda: https://conda.io/docs/
|
||||
.. _Anaconda: http://docs.continuum.io/anaconda/
|
||||
.. _downloads page: https://www.ghostscript.com/download/gsdnld.html
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
|
||||
Using pip
|
||||
---------
|
||||
|
||||
First, you'll need to install the dependencies, which include `Tkinter`_ and `ghostscript`_.
|
||||
|
||||
.. _Tkinter: https://wiki.python.org/moin/TkInter
|
||||
.. _ghostscript: https://www.ghostscript.com
|
||||
|
||||
Install the dependencies
|
||||
------------------------
|
||||
|
||||
These can be installed using your system's package manager. You can run one of the following, based on your OS.
|
||||
|
||||
For Ubuntu
|
||||
@@ -76,17 +98,14 @@ Or for Windows 32-bit::
|
||||
|
||||
If you have ghostscript, you should see the ghostscript version and copyright information.
|
||||
|
||||
$ pip install camelot-py[all]
|
||||
-----------------------------
|
||||
|
||||
After installing the dependencies, you can simply use pip to install Camelot::
|
||||
Finally, you can use pip to install Camelot::
|
||||
|
||||
$ pip install camelot-py[all]
|
||||
|
||||
Get the source code
|
||||
-------------------
|
||||
From the source code
|
||||
--------------------
|
||||
|
||||
Alternatively, you can install from the source by:
|
||||
After `installing the dependencies`_, you can install from the source by:
|
||||
|
||||
1. Cloning the GitHub repository.
|
||||
::
|
||||
@@ -97,4 +116,6 @@ Alternatively, you can install from the source by:
|
||||
::
|
||||
|
||||
$ cd camelot
|
||||
$ pip install ".[all]"
|
||||
$ pip install ".[all]"
|
||||
|
||||
.. _installing the dependencies: https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip
|
||||
@@ -87,6 +87,28 @@ By default, Camelot only uses the first page of the PDF to extract tables. To sp
|
||||
|
||||
The ``pages`` keyword argument accepts pages as comma-separated string of page numbers. You can also specify page ranges — for example, ``pages=1,4-10,20-30`` or ``pages=1,4-10,20-end``.
|
||||
|
||||
------------------------
|
||||
Reading encrypted PDFs
|
||||
----------------------
|
||||
|
||||
Ready for more? Check out the :ref:`advanced <advanced>` section.
|
||||
To extract tables from encrypted PDF files you must provide a password when calling :meth:`read_pdf() <camelot.read_pdf>`.
|
||||
|
||||
::
|
||||
|
||||
>>> tables = camelot.read_pdf('foo.pdf', password='userpass')
|
||||
>>> tables
|
||||
<TableList n=1>
|
||||
|
||||
Currently Camelot only supports PDFs encrypted with ASCII passwords and algorithm `code 1 or 2`_. An exception is thrown if the PDF cannot be read. This may be due to no password being provided, an incorrect password, or an unsupported encryption algorithm.
|
||||
|
||||
Further encryption support may be added in future, however in the meantime if your PDF files are using unsupported encryption algorithms you are advised to remove encryption before calling :meth:`read_pdf() <camelot.read_pdf>`. This can been successfully achieved with third-party tools such as `QPDF`_.
|
||||
|
||||
::
|
||||
|
||||
$ qpdf --password=<PASSWORD> --decrypt input.pdf output.pdf
|
||||
|
||||
.. _code 1 or 2: https://github.com/mstamy2/PyPDF2/issues/378
|
||||
.. _QPDF: https://www.github.com/qpdf/qpdf
|
||||
|
||||
----
|
||||
|
||||
Ready for more? Check out the :ref:`advanced <advanced>` section.
|
||||
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
click>=6.7
|
||||
matplotlib>=2.2.3
|
||||
numpy>=1.13.3
|
||||
opencv-python>=3.4.2.17
|
||||
openpyxl>=2.5.8
|
||||
pandas>=0.23.4
|
||||
pdfminer.six>=20170720
|
||||
PyPDF2>=1.26.0
|
||||
@@ -14,25 +14,25 @@ with open('README.md', 'r') as f:
|
||||
|
||||
|
||||
requires = [
|
||||
'click==6.7',
|
||||
'matplotlib==2.2.3',
|
||||
'numpy==1.15.2',
|
||||
'openpyxl==2.5.8',
|
||||
'pandas==0.23.4',
|
||||
'pdfminer.six==20170720',
|
||||
'PyPDF2==1.26.0'
|
||||
'click>=6.7',
|
||||
'matplotlib>=2.2.3',
|
||||
'numpy>=1.13.3',
|
||||
'openpyxl>=2.5.8',
|
||||
'pandas>=0.23.4',
|
||||
'pdfminer.six>=20170720',
|
||||
'PyPDF2>=1.26.0'
|
||||
]
|
||||
|
||||
all_requires = [
|
||||
'opencv-python==3.4.2.17'
|
||||
'opencv-python>=3.4.2.17'
|
||||
]
|
||||
|
||||
dev_requires = [
|
||||
'codecov==2.0.15',
|
||||
'pytest==3.8.0',
|
||||
'pytest-cov==2.6.0',
|
||||
'pytest-runner==4.2',
|
||||
'Sphinx==1.7.9'
|
||||
'codecov>=2.0.15',
|
||||
'pytest>=3.8.0',
|
||||
'pytest-cov>=2.6.0',
|
||||
'pytest-runner>=4.2',
|
||||
'Sphinx>=1.7.9'
|
||||
]
|
||||
dev_requires = dev_requires + all_requires
|
||||
|
||||
|
||||
+2
-2
@@ -81,7 +81,7 @@ data_stream_table_rotated = [
|
||||
["", "", "", "", "", "", "", "", "54", "", "", "", "", "", "", "", "", ""]
|
||||
]
|
||||
|
||||
data_stream_table_area = [
|
||||
data_stream_table_areas = [
|
||||
["", "One Withholding"],
|
||||
["Payroll Period", "Allowance"],
|
||||
["Weekly", "$71.15"],
|
||||
@@ -261,7 +261,7 @@ data_lattice_table_rotated = [
|
||||
["Pooled", "38742", "53618", "60601", "86898", "4459", "21918", "27041", "14312", "18519"]
|
||||
]
|
||||
|
||||
data_lattice_table_area = [
|
||||
data_lattice_table_areas = [
|
||||
["", "", "", "", "", "", "", "", ""],
|
||||
["State", "n", "Literacy Status", "", "", "", "", "", ""],
|
||||
["", "", "Illiterate", "Read & Write", "1-4 std.", "5-8 std.", "9-12 std.", "College", ""],
|
||||
|
||||
Binary file not shown.
+117
-79
@@ -1,79 +1,117 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from camelot.cli import cli
|
||||
from camelot.utils import TemporaryDirectory
|
||||
|
||||
|
||||
testdir = os.path.dirname(os.path.abspath(__file__))
|
||||
testdir = os.path.join(testdir, 'files')
|
||||
|
||||
|
||||
def test_cli_lattice():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'foo.pdf')
|
||||
outfile = os.path.join(tempdir, 'foo.csv')
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
|
||||
'lattice', infile])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == 'Found 1 tables\n'
|
||||
|
||||
result = runner.invoke(cli, ['--format', 'csv',
|
||||
'lattice', infile])
|
||||
output_error = 'Error: Please specify output file path using --output'
|
||||
assert output_error in result.output
|
||||
|
||||
result = runner.invoke(cli, ['--output', outfile,
|
||||
'lattice', infile])
|
||||
format_error = 'Please specify output file format using --format'
|
||||
assert format_error in result.output
|
||||
|
||||
|
||||
def test_cli_stream():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'budget.pdf')
|
||||
outfile = os.path.join(tempdir, 'budget.csv')
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == 'Found 1 tables\n'
|
||||
|
||||
result = runner.invoke(cli, ['--format', 'csv', 'stream', infile])
|
||||
output_error = 'Error: Please specify output file path using --output'
|
||||
assert output_error in result.output
|
||||
|
||||
result = runner.invoke(cli, ['--output', outfile, 'stream', infile])
|
||||
format_error = 'Please specify output file format using --format'
|
||||
assert format_error in result.output
|
||||
|
||||
|
||||
def test_cli_output_format():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'health.pdf')
|
||||
outfile = os.path.join(tempdir, 'health.{}')
|
||||
runner = CliRunner()
|
||||
|
||||
# json
|
||||
result = runner.invoke(cli, ['--format', 'json', '--output', outfile.format('json'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# excel
|
||||
result = runner.invoke(cli, ['--format', 'excel', '--output', outfile.format('xlsx'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# html
|
||||
result = runner.invoke(cli, ['--format', 'html', '--output', outfile.format('html'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# zip
|
||||
result = runner.invoke(cli, ['--zip', '--format', 'csv', '--output', outfile.format('csv'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from camelot.cli import cli
|
||||
from camelot.utils import TemporaryDirectory
|
||||
|
||||
|
||||
testdir = os.path.dirname(os.path.abspath(__file__))
|
||||
testdir = os.path.join(testdir, 'files')
|
||||
|
||||
|
||||
def test_cli_lattice():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'foo.pdf')
|
||||
outfile = os.path.join(tempdir, 'foo.csv')
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
|
||||
'lattice', infile])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == 'Found 1 tables\n'
|
||||
|
||||
result = runner.invoke(cli, ['--format', 'csv',
|
||||
'lattice', infile])
|
||||
output_error = 'Error: Please specify output file path using --output'
|
||||
assert output_error in result.output
|
||||
|
||||
result = runner.invoke(cli, ['--output', outfile,
|
||||
'lattice', infile])
|
||||
format_error = 'Please specify output file format using --format'
|
||||
assert format_error in result.output
|
||||
|
||||
|
||||
def test_cli_stream():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'budget.pdf')
|
||||
outfile = os.path.join(tempdir, 'budget.csv')
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == 'Found 1 tables\n'
|
||||
|
||||
result = runner.invoke(cli, ['--format', 'csv', 'stream', infile])
|
||||
output_error = 'Error: Please specify output file path using --output'
|
||||
assert output_error in result.output
|
||||
|
||||
result = runner.invoke(cli, ['--output', outfile, 'stream', infile])
|
||||
format_error = 'Please specify output file format using --format'
|
||||
assert format_error in result.output
|
||||
|
||||
|
||||
def test_cli_password():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'health_protected.pdf')
|
||||
outfile = os.path.join(tempdir, 'health_protected.csv')
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ['--password', 'userpass',
|
||||
'--format', 'csv', '--output', outfile,
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == 'Found 1 tables\n'
|
||||
|
||||
output_error = 'file has not been decrypted'
|
||||
# no password
|
||||
result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
|
||||
'stream', infile])
|
||||
assert output_error in str(result.exception)
|
||||
|
||||
# bad password
|
||||
result = runner.invoke(cli, ['--password', 'wrongpass',
|
||||
'--format', 'csv', '--output', outfile,
|
||||
'stream', infile])
|
||||
assert output_error in str(result.exception)
|
||||
|
||||
|
||||
def test_cli_output_format():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'health.pdf')
|
||||
outfile = os.path.join(tempdir, 'health.{}')
|
||||
runner = CliRunner()
|
||||
|
||||
# json
|
||||
result = runner.invoke(cli, ['--format', 'json', '--output', outfile.format('json'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# excel
|
||||
result = runner.invoke(cli, ['--format', 'excel', '--output', outfile.format('xlsx'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# html
|
||||
result = runner.invoke(cli, ['--format', 'html', '--output', outfile.format('html'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# zip
|
||||
result = runner.invoke(cli, ['--zip', '--format', 'csv', '--output', outfile.format('csv'),
|
||||
'stream', infile])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_cli_quiet():
|
||||
with TemporaryDirectory() as tempdir:
|
||||
infile = os.path.join(testdir, 'blank.pdf')
|
||||
outfile = os.path.join(tempdir, 'blank.csv')
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
|
||||
'stream', infile])
|
||||
assert 'No tables found on page-1' in result.output
|
||||
|
||||
result = runner.invoke(cli, ['--quiet', '--format', 'csv',
|
||||
'--output', outfile, 'stream', infile])
|
||||
assert 'No tables found on page-1' not in result.output
|
||||
|
||||
+18
-7
@@ -25,6 +25,17 @@ def test_parsing_report():
|
||||
assert tables[0].parsing_report == parsing_report
|
||||
|
||||
|
||||
def test_password():
|
||||
df = pd.DataFrame(data_stream)
|
||||
|
||||
filename = os.path.join(testdir, "health_protected.pdf")
|
||||
tables = camelot.read_pdf(filename, password="ownerpass", flavor="stream")
|
||||
assert df.equals(tables[0].df)
|
||||
|
||||
tables = camelot.read_pdf(filename, password="userpass", flavor="stream")
|
||||
assert df.equals(tables[0].df)
|
||||
|
||||
|
||||
def test_stream():
|
||||
df = pd.DataFrame(data_stream)
|
||||
|
||||
@@ -45,11 +56,11 @@ def test_stream_table_rotated():
|
||||
assert df.equals(tables[0].df)
|
||||
|
||||
|
||||
def test_stream_table_area():
|
||||
df = pd.DataFrame(data_stream_table_area)
|
||||
def test_stream_table_areas():
|
||||
df = pd.DataFrame(data_stream_table_areas)
|
||||
|
||||
filename = os.path.join(testdir, "tabula/us-007.pdf")
|
||||
tables = camelot.read_pdf(filename, flavor="stream", table_area=["320,500,573,335"])
|
||||
tables = camelot.read_pdf(filename, flavor="stream", table_areas=["320,500,573,335"])
|
||||
assert df.equals(tables[0].df)
|
||||
|
||||
|
||||
@@ -100,11 +111,11 @@ def test_lattice_table_rotated():
|
||||
assert df.equals(tables[0].df)
|
||||
|
||||
|
||||
def test_lattice_table_area():
|
||||
df = pd.DataFrame(data_lattice_table_area)
|
||||
def test_lattice_table_areas():
|
||||
df = pd.DataFrame(data_lattice_table_areas)
|
||||
|
||||
filename = os.path.join(testdir, "twotables_2.pdf")
|
||||
tables = camelot.read_pdf(filename, table_area=["80,693,535,448"])
|
||||
tables = camelot.read_pdf(filename, table_areas=["80,693,535,448"])
|
||||
assert df.equals(tables[0].df)
|
||||
|
||||
|
||||
@@ -139,7 +150,7 @@ def test_lattice_shift_text():
|
||||
tables = camelot.read_pdf(filename, line_size_scaling=40, shift_text=['r', 'b'])
|
||||
assert df_rb.equals(tables[0].df)
|
||||
|
||||
|
||||
|
||||
def test_repr():
|
||||
filename = os.path.join(testdir, "foo.pdf")
|
||||
tables = camelot.read_pdf(filename)
|
||||
|
||||
+45
-7
@@ -34,20 +34,58 @@ def test_unsupported_format():
|
||||
|
||||
|
||||
def test_stream_equal_length():
|
||||
message = ("Length of table_area and columns"
|
||||
message = ("Length of table_areas and columns"
|
||||
" should be equal")
|
||||
with pytest.raises(ValueError, message=message):
|
||||
tables = camelot.read_pdf(filename, flavor='stream',
|
||||
table_area=['10,20,30,40'], columns=['10,20,30,40', '10,20,30,40'])
|
||||
table_areas=['10,20,30,40'], columns=['10,20,30,40', '10,20,30,40'])
|
||||
|
||||
|
||||
def test_no_tables_found():
|
||||
filename = os.path.join(testdir, 'blank.pdf')
|
||||
# TODO: use pytest.warns
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error')
|
||||
try:
|
||||
with pytest.raises(UserWarning) as e:
|
||||
tables = camelot.read_pdf(filename)
|
||||
except Exception as e:
|
||||
assert type(e).__name__ == 'UserWarning'
|
||||
assert str(e) == 'No tables found on page-1'
|
||||
assert str(e.value) == 'No tables found on page-1'
|
||||
|
||||
|
||||
def test_no_tables_found_warnings_suppressed():
|
||||
filename = os.path.join(testdir, 'blank.pdf')
|
||||
with warnings.catch_warnings():
|
||||
# the test should fail if any warning is thrown
|
||||
warnings.simplefilter('error')
|
||||
try:
|
||||
tables = camelot.read_pdf(filename, suppress_warnings=True)
|
||||
except Warning as e:
|
||||
warning_text = str(e)
|
||||
pytest.fail('Unexpected warning: {}'.format(warning_text))
|
||||
|
||||
|
||||
def test_ghostscript_not_found(monkeypatch):
|
||||
import distutils
|
||||
|
||||
def _find_executable_patch(arg):
|
||||
return ''
|
||||
|
||||
monkeypatch.setattr(distutils.spawn, 'find_executable', _find_executable_patch)
|
||||
|
||||
message = ('Please make sure that Ghostscript is installed and available'
|
||||
' on the PATH environment variable')
|
||||
filename = os.path.join(testdir, 'foo.pdf')
|
||||
with pytest.raises(Exception, message=message):
|
||||
tables = camelot.read_pdf(filename)
|
||||
|
||||
|
||||
def test_no_password():
|
||||
filename = os.path.join(testdir, 'health_protected.pdf')
|
||||
message = 'file has not been decrypted'
|
||||
with pytest.raises(Exception, message=message):
|
||||
tables = camelot.read_pdf(filename)
|
||||
|
||||
|
||||
def test_bad_password():
|
||||
filename = os.path.join(testdir, 'health_protected.pdf')
|
||||
message = 'file has not been decrypted'
|
||||
with pytest.raises(Exception, message=message):
|
||||
tables = camelot.read_pdf(filename, password='wrongpass')
|
||||
|
||||
Reference in New Issue
Block a user