Bump version

Update HISTORY.md

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.py]
+charset = utf-8
+indent_style = space
+indent_size = 4

.travis.yml

@@ -1,20 +1,32 @@
-sudo: false
+sudo: true
 language: python
 cache: pip
-python:
+addons:
-  - "2.7"
+  apt:
-  - "3.5"
+    update: true
-  - "3.6"
-matrix:
-  include:
-    - python: 3.7
-      dist: xenial
-      sudo: true
-before_install:
-  - sudo apt-get install python-tk python3-tk ghostscript
 install:
-  - pip install ".[dev]"
+  - make install
-script:
+jobs:
-  - pytest --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot tests
+  include:
-after_success:
+    - stage: test
-  - codecov --verbose
+      script:
+        - make test
+      python: '2.7'
+    - stage: test
+      script:
+        - make test
+      python: '3.5'
+    - stage: test
+      script:
+        - make test
+      python: '3.6'
+    - stage: test
+      script:
+        - make test
+      python: '3.7'
+      dist: xenial
+    - stage: coverage
+      python: '3.6'
+      script:
+        - make test
+        - codecov --verbose

HISTORY.md

@@ -1,8 +1,45 @@
 Release History
 ===============
 
+master
+------
+
+0.3.0 (2018-10-28)
+------------------
+
+**Improvements**
+
+* [#162](https://github.com/socialcopsdev/camelot/issues/162) Add password keyword argument. [#180](https://github.com/socialcopsdev/camelot/pull/180) by [rbares](https://github.com/rbares).
+    * An encrypted PDF can now be decrypted by passing `password='<PASSWORD>'` to `read_pdf` or `--password <PASSWORD>` to the command-line interface. (Limited encryption algorithm support from PyPDF2.)
+* [#139](https://github.com/socialcopsdev/camelot/issues/139) Add suppress_warnings keyword argument. [#155](https://github.com/socialcopsdev/camelot/pull/155) by [Jonathan Lloyd](https://github.com/jonathanlloyd).
+    * Warnings raised by Camelot can now be suppressed by passing `suppress_warnings=True` to `read_pdf` or `--quiet` to the command-line interface.
+* [#154](https://github.com/socialcopsdev/camelot/issues/154) The CLI can now be run using `python -m`. Try `python -m camelot --help`. [#159](https://github.com/socialcopsdev/camelot/pull/159) by [Parth P Panchal](https://github.com/pqrth).
+* [#165](https://github.com/socialcopsdev/camelot/issues/165) Rename `table_area` to `table_areas`. [#171](https://github.com/socialcopsdev/camelot/pull/171) by [Parth P Panchal](https://github.com/pqrth).
+
+**Bugfixes**
+
+* Raise error if the ghostscript executable is not on the PATH variable. [#166](https://github.com/socialcopsdev/camelot/pull/166) by Vinayak Mehta.
+* Convert filename to lowercase to check for PDF extension. [#169](https://github.com/socialcopsdev/camelot/pull/169) by [Vinicius Mesel](https://github.com/vmesel).
+
+**Files**
+
+* [#114](https://github.com/socialcopsdev/camelot/issues/114) Add Makefile and make codecov run only once. [#132](https://github.com/socialcopsdev/camelot/pull/132) by [Vaibhav Mule](https://github.com/vaibhavmule).
+* Add .editorconfig. [#151](https://github.com/socialcopsdev/camelot/pull/151) by [KOLANICH](https://github.com/KOLANICH).
+* Downgrade numpy version from 1.15.2 to 1.13.3.
+* Add requirements.txt for readthedocs.
+
+**Documentation**
+
+* Add "Using conda" section to installation instructions.
+* Add readthedocs badge.
+
+0.2.3 (2018-10-08)
+------------------
+
+* Remove hard dependencies on requirements versions.
+
 0.2.2 (2018-10-08)
------------------
+------------------
 
 **Bugfixes**
 
@@ -58,4 +95,4 @@ Release History
 0.1.0 (2018-09-24)
 ------------------
 
-* Birth!
+* Rebirth!

Makefile

@@ -0,0 +1,28 @@
+.PHONY: docs
+INSTALL :=
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+	INSTALL := @sudo apt install python-tk python3-tk ghostscript
+else ifeq ($(UNAME_S),Darwin)
+	INSTALL := @brew install tcl-tk ghostscript
+else
+	INSTALL := @echo "Please install tk and ghostscript"
+endif
+
+install:
+	$(INSTALL)
+	pip install --upgrade pip
+	pip install ".[dev]"
+
+test:
+	pytest --verbose --cov-config .coveragerc --cov-report term --cov-report xml --cov=camelot tests
+
+docs:
+	cd docs && make html
+	@echo "\033[95m\n\nBuild successful! View the docs homepage at docs/_build/html/index.html.\n\033[0m"
+
+publish:
+	pip install twine
+	python setup.py sdist
+	twine upload dist/*
+	rm -fr build dist .egg camelot_py.egg-info

README.md

@@ -4,11 +4,14 @@
 
 # Camelot: PDF Table Extraction for Humans
 
-[![Build Status](https://travis-ci.org/socialcopsdev/camelot.svg?branch=master)](https://travis-ci.org/socialcopsdev/camelot) [![codecov.io](https://codecov.io/github/socialcopsdev/camelot/badge.svg?branch=master&service=github)](https://codecov.io/github/socialcopsdev/camelot?branch=master)
+[![Build Status](https://travis-ci.org/socialcopsdev/camelot.svg?branch=master)](https://travis-ci.org/socialcopsdev/camelot) [![Documentation Status](https://readthedocs.org/projects/camelot-py/badge/?version=master)](https://camelot-py.readthedocs.io/en/master/)
+ [![codecov.io](https://codecov.io/github/socialcopsdev/camelot/badge.svg?branch=master&service=github)](https://codecov.io/github/socialcopsdev/camelot?branch=master)
  [![image](https://img.shields.io/pypi/v/camelot-py.svg)](https://pypi.org/project/camelot-py/) [![image](https://img.shields.io/pypi/l/camelot-py.svg)](https://pypi.org/project/camelot-py/) [![image](https://img.shields.io/pypi/pyversions/camelot-py.svg)](https://pypi.org/project/camelot-py/)
 
 **Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
 
+**Note:** You can also check out [Excalibur](https://github.com/camelot-dev/excalibur), which is a web interface for Camelot!
+
 ---
 
 **Here's how you can extract tables from PDF files.** Check out the PDF used in this example [here](https://github.com/socialcopsdev/camelot/blob/master/docs/_static/pdf/foo.pdf).
@@ -41,7 +44,7 @@
 | 2032_2     | 0.17      | 57.8          | 21.7%                | 0.3%            | 2.7%            | 1.2%           |
 | 4171_1     | 0.07      | 173.9         | 58.1%                | 1.6%            | 2.1%            | 0.5%           |
 
-There's a [command-line interface](https://camelot-py.readthedocs.io/en/latest/user/cli.html) too!
+There's a [command-line interface](https://camelot-py.readthedocs.io/en/master/user/cli.html) too!
 
 **Note:** Camelot only works with text-based PDFs and not scanned documents. (As Tabula [explains](https://github.com/tabulapdf/tabula#why-tabula), "If you can click and drag to select text in your table in a PDF viewer, then your PDF is text-based".)
 
@@ -56,15 +59,33 @@ See [comparison with other PDF table extraction libraries and tools](https://git
 
 ## Installation
 
-After [installing the dependencies](https://camelot-py.readthedocs.io/en/latest/user/install.html) ([tk](https://packages.ubuntu.com/trusty/python-tk) and [ghostscript](https://www.ghostscript.com/)), you can simply use pip to install Camelot:
+### Using conda
+
+The easiest way to install Camelot is to install it with [conda](https://conda.io/docs/), which is the package manager that the [Anaconda](http://docs.continuum.io/anaconda/) distribution is built upon.
+
+First, let's add the [conda-forge](https://conda-forge.org/) channel to conda's config:
+
+<pre>
+$ conda config --add channels conda-forge
+</pre>
+
+Now, you can simply use conda to install Camelot:
+
+<pre>
+$ conda install -c camelot-dev camelot-py
+</pre>
+
+### Using pip
+
+After [installing the dependencies](https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip) ([tk](https://packages.ubuntu.com/trusty/python-tk) and [ghostscript](https://www.ghostscript.com/)), you can simply use pip to install Camelot:
 
 <pre>
 $ pip install camelot-py[all]
 </pre>
 
-### Alternatively
+### From the source code
 
-After [installing the dependencies](https://camelot-py.readthedocs.io/en/latest/user/install.html), clone the repo using:
+After [installing the dependencies](https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip), clone the repo using:
 
 <pre>
 $ git clone https://www.github.com/socialcopsdev/camelot
@@ -77,15 +98,13 @@ $ cd camelot
 $ pip install ".[all]"
 </pre>
 
-**Note:** Use a [virtualenv](https://virtualenv.pypa.io/en/stable/) if you don't want to affect your global Python installation.
-
 ## Documentation
 
 Great documentation is available at [http://camelot-py.readthedocs.io/](http://camelot-py.readthedocs.io/).
 
 ## Development
 
-The [Contributor's Guide](https://camelot-py.readthedocs.io/en/latest/dev/contributing.html) has detailed information about contributing code, documentation, tests and more. We've included some basic information in this README.
+The [Contributor's Guide](https://camelot-py.readthedocs.io/en/master/dev/contributing.html) has detailed information about contributing code, documentation, tests and more. We've included some basic information in this README.
 
 ### Source code
 

camelot/__init__.py

@@ -2,10 +2,20 @@
 
 import logging
 
+from click import HelpFormatter
+
 from .__version__ import __version__
 from .io import read_pdf
 
 
+def _write_usage(self, prog, args='', prefix='Usage: '):
+    return self._write_usage('camelot', args, prefix=prefix)
+
+
+# monkey patch click.HelpFormatter
+HelpFormatter._write_usage = HelpFormatter.write_usage
+HelpFormatter.write_usage = _write_usage
+
 # set up logging
 logger = logging.getLogger('camelot')
 

camelot/__main__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+
+__all__ = ('main',)
+
+
+def main():
+    from camelot.cli import cli
+
+    cli()
+
+
+if __name__ == "__main__":
+    main()

camelot/__version__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-VERSION = (0, 2, 2)
+VERSION = (0, 3, 0)
 
 __title__ = 'camelot-py'
 __description__ = 'PDF Table Extraction for Humans.'

camelot/cli.py

@@ -27,6 +27,7 @@ pass_config = click.make_pass_decorator(Config)
 @click.version_option(version=__version__)
 @click.option('-p', '--pages', default='1', help='Comma-separated page numbers.'
               ' Example: 1,3,4 or 1,4-end.')
+@click.option('-pw', '--password', help='Password for decryption.')
 @click.option('-o', '--output', help='Output file path.')
 @click.option('-f', '--format',
               type=click.Choice(['csv', 'json', 'excel', 'html']),
@@ -38,6 +39,7 @@ pass_config = click.make_pass_decorator(Config)
               ' font size. Useful to detect super/subscripts.')
 @click.option('-M', '--margins', nargs=3, default=(1.0, 0.5, 0.1),
               help='PDFMiner char_margin, line_margin and word_margin.')
+@click.option('-q', '--quiet', is_flag=True, help='Suppress warnings.')
 @click.pass_context
 def cli(ctx, *args, **kwargs):
     """Camelot: PDF Table Extraction for Humans"""
@@ -47,7 +49,7 @@ def cli(ctx, *args, **kwargs):
 
 
 @cli.command('lattice')
-@click.option('-T', '--table_area', default=[], multiple=True,
+@click.option('-T', '--table_areas', default=[], multiple=True,
               help='Table areas to process. Example: x1,y1,x2,y2'
               ' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
 @click.option('-back', '--process_background', is_flag=True,
@@ -89,17 +91,19 @@ def lattice(c, *args, **kwargs):
     output = conf.pop('output')
     f = conf.pop('format')
     compress = conf.pop('zip')
+    suppress_warnings = conf.pop('quiet')
     plot_type = kwargs.pop('plot_type')
     filepath = kwargs.pop('filepath')
     kwargs.update(conf)
 
-    table_area = list(kwargs['table_area'])
+    table_areas = list(kwargs['table_areas'])
-    kwargs['table_area'] = None if not table_area else table_area
+    kwargs['table_areas'] = None if not table_areas else table_areas
     copy_text = list(kwargs['copy_text'])
     kwargs['copy_text'] = None if not copy_text else copy_text
     kwargs['shift_text'] = list(kwargs['shift_text'])
 
-    tables = read_pdf(filepath, pages=pages, flavor='lattice', **kwargs)
+    tables = read_pdf(filepath, pages=pages, flavor='lattice',
+                      suppress_warnings=suppress_warnings, **kwargs)
     click.echo('Found {} tables'.format(tables.n))
     if plot_type is not None:
         for table in tables:
@@ -113,7 +117,7 @@ def lattice(c, *args, **kwargs):
 
 
 @cli.command('stream')
-@click.option('-T', '--table_area', default=[], multiple=True,
+@click.option('-T', '--table_areas', default=[], multiple=True,
               help='Table areas to process. Example: x1,y1,x2,y2'
               ' where x1, y1 -> left-top and x2, y2 -> right-bottom.')
 @click.option('-C', '--columns', default=[], multiple=True,
@@ -134,16 +138,18 @@ def stream(c, *args, **kwargs):
     output = conf.pop('output')
     f = conf.pop('format')
     compress = conf.pop('zip')
+    suppress_warnings = conf.pop('quiet')
     plot_type = kwargs.pop('plot_type')
     filepath = kwargs.pop('filepath')
     kwargs.update(conf)
 
-    table_area = list(kwargs['table_area'])
+    table_areas = list(kwargs['table_areas'])
-    kwargs['table_area'] = None if not table_area else table_area
+    kwargs['table_areas'] = None if not table_areas else table_areas
     columns = list(kwargs['columns'])
     kwargs['columns'] = None if not columns else columns
 
-    tables = read_pdf(filepath, pages=pages, flavor='stream', **kwargs)
+    tables = read_pdf(filepath, pages=pages, flavor='stream',
+                      suppress_warnings=suppress_warnings, **kwargs)
     click.echo('Found {} tables'.format(tables.n))
     if plot_type is not None:
         for table in tables:

camelot/handlers.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import os
+import sys
 
 from PyPDF2 import PdfFileReader, PdfFileWriter
 
@@ -21,14 +22,22 @@ class PDFHandler(object):
         Path to PDF file.
     pages : str, optional (default: '1')
         Comma-separated page numbers.
-        Example: 1,3,4 or 1,4-end.
+        Example: '1,3,4' or '1,4-end'.
+    password : str, optional (default: None)
+        Password for decryption.
 
     """
-    def __init__(self, filename, pages='1'):
+    def __init__(self, filename, pages='1', password=None):
         self.filename = filename
-        if not self.filename.endswith('.pdf'):
+        if not filename.lower().endswith('.pdf'):
             raise NotImplementedError("File format not supported")
         self.pages = self._get_pages(self.filename, pages)
+        if password is None:
+            self.password = ''
+        else:
+            self.password = password
+            if sys.version_info[0] < 3:
+                self.password = self.password.encode('ascii')
 
     def _get_pages(self, filename, pages):
         """Converts pages string to list of ints.
@@ -52,6 +61,8 @@ class PDFHandler(object):
             page_numbers.append({'start': 1, 'end': 1})
         else:
             infile = PdfFileReader(open(filename, 'rb'), strict=False)
+            if infile.isEncrypted:
+                infile.decrypt(self.password)
             if pages == 'all':
                 page_numbers.append({'start': 1, 'end': infile.getNumPages()})
             else:
@@ -84,7 +95,7 @@ class PDFHandler(object):
         with open(filename, 'rb') as fileobj:
             infile = PdfFileReader(fileobj, strict=False)
             if infile.isEncrypted:
-                infile.decrypt('')
+                infile.decrypt(self.password)
             fpath = os.path.join(temp, 'page-{0}.pdf'.format(page))
             froot, fext = os.path.splitext(fpath)
             p = infile.getPage(page - 1)
@@ -103,7 +114,7 @@ class PDFHandler(object):
                 os.rename(fpath, fpath_new)
                 infile = PdfFileReader(open(fpath_new, 'rb'), strict=False)
                 if infile.isEncrypted:
-                    infile.decrypt('')
+                    infile.decrypt(self.password)
                 outfile = PdfFileWriter()
                 p = infile.getPage(0)
                 if rotation == 'anticlockwise':

camelot/io.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
+import warnings
 
 from .handlers import PDFHandler
 from .utils import validate_input, remove_extra
 
 
-def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
+def read_pdf(filepath, pages='1', password=None, flavor='lattice',
+             suppress_warnings=False, **kwargs):
     """Read PDF and return extracted tables.
 
     Note: kwargs annotated with ^ can only be used with flavor='stream'
@@ -16,11 +18,15 @@ def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
         Path to PDF file.
     pages : str, optional (default: '1')
         Comma-separated page numbers.
-        Example: 1,3,4 or 1,4-end.
+        Example: '1,3,4' or '1,4-end'.
+    password : str, optional (default: None)
+        Password for decryption.
     flavor : str (default: 'lattice')
         The parsing method to use ('lattice' or 'stream').
         Lattice is used by default.
-    table_area : list, optional (default: None)
+    suppress_warnings : bool, optional (default: False)
+        Prevent warnings from being emitted by Camelot.
+    table_areas : list, optional (default: None)
         List of table area strings of the form x1,y1,x2,y2
         where (x1, y1) -> left-top and (x2, y2) -> right-bottom
         in PDF coordinate space.
@@ -85,8 +91,12 @@ def read_pdf(filepath, pages='1', flavor='lattice', **kwargs):
         raise NotImplementedError("Unknown flavor specified."
                                   " Use either 'lattice' or 'stream'")
 
-    validate_input(kwargs, flavor=flavor)
+    with warnings.catch_warnings():
-    p = PDFHandler(filepath, pages)
+        if suppress_warnings:
-    kwargs = remove_extra(kwargs, flavor=flavor)
+            warnings.simplefilter("ignore")
-    tables = p.parse(flavor=flavor, **kwargs)
+
-    return tables
+        validate_input(kwargs, flavor=flavor)
+        p = PDFHandler(filepath, pages=pages, password=password)
+        kwargs = remove_extra(kwargs, flavor=flavor)
+        tables = p.parse(flavor=flavor, **kwargs)
+        return tables

camelot/parsers/lattice.py

@@ -28,7 +28,7 @@ class Lattice(BaseParser):
 
     Parameters
     ----------
-    table_area : list, optional (default: None)
+    table_areas : list, optional (default: None)
         List of table area strings of the form x1,y1,x2,y2
         where (x1, y1) -> left-top and (x2, y2) -> right-bottom
         in PDF coordinate space.
@@ -76,12 +76,12 @@ class Lattice(BaseParser):
         For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
 
     """
-    def __init__(self, table_area=None, process_background=False,
+    def __init__(self, table_areas=None, process_background=False,
                  line_size_scaling=15, copy_text=None, shift_text=['l', 't'],
                  split_text=False, flag_size=False, line_close_tol=2,
                  joint_close_tol=2, threshold_blocksize=15, threshold_constant=-2,
                  iterations=0, margins=(1.0, 0.5, 0.1), **kwargs):
-        self.table_area = table_area
+        self.table_areas = table_areas
         self.process_background = process_background
         self.line_size_scaling = line_size_scaling
         self.copy_text = copy_text
@@ -174,15 +174,39 @@ class Lattice(BaseParser):
         return t
 
     def _generate_image(self):
-        # TODO: hacky, get rid of ghostscript #96
+        # TODO: get rid of ghostscript #96
-        def get_platform():
+        def get_executable():
             import platform
+            from distutils.spawn import find_executable
 
-            info = {
+            class GhostscriptNotFound(Exception): pass
-                'system': platform.system().lower(),
+
-                'machine': platform.machine().lower()
+            gs = None
-            }
+            system = platform.system().lower()
-            return info
+            try:
+                if system == 'windows':
+                    if find_executable('gswin32c.exe'):
+                        gs = 'gswin32c.exe'
+                    elif find_executable('gswin64c.exe'):
+                        gs = 'gswin64c.exe'
+                    else:
+                        raise ValueError
+                else:
+                    if find_executable('gs'):
+                        gs = 'gs'
+                    elif find_executable('gsc'):
+                        gs = 'gsc'
+                    else:
+                        raise ValueError
+                if 'ghostscript' not in subprocess.check_output(
+                        [gs, '-version']).decode('utf-8').lower():
+                    raise ValueError
+            except ValueError:
+                raise GhostscriptNotFound(
+                    'Please make sure that Ghostscript is installed'
+                    ' and available on the PATH environment variable')
+
+            return gs
 
         self.imagename = ''.join([self.rootname, '.png'])
         gs_call = [
@@ -193,15 +217,9 @@ class Lattice(BaseParser):
             '-r600',
             self.filename
         ]
-        info = get_platform()
+        gs = get_executable()
-        if info['system'] == 'windows':
+        gs_call.insert(0, gs)
-            bit = info['machine'][-2:]
+
-            gs_call.insert(0, 'gswin{}c.exe'.format(bit))
-        else:
-            if 'ghostscript' in subprocess.check_output(['gs', '-version']).decode('utf-8').lower():
-                gs_call.insert(0, 'gs')
-            else:
-                gs_call.insert(0, "gsc")
         subprocess.call(
             gs_call, stdout=open(os.devnull, 'w'),
             stderr=subprocess.STDOUT)
@@ -226,9 +244,9 @@ class Lattice(BaseParser):
             self.threshold, direction='horizontal',
             line_size_scaling=self.line_size_scaling, iterations=self.iterations)
 
-        if self.table_area is not None:
+        if self.table_areas is not None:
             areas = []
-            for area in self.table_area:
+            for area in self.table_areas:
                 x1, y1, x2, y2 = area.split(",")
                 x1 = float(x1)
                 y1 = float(y1)

camelot/parsers/stream.py

@@ -26,7 +26,7 @@ class Stream(BaseParser):
 
     Parameters
     ----------
-    table_area : list, optional (default: None)
+    table_areas : list, optional (default: None)
         List of table area strings of the form x1,y1,x2,y2
         where (x1, y1) -> left-top and (x2, y2) -> right-bottom
         in PDF coordinate space.
@@ -50,10 +50,10 @@ class Stream(BaseParser):
         For more information, refer `PDFMiner docs <https://euske.github.io/pdfminer/>`_.
 
     """
-    def __init__(self, table_area=None, columns=None, split_text=False,
+    def __init__(self, table_areas=None, columns=None, split_text=False,
                  flag_size=False, row_close_tol=2, col_close_tol=0,
                  margins=(1.0, 0.5, 0.1), **kwargs):
-        self.table_area = table_area
+        self.table_areas = table_areas
         self.columns = columns
         self._validate_columns()
         self.split_text = split_text
@@ -241,15 +241,15 @@ class Stream(BaseParser):
         return cols
 
     def _validate_columns(self):
-        if self.table_area is not None and self.columns is not None:
+        if self.table_areas is not None and self.columns is not None:
-            if len(self.table_area) != len(self.columns):
+            if len(self.table_areas) != len(self.columns):
-                raise ValueError("Length of table_area and columns"
+                raise ValueError("Length of table_areas and columns"
                                  " should be equal")
 
     def _generate_table_bbox(self):
-        if self.table_area is not None:
+        if self.table_areas is not None:
             table_bbox = {}
-            for area in self.table_area:
+            for area in self.table_areas:
                 x1, y1, x2, y2 = area.split(",")
                 x1 = float(x1)
                 y1 = float(y1)

docs/conf.py

@@ -63,7 +63,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Camelot'
-copyright = u'2018, Peeply Private Ltd (Singapore)'
+copyright = u'2018, <a href="https://socialcops.com" target="_blank">SocialCops</a>'
 author = u'Vinayak Mehta'
 
 # The version info for the project you're documenting, acts as replacement for

docs/index.rst

@@ -11,6 +11,10 @@ Release v\ |version|. (:ref:`Installation <install>`)
 .. image:: https://travis-ci.org/socialcopsdev/camelot.svg?branch=master
     :target: https://travis-ci.org/socialcopsdev/camelot
 
+.. image:: https://readthedocs.org/projects/camelot-py/badge/?version=master
+    :target: https://camelot-py.readthedocs.io/en/master/
+    :alt: Documentation Status
+
 .. image:: https://codecov.io/github/socialcopsdev/camelot/badge.svg?branch=master&service=github
     :target: https://codecov.io/github/socialcopsdev/camelot?branch=master
 
@@ -25,6 +29,10 @@ Release v\ |version|. (:ref:`Installation <install>`)
 
 **Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
 
+.. note:: You can also check out `Excalibur`_, which is a web interface for Camelot!
+
+.. _Excalibur: https://github.com/camelot-dev/excalibur
+
 ----
 
 **Here's how you can extract tables from PDF files.** Check out the PDF used in this example `here`_.

docs/user/advanced.rst

@@ -149,7 +149,7 @@ Finally, let's plot all line intersections present on the table's PDF page.
 Specify table areas
 -------------------
 
-Since :ref:`Stream <stream>` treats the whole page as a table, `for now`_, it's useful to specify table boundaries in cases such as `these <../_static/pdf/table_areas.pdf>`__. You can :ref:`plot the text <geometry_text>` on this page and note the left-top and right-bottom coordinates of the table.
+Since :ref:`Stream <stream>` treats the whole page as a table, `for now`_, it's useful to specify table boundaries in cases such as `these <../_static/pdf/table_areas.pdf>`__. You can :ref:`plot the text <geometry_text>` on this page and note the top left and bottom right coordinates of the table.
 
 Table areas that you want Camelot to analyze can be passed as a list of comma-separated strings to :meth:`read_pdf() <camelot.read_pdf>`, using the ``table_areas`` keyword argument.
 
@@ -416,4 +416,4 @@ We don't need anything else. Now, let's pass ``copy_text=['v']`` to copy text in
     "3","Odisha","Kalahandi","iii. Food Poisoning","42","0","02/01/14","03/01/14","Under control","..."
     "4","West Bengal","West Medinipur","iv. Acute Diarrhoeal Disease","145","0","04/01/14","05/01/14","Under control","..."
     "4","West Bengal","Birbhum","v.  Food Poisoning","199","0","31/12/13","31/12/13","Under control","..."
-    "4","West Bengal","Howrah","vi. Viral Hepatitis A &E","85","0","26/12/13","27/12/13","Under surveillance","..."
+    "4","West Bengal","Howrah","vi. Viral Hepatitis A &E","85","0","26/12/13","27/12/13","Under surveillance","..."

docs/user/cli.rst

@@ -9,26 +9,28 @@ You can print the help for the interface by typing ``camelot --help`` in your fa
 
 ::
 
-  Usage: camelot [OPTIONS] COMMAND [ARGS]...
+Usage: camelot [OPTIONS] COMMAND [ARGS]...
 
   Camelot: PDF Table Extraction for Humans
 
-  Options:
+Options:
-    --version                       Show the version and exit.
+  --version                       Show the version and exit.
-    -p, --pages TEXT                Comma-separated page numbers. Example: 1,3,4
+  -p, --pages TEXT                Comma-separated page numbers. Example: 1,3,4
-                                    or 1,4-end.
+                                  or 1,4-end.
-    -o, --output TEXT               Output file path.
+  -pw, --password TEXT            Password for decryption.
-    -f, --format [csv|json|excel|html]
+  -o, --output TEXT               Output file path.
-                                    Output file format.
+  -f, --format [csv|json|excel|html]
-    -z, --zip                       Create ZIP archive.
+                                  Output file format.
-    -split, --split_text            Split text that spans across multiple cells.
+  -z, --zip                       Create ZIP archive.
-    -flag, --flag_size              Flag text based on font size. Useful to
+  -split, --split_text            Split text that spans across multiple cells.
-                                    detect super/subscripts.
+  -flag, --flag_size              Flag text based on font size. Useful to
-    -M, --margins <FLOAT FLOAT FLOAT>...
+                                  detect super/subscripts.
-                                    PDFMiner char_margin, line_margin and
+  -M, --margins <FLOAT FLOAT FLOAT>...
-                                    word_margin.
+                                  PDFMiner char_margin, line_margin and
-    --help                          Show this message and exit.
+                                  word_margin.
+  -q, --quiet                     Suppress warnings.
+  --help                          Show this message and exit.
 
-  Commands:
+Commands:
-    lattice  Use lines between text to parse the table.
+  lattice  Use lines between text to parse the table.
-    stream   Use spaces between text to parse the table.
+  stream   Use spaces between text to parse the table.

docs/user/install.rst

@@ -3,14 +3,36 @@
 Installation of Camelot
 =======================
 
-This part of the documentation covers how to install Camelot. First, you'll need to install the dependencies, which include `Tkinter`_ and `ghostscript`_.
+This part of the documentation covers how to install Camelot.
+
+Using conda
+-----------
+
+The easiest way to install Camelot is to install it with `conda`_, which is the package manager that the `Anaconda`_ distribution is built upon.
+
+First, let's add the `conda-forge`_ channel to conda's config::
+
+    $ conda config --add channels conda-forge
+
+Now, you can simply use conda to install Camelot::
+
+    $ conda install -c camelot-dev camelot-py
+
+.. note:: Camelot is available for Python 2.7, 3.5 and 3.6 on Linux, macOS and Windows. For Windows, you will need to install ghostscript which you can get from their `downloads page`_.
+
+.. _conda: https://conda.io/docs/
+.. _Anaconda: http://docs.continuum.io/anaconda/
+.. _downloads page: https://www.ghostscript.com/download/gsdnld.html
+.. _conda-forge: https://conda-forge.org/
+
+Using pip
+---------
+
+First, you'll need to install the dependencies, which include `Tkinter`_ and `ghostscript`_.
 
 .. _Tkinter: https://wiki.python.org/moin/TkInter
 .. _ghostscript: https://www.ghostscript.com
 
-Install the dependencies
-------------------------
-
 These can be installed using your system's package manager. You can run one of the following, based on your OS.
 
 For Ubuntu
@@ -76,17 +98,14 @@ Or for Windows 32-bit::
 
 If you have ghostscript, you should see the ghostscript version and copyright information.
 
-$ pip install camelot-py[all]
+Finally, you can use pip to install Camelot::
------------------------------
-
-After installing the dependencies, you can simply use pip to install Camelot::
 
     $ pip install camelot-py[all]
 
-Get the source code
+From the source code
--------------------
+--------------------
 
-Alternatively, you can install from the source by:
+After `installing the dependencies`_, you can install from the source by:
 
 1. Cloning the GitHub repository.
 ::
@@ -97,4 +116,6 @@ Alternatively, you can install from the source by:
 ::
 
     $ cd camelot
-    $ pip install ".[all]"
+    $ pip install ".[all]"
+
+.. _installing the dependencies: https://camelot-py.readthedocs.io/en/master/user/install.html#using-pip

docs/user/quickstart.rst

@@ -87,6 +87,28 @@ By default, Camelot only uses the first page of the PDF to extract tables. To sp
 
 The ``pages`` keyword argument accepts pages as comma-separated string of page numbers. You can also specify page ranges — for example, ``pages=1,4-10,20-30`` or ``pages=1,4-10,20-end``.
 
-------------------------
+Reading encrypted PDFs
+----------------------
 
-Ready for more? Check out the :ref:`advanced <advanced>` section.
+To extract tables from encrypted PDF files you must provide a password when calling :meth:`read_pdf() <camelot.read_pdf>`.
+
+::
+
+    >>> tables = camelot.read_pdf('foo.pdf', password='userpass')
+    >>> tables
+    <TableList n=1>
+
+Currently Camelot only supports PDFs encrypted with ASCII passwords and algorithm `code 1 or 2`_. An exception is thrown if the PDF cannot be read. This may be due to no password being provided, an incorrect password, or an unsupported encryption algorithm.
+
+Further encryption support may be added in future, however in the meantime if your PDF files are using unsupported encryption algorithms you are advised to remove encryption before calling :meth:`read_pdf() <camelot.read_pdf>`. This can been successfully achieved with third-party tools such as `QPDF`_.
+
+::
+
+    $ qpdf --password=<PASSWORD> --decrypt input.pdf output.pdf
+
+.. _code 1 or 2: https://github.com/mstamy2/PyPDF2/issues/378
+.. _QPDF: https://www.github.com/qpdf/qpdf
+
+----
+
+Ready for more? Check out the :ref:`advanced <advanced>` section.

requirements.txt

@@ -0,0 +1,8 @@
+click>=6.7
+matplotlib>=2.2.3
+numpy>=1.13.3
+opencv-python>=3.4.2.17
+openpyxl>=2.5.8
+pandas>=0.23.4
+pdfminer.six>=20170720
+PyPDF2>=1.26.0

setup.py

@@ -14,25 +14,25 @@ with open('README.md', 'r') as f:
 
 
 requires = [
-    'click==6.7',
+    'click>=6.7',
-    'matplotlib==2.2.3',
+    'matplotlib>=2.2.3',
-    'numpy==1.15.2',
+    'numpy>=1.13.3',
-    'openpyxl==2.5.8',
+    'openpyxl>=2.5.8',
-    'pandas==0.23.4',
+    'pandas>=0.23.4',
-    'pdfminer.six==20170720',
+    'pdfminer.six>=20170720',
-    'PyPDF2==1.26.0'
+    'PyPDF2>=1.26.0'
 ]
 
 all_requires = [
-    'opencv-python==3.4.2.17'
+    'opencv-python>=3.4.2.17'
 ]
 
 dev_requires = [
-    'codecov==2.0.15',
+    'codecov>=2.0.15',
-    'pytest==3.8.0',
+    'pytest>=3.8.0',
-    'pytest-cov==2.6.0',
+    'pytest-cov>=2.6.0',
-    'pytest-runner==4.2',
+    'pytest-runner>=4.2',
-    'Sphinx==1.7.9'
+    'Sphinx>=1.7.9'
 ]
 dev_requires = dev_requires + all_requires
 

tests/data.py

@@ -81,7 +81,7 @@ data_stream_table_rotated = [
     ["", "", "", "", "", "", "", "", "54", "", "", "", "", "", "", "", "", ""]
 ]
 
-data_stream_table_area = [
+data_stream_table_areas = [
     ["", "One Withholding"],
     ["Payroll Period", "Allowance"],
     ["Weekly", "$71.15"],
@@ -261,7 +261,7 @@ data_lattice_table_rotated = [
     ["Pooled", "38742", "53618", "60601", "86898", "4459", "21918", "27041", "14312", "18519"]
 ]
 
-data_lattice_table_area = [
+data_lattice_table_areas = [
     ["", "", "", "", "", "", "", "", ""],
     ["State", "n", "Literacy Status", "", "", "", "", "", ""],
     ["", "", "Illiterate", "Read & Write", "1-4 std.", "5-8 std.", "9-12 std.", "College", ""],

tests/test_cli.py

@@ -1,79 +1,117 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
-
+
-import os
+import os
-
+
-from click.testing import CliRunner
+from click.testing import CliRunner
-
+
-from camelot.cli import cli
+from camelot.cli import cli
-from camelot.utils import TemporaryDirectory
+from camelot.utils import TemporaryDirectory
-
+
-
+
-testdir = os.path.dirname(os.path.abspath(__file__))
+testdir = os.path.dirname(os.path.abspath(__file__))
-testdir = os.path.join(testdir, 'files')
+testdir = os.path.join(testdir, 'files')
-
+
-
+
-def test_cli_lattice():
+def test_cli_lattice():
-    with TemporaryDirectory() as tempdir:
+    with TemporaryDirectory() as tempdir:
-        infile = os.path.join(testdir, 'foo.pdf')
+        infile = os.path.join(testdir, 'foo.pdf')
-        outfile = os.path.join(tempdir, 'foo.csv')
+        outfile = os.path.join(tempdir, 'foo.csv')
-        runner = CliRunner()
+        runner = CliRunner()
-        result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
+        result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
-                                     'lattice', infile])
+                                     'lattice', infile])
-        assert result.exit_code == 0
+        assert result.exit_code == 0
-        assert result.output == 'Found 1 tables\n'
+        assert result.output == 'Found 1 tables\n'
-
+
-        result = runner.invoke(cli, ['--format', 'csv',
+        result = runner.invoke(cli, ['--format', 'csv',
-                                     'lattice', infile])
+                                     'lattice', infile])
-        output_error = 'Error: Please specify output file path using --output'
+        output_error = 'Error: Please specify output file path using --output'
-        assert output_error in result.output
+        assert output_error in result.output
-
+
-        result = runner.invoke(cli, ['--output', outfile,
+        result = runner.invoke(cli, ['--output', outfile,
-                                     'lattice', infile])
+                                     'lattice', infile])
-        format_error = 'Please specify output file format using --format'
+        format_error = 'Please specify output file format using --format'
-        assert format_error in result.output
+        assert format_error in result.output
-
+
-
+
-def test_cli_stream():
+def test_cli_stream():
-    with TemporaryDirectory() as tempdir:
+    with TemporaryDirectory() as tempdir:
-        infile = os.path.join(testdir, 'budget.pdf')
+        infile = os.path.join(testdir, 'budget.pdf')
-        outfile = os.path.join(tempdir, 'budget.csv')
+        outfile = os.path.join(tempdir, 'budget.csv')
-        runner = CliRunner()
+        runner = CliRunner()
-        result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
+        result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
-                                     'stream', infile])
+                                     'stream', infile])
-        assert result.exit_code == 0
+        assert result.exit_code == 0
-        assert result.output == 'Found 1 tables\n'
+        assert result.output == 'Found 1 tables\n'
-
+
-        result = runner.invoke(cli, ['--format', 'csv', 'stream', infile])
+        result = runner.invoke(cli, ['--format', 'csv', 'stream', infile])
-        output_error = 'Error: Please specify output file path using --output'
+        output_error = 'Error: Please specify output file path using --output'
-        assert output_error in result.output
+        assert output_error in result.output
-
+
-        result = runner.invoke(cli, ['--output', outfile, 'stream', infile])
+        result = runner.invoke(cli, ['--output', outfile, 'stream', infile])
-        format_error = 'Please specify output file format using --format'
+        format_error = 'Please specify output file format using --format'
-        assert format_error in result.output
+        assert format_error in result.output
-
+
-
+
-def test_cli_output_format():
+def test_cli_password():
-    with TemporaryDirectory() as tempdir:
+    with TemporaryDirectory() as tempdir:
-        infile = os.path.join(testdir, 'health.pdf')
+        infile = os.path.join(testdir, 'health_protected.pdf')
-        outfile = os.path.join(tempdir, 'health.{}')
+        outfile = os.path.join(tempdir, 'health_protected.csv')
-        runner = CliRunner()
+        runner = CliRunner()
-
+        result = runner.invoke(cli, ['--password', 'userpass',
-        # json
+                                     '--format', 'csv', '--output', outfile,
-        result = runner.invoke(cli, ['--format', 'json', '--output', outfile.format('json'),
+                                     'stream', infile])
-                                     'stream', infile])
+        assert result.exit_code == 0
-        assert result.exit_code == 0
+        assert result.output == 'Found 1 tables\n'
-
+
-        # excel
+        output_error = 'file has not been decrypted'
-        result = runner.invoke(cli, ['--format', 'excel', '--output', outfile.format('xlsx'),
+        # no password
-                                     'stream', infile])
+        result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
-        assert result.exit_code == 0
+                                     'stream', infile])
-
+        assert output_error in str(result.exception)
-        # html
+
-        result = runner.invoke(cli, ['--format', 'html', '--output', outfile.format('html'),
+        # bad password
-                                     'stream', infile])
+        result = runner.invoke(cli, ['--password', 'wrongpass',
-        assert result.exit_code == 0
+                                     '--format', 'csv', '--output', outfile,
-
+                                     'stream', infile])
-        # zip
+        assert output_error in str(result.exception)
-        result = runner.invoke(cli, ['--zip', '--format', 'csv', '--output', outfile.format('csv'),
+
-                                     'stream', infile])
+
-        assert result.exit_code == 0
+def test_cli_output_format():
+    with TemporaryDirectory() as tempdir:
+        infile = os.path.join(testdir, 'health.pdf')
+        outfile = os.path.join(tempdir, 'health.{}')
+        runner = CliRunner()
+
+        # json
+        result = runner.invoke(cli, ['--format', 'json', '--output', outfile.format('json'),
+                                     'stream', infile])
+        assert result.exit_code == 0
+
+        # excel
+        result = runner.invoke(cli, ['--format', 'excel', '--output', outfile.format('xlsx'),
+                                     'stream', infile])
+        assert result.exit_code == 0
+
+        # html
+        result = runner.invoke(cli, ['--format', 'html', '--output', outfile.format('html'),
+                                     'stream', infile])
+        assert result.exit_code == 0
+
+        # zip
+        result = runner.invoke(cli, ['--zip', '--format', 'csv', '--output', outfile.format('csv'),
+                                     'stream', infile])
+        assert result.exit_code == 0
+
+def test_cli_quiet():
+    with TemporaryDirectory() as tempdir:
+        infile = os.path.join(testdir, 'blank.pdf')
+        outfile = os.path.join(tempdir, 'blank.csv')
+        runner = CliRunner()
+
+        result = runner.invoke(cli, ['--format', 'csv', '--output', outfile,
+                                     'stream', infile])
+        assert 'No tables found on page-1' in result.output
+
+        result = runner.invoke(cli, ['--quiet', '--format', 'csv',
+                                     '--output', outfile, 'stream', infile])
+        assert 'No tables found on page-1' not in result.output

tests/test_common.py

@@ -25,6 +25,17 @@ def test_parsing_report():
     assert tables[0].parsing_report == parsing_report
 
 
+def test_password():
+    df = pd.DataFrame(data_stream)
+
+    filename = os.path.join(testdir, "health_protected.pdf")
+    tables = camelot.read_pdf(filename, password="ownerpass", flavor="stream")
+    assert df.equals(tables[0].df)
+
+    tables = camelot.read_pdf(filename, password="userpass", flavor="stream")
+    assert df.equals(tables[0].df)
+
+
 def test_stream():
     df = pd.DataFrame(data_stream)
 
@@ -45,11 +56,11 @@ def test_stream_table_rotated():
     assert df.equals(tables[0].df)
 
 
-def test_stream_table_area():
+def test_stream_table_areas():
-    df = pd.DataFrame(data_stream_table_area)
+    df = pd.DataFrame(data_stream_table_areas)
 
     filename = os.path.join(testdir, "tabula/us-007.pdf")
-    tables = camelot.read_pdf(filename, flavor="stream", table_area=["320,500,573,335"])
+    tables = camelot.read_pdf(filename, flavor="stream", table_areas=["320,500,573,335"])
     assert df.equals(tables[0].df)
 
 
@@ -100,11 +111,11 @@ def test_lattice_table_rotated():
     assert df.equals(tables[0].df)
 
 
-def test_lattice_table_area():
+def test_lattice_table_areas():
-    df = pd.DataFrame(data_lattice_table_area)
+    df = pd.DataFrame(data_lattice_table_areas)
 
     filename = os.path.join(testdir, "twotables_2.pdf")
-    tables = camelot.read_pdf(filename, table_area=["80,693,535,448"])
+    tables = camelot.read_pdf(filename, table_areas=["80,693,535,448"])
     assert df.equals(tables[0].df)
 
 
@@ -139,7 +150,7 @@ def test_lattice_shift_text():
     tables = camelot.read_pdf(filename, line_size_scaling=40, shift_text=['r', 'b'])
     assert df_rb.equals(tables[0].df)
 
-    
+
 def test_repr():
     filename = os.path.join(testdir, "foo.pdf")
     tables = camelot.read_pdf(filename)

tests/test_errors.py

@@ -34,20 +34,58 @@ def test_unsupported_format():
 
 
 def test_stream_equal_length():
-    message = ("Length of table_area and columns"
+    message = ("Length of table_areas and columns"
                " should be equal")
     with pytest.raises(ValueError, message=message):
         tables = camelot.read_pdf(filename, flavor='stream',
-            table_area=['10,20,30,40'], columns=['10,20,30,40', '10,20,30,40'])
+            table_areas=['10,20,30,40'], columns=['10,20,30,40', '10,20,30,40'])
 
 
 def test_no_tables_found():
     filename = os.path.join(testdir, 'blank.pdf')
-    # TODO: use pytest.warns
     with warnings.catch_warnings():
         warnings.simplefilter('error')
-        try:
+        with pytest.raises(UserWarning) as e:
             tables = camelot.read_pdf(filename)
-        except Exception as e:
+        assert str(e.value) == 'No tables found on page-1'
-            assert type(e).__name__ == 'UserWarning'
+
-            assert str(e) == 'No tables found on page-1'
+
+def test_no_tables_found_warnings_suppressed():
+    filename = os.path.join(testdir, 'blank.pdf')
+    with warnings.catch_warnings():
+        # the test should fail if any warning is thrown
+        warnings.simplefilter('error')
+        try:
+            tables = camelot.read_pdf(filename, suppress_warnings=True)
+        except Warning as e:
+            warning_text = str(e)
+            pytest.fail('Unexpected warning: {}'.format(warning_text))
+
+
+def test_ghostscript_not_found(monkeypatch):
+    import distutils
+
+    def _find_executable_patch(arg):
+        return ''
+
+    monkeypatch.setattr(distutils.spawn, 'find_executable', _find_executable_patch)
+
+    message = ('Please make sure that Ghostscript is installed and available'
+               ' on the PATH environment variable')
+    filename = os.path.join(testdir, 'foo.pdf')
+    with pytest.raises(Exception, message=message):
+        tables = camelot.read_pdf(filename)
+
+
+def test_no_password():
+    filename = os.path.join(testdir, 'health_protected.pdf')
+    message = 'file has not been decrypted'
+    with pytest.raises(Exception, message=message):
+        tables = camelot.read_pdf(filename)
+
+
+def test_bad_password():
+    filename = os.path.join(testdir, 'health_protected.pdf')
+    message = 'file has not been decrypted'
+    with pytest.raises(Exception, message=message):
+        tables = camelot.read_pdf(filename, password='wrongpass')