Fix #21

.deepsource.toml

@@ -1,12 +0,0 @@
-version = 1
-
-exclude_patterns = [
-  "camelot/ext/**"
-]
-
-[[analyzers]]
-name = "python"
-enabled = true
-
-[analyzers.meta]
-runtime_version = "3.x.x"

.github/FUNDING.yml

@@ -1 +0,0 @@
-open_collective: camelot

.travis.yml

@@ -8,6 +8,14 @@ install:
   - make install
 jobs:
   include:
+    - stage: test
+      script:
+        - make test
+      python: '2.7'
+    - stage: test
+      script:
+        - make test
+      python: '3.5'
     - stage: test
       script:
         - make test
@@ -17,13 +25,8 @@ jobs:
         - make test
       python: '3.7'
       dist: xenial
-    - stage: test
-      script:
-        - make test
-      python: '3.8'
-      dist: xenial
     - stage: coverage
-      python: '3.8'
+      python: '3.6'
       script:
         - make test
         - codecov --verbose

HISTORY.md

@@ -4,18 +4,6 @@ Release History
 master
 ------
 
-0.8.0 (2020-05-24)
-------------------
-
-**Improvements**
-
-* Drop Python 2 support!
-    * Remove Python 2.7 and 3.5 support.
-    * Replace all instances of `.format` with f-strings.
-    * Remove all `__future__` imports.
-    * Fix HTTP 403 forbidden exception in read_pdf(url) and remove Python 2 urllib support.
-    * Fix test data.
-
 **Bugfixes**
 
 * Fix library discovery on Windows. [#32](https://github.com/camelot-dev/camelot/pull/32) by [KOLANICH](https://github.com/KOLANICH).

README.md

@@ -7,7 +7,7 @@
 [![Build Status](https://travis-ci.org/camelot-dev/camelot.svg?branch=master)](https://travis-ci.org/camelot-dev/camelot) [![Documentation Status](https://readthedocs.org/projects/camelot-py/badge/?version=master)](https://camelot-py.readthedocs.io/en/master/)
  [![codecov.io](https://codecov.io/github/camelot-dev/camelot/badge.svg?branch=master&service=github)](https://codecov.io/github/camelot-dev/camelot?branch=master)
  [![image](https://img.shields.io/pypi/v/camelot-py.svg)](https://pypi.org/project/camelot-py/) [![image](https://img.shields.io/pypi/l/camelot-py.svg)](https://pypi.org/project/camelot-py/) [![image](https://img.shields.io/pypi/pyversions/camelot-py.svg)](https://pypi.org/project/camelot-py/) [![Gitter chat](https://badges.gitter.im/camelot-dev/Lobby.png)](https://gitter.im/camelot-dev/Lobby)
-[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![image](https://img.shields.io/badge/continous%20quality-deepsource-lightgrey)](https://deepsource.io/gh/camelot-dev/camelot/?ref=repository-badge)
+[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)
 
 
 **Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
@@ -124,10 +124,6 @@ After installation, you can run tests using:
 $ python setup.py test
 </pre>
 
-## Wrappers
-
-- [camelot-php](https://github.com/randomstate/camelot-php) provides a [PHP](https://www.php.net/) wrapper on Camelot.
-
 ## Versioning
 
 Camelot uses [Semantic Versioning](https://semver.org/). For the available versions, see the tags on this repository. For the changelog, you can check out [HISTORY.md](https://github.com/camelot-dev/camelot/blob/master/HISTORY.md).

camelot/__main__.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
+
 
 __all__ = ("main",)
 

camelot/__version__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-VERSION = (0, 8, 0)
+VERSION = (0, 7, 3)
 PRERELEASE = None  # alpha, beta or rc
 REVISION = None
 
@@ -8,9 +8,9 @@ REVISION = None
 def generate_version(version, prerelease=None, revision=None):
     version_parts = [".".join(map(str, version))]
     if prerelease is not None:
-        version_parts.append(f"-{prerelease}")
+        version_parts.append("-{}".format(prerelease))
     if revision is not None:
-        version_parts.append(f".{revision}")
+        version_parts.append(".{}".format(revision))
     return "".join(version_parts)
 
 

camelot/cli.py

@@ -204,7 +204,7 @@ def lattice(c, *args, **kwargs):
     tables = read_pdf(
         filepath, pages=pages, flavor="lattice", suppress_stdout=quiet, **kwargs
     )
-    click.echo(f"Found {tables.n} tables")
+    click.echo("Found {} tables".format(tables.n))
     if plot_type is not None:
         for table in tables:
             plot(table, kind=plot_type)
@@ -295,7 +295,7 @@ def stream(c, *args, **kwargs):
     tables = read_pdf(
         filepath, pages=pages, flavor="stream", suppress_stdout=quiet, **kwargs
     )
-    click.echo(f"Found {tables.n} tables")
+    click.echo("Found {} tables".format(tables.n))
     if plot_type is not None:
         for table in tables:
             plot(table, kind=plot_type)

camelot/core.py

@@ -52,10 +52,13 @@ class TextEdge(object):
         self.is_valid = False
 
     def __repr__(self):
-        x = round(self.x, 2)
-        y0 = round(self.y0, 2)
-        y1 = round(self.y1, 2)
-        return f"<TextEdge x={x} y0={y0} y1={y1} align={self.align} valid={self.is_valid}>"
+        return "<TextEdge x={} y0={} y1={} align={} valid={}>".format(
+            round(self.x, 2),
+            round(self.y0, 2),
+            round(self.y1, 2),
+            self.align,
+            self.is_valid,
+        )
 
     def update_coords(self, x, y0, edge_tol=50):
         """Updates the text edge's x and bottom y coordinates and sets
@@ -288,11 +291,9 @@ class Cell(object):
         self._text = ""
 
     def __repr__(self):
-        x1 = round(self.x1, 2)
-        y1 = round(self.y1, 2)
-        x2 = round(self.x2, 2)
-        y2 = round(self.y2, 2)
-        return f"<Cell x1={x1} y1={y1} x2={x2} y2={y2}>"
+        return "<Cell x1={} y1={} x2={} y2={}>".format(
+            round(self.x1, 2), round(self.y1, 2), round(self.x2, 2), round(self.y2, 2)
+        )
 
     @property
     def text(self):
@@ -350,7 +351,7 @@ class Table(object):
         self.page = None
 
     def __repr__(self):
-        return f"<{self.__class__.__name__} shape={self.shape}>"
+        return "<{} shape={}>".format(self.__class__.__name__, self.shape)
 
     def __lt__(self, other):
         if self.page == other.page:
@@ -611,7 +612,7 @@ class Table(object):
 
         """
         kw = {
-            "sheet_name": f"page-{self.page}-table-{self.order}",
+            "sheet_name": "page-{}-table-{}".format(self.page, self.order),
             "encoding": "utf-8",
         }
         kw.update(kwargs)
@@ -648,7 +649,7 @@ class Table(object):
         kw = {"if_exists": "replace", "index": False}
         kw.update(kwargs)
         conn = sqlite3.connect(path)
-        table_name = f"page-{self.page}-table-{self.order}"
+        table_name = "page-{}-table-{}".format(self.page, self.order)
         self.df.to_sql(table_name, conn, **kw)
         conn.commit()
         conn.close()
@@ -669,7 +670,7 @@ class TableList(object):
         self._tables = tables
 
     def __repr__(self):
-        return f"<{self.__class__.__name__} n={self.n}>"
+        return "<{} n={}>".format(self.__class__.__name__, self.n)
 
     def __len__(self):
         return len(self._tables)
@@ -679,7 +680,7 @@ class TableList(object):
 
     @staticmethod
     def _format_func(table, f):
-        return getattr(table, f"to_{f}")
+        return getattr(table, "to_{}".format(f))
 
     @property
     def n(self):
@@ -690,7 +691,9 @@ class TableList(object):
         root = kwargs.get("root")
         ext = kwargs.get("ext")
         for table in self._tables:
-            filename = f"{root}-page-{table.page}-table-{table.order}{ext}"
+            filename = os.path.join(
+                "{}-page-{}-table-{}{}".format(root, table.page, table.order, ext)
+            )
             filepath = os.path.join(dirname, filename)
             to_format = self._format_func(table, f)
             to_format(filepath)
@@ -703,7 +706,9 @@ class TableList(object):
         zipname = os.path.join(os.path.dirname(path), root) + ".zip"
         with zipfile.ZipFile(zipname, "w", allowZip64=True) as z:
             for table in self._tables:
-                filename = f"{root}-page-{table.page}-table-{table.order}{ext}"
+                filename = os.path.join(
+                    "{}-page-{}-table-{}{}".format(root, table.page, table.order, ext)
+                )
                 filepath = os.path.join(dirname, filename)
                 z.write(filepath, os.path.basename(filepath))
 
@@ -736,7 +741,7 @@ class TableList(object):
             filepath = os.path.join(dirname, basename)
             writer = pd.ExcelWriter(filepath)
             for table in self._tables:
-                sheet_name = f"page-{table.page}-table-{table.order}"
+                sheet_name = "page-{}-table-{}".format(table.page, table.order)
                 table.df.to_excel(writer, sheet_name=sheet_name, encoding="utf-8")
             writer.save()
             if compress:

camelot/ext/ghostscript/_gsprint.py

@@ -81,7 +81,6 @@ def delete_instance(instance):
     """
     return libgs.gsapi_delete_instance(instance)
 
-
 if sys.platform == "win32":
     c_stdstream_call_t = WINFUNCTYPE(c_int, gs_main_instance, POINTER(c_char), c_int)
 else:
@@ -248,10 +247,7 @@ if sys.platform == "win32":
     libgs = __win32_finddll()
     if not libgs:
         import ctypes.util
-
-        libgs = ctypes.util.find_library(
-            "".join(("gsdll", str(ctypes.sizeof(ctypes.c_voidp) * 8), ".dll"))
-        )  # finds in %PATH%
+        libgs = ctypes.util.find_library("".join(("gsdll", str(ctypes.sizeof(ctypes.c_voidp) * 8), ".dll"))) # finds in %PATH%
     if not libgs:
         raise RuntimeError("Please make sure that Ghostscript is installed")
     libgs = windll.LoadLibrary(libgs)

camelot/handlers.py

@@ -2,6 +2,7 @@
 
 import os
 import sys
+import copy
 
 from PyPDF2 import PdfFileReader, PdfFileWriter
 
@@ -89,51 +90,56 @@ class PDFHandler(object):
             P.extend(range(p["start"], p["end"] + 1))
         return sorted(set(P))
 
-    def _save_page(self, filepath, page, temp):
+    def _save_pages(self, filepath, pages, temp):
         """Saves specified page from PDF into a temporary directory.
 
         Parameters
         ----------
         filepath : str
             Filepath or URL of the PDF file.
-        page : int
-            Page number.
+        pages : int
+            Page numbers.
         temp : str
             Tmp directory.
 
         """
         with open(filepath, "rb") as fileobj:
-            infile = PdfFileReader(fileobj, strict=False)
-            if infile.isEncrypted:
-                infile.decrypt(self.password)
-            fpath = os.path.join(temp, f"page-{page}.pdf")
-            froot, fext = os.path.splitext(fpath)
-            p = infile.getPage(page - 1)
-            outfile = PdfFileWriter()
-            outfile.addPage(p)
-            with open(fpath, "wb") as f:
-                outfile.write(f)
-            layout, dim = get_page_layout(fpath)
-            # fix rotated PDF
-            chars = get_text_objects(layout, ltype="char")
-            horizontal_text = get_text_objects(layout, ltype="horizontal_text")
-            vertical_text = get_text_objects(layout, ltype="vertical_text")
-            rotation = get_rotation(chars, horizontal_text, vertical_text)
-            if rotation != "":
-                fpath_new = "".join([froot.replace("page", "p"), "_rotated", fext])
-                os.rename(fpath, fpath_new)
-                infile = PdfFileReader(open(fpath_new, "rb"), strict=False)
-                if infile.isEncrypted:
-                    infile.decrypt(self.password)
+            infile_original = PdfFileReader(fileobj, strict=False)
+            if infile_original.isEncrypted:
+                infile_original.decrypt(self.password)
+
+            for page in pages:
+                # Ensure PdfFileReader object is unmodified
+                infile = copy.copy(infile_original)
+                fpath = os.path.join(temp, 'page-{0}.pdf'.format(page))
+                froot, fext = os.path.splitext(fpath)
+                p = infile.getPage(page - 1)
                 outfile = PdfFileWriter()
-                p = infile.getPage(0)
-                if rotation == "anticlockwise":
-                    p.rotateClockwise(90)
-                elif rotation == "clockwise":
-                    p.rotateCounterClockwise(90)
                 outfile.addPage(p)
-                with open(fpath, "wb") as f:
-                    outfile.write(f)
+                with open(fpath, 'wb') as f:
+                     outfile.write(f)
+
+                # Orient rotated pages correctly
+                layout, dim = get_page_layout(fpath)
+                chars = get_text_objects(layout, ltype="char")
+                horizontal_text = get_text_objects(layout, ltype="horizontal_text")
+                vertical_text = get_text_objects(layout, ltype="vertical_text")
+                rotation = get_rotation(chars, horizontal_text, vertical_text)
+                if rotation != '':
+                    fpath_new = ''.join([froot.replace('page', 'p'), '_rotated', fext])
+                    os.rename(fpath, fpath_new)
+                    infile = PdfFileReader(open(fpath_new, 'rb'), strict=False)
+                    if infile.isEncrypted:
+                        infile.decrypt(self.password)
+                    outfile = PdfFileWriter()
+                    p = infile.getPage(0)
+                    if rotation == 'anticlockwise':
+                        p.rotateClockwise(90)
+                    elif rotation == 'clockwise':
+                        p.rotateCounterClockwise(90)
+                    outfile.addPage(p)
+                    with open(fpath, 'wb') as f:
+                        outfile.write(f)
 
     def parse(
         self, flavor="lattice", suppress_stdout=False, layout_kwargs={}, **kwargs
@@ -161,10 +167,9 @@ class PDFHandler(object):
         """
         tables = []
         with TemporaryDirectory() as tempdir:
-            for p in self.pages:
-                self._save_page(self.filepath, p, tempdir)
+            self._save_pages(self.filepath, self.pages, tempdir)
             pages = [
-                os.path.join(tempdir, f"page-{p}.pdf") for p in self.pages
+                os.path.join(tempdir, "page-{0}.pdf".format(p)) for p in self.pages
             ]
             parser = Lattice(**kwargs) if flavor == "lattice" else Stream(**kwargs)
             for p in pages:

camelot/image_processing.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import division
+
 import cv2
 import numpy as np
 

camelot/parsers/lattice.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import division
 import os
 import sys
 import copy

camelot/parsers/stream.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import division
 import os
 import logging
 import warnings
@@ -357,7 +358,7 @@ class Stream(BaseParser):
                     ncols = max(set(elements), key=elements.count)
                 else:
                     warnings.warn(
-                        f"No tables found in table area {table_idx + 1}"
+                        "No tables found in table area {}".format(table_idx + 1)
                     )
             cols = [(t.x0, t.x1) for r in rows_grouped if len(r) == ncols for t in r]
             cols = self._merge_columns(sorted(cols), column_tol=self.column_tol)
@@ -432,19 +433,19 @@ class Stream(BaseParser):
 
     def extract_tables(self, filename, suppress_stdout=False, layout_kwargs={}):
         self._generate_layout(filename, layout_kwargs)
-        base_filename = os.path.basename(self.rootname)
-
         if not suppress_stdout:
-            logger.info(f"Processing {base_filename}")
+            logger.info("Processing {}".format(os.path.basename(self.rootname)))
 
         if not self.horizontal_text:
             if self.images:
                 warnings.warn(
-                    f"{base_filename} is image-based, camelot only works on"
-                    " text-based pages."
+                    "{} is image-based, camelot only works on"
+                    " text-based pages.".format(os.path.basename(self.rootname))
                 )
             else:
-                warnings.warn(f"No tables found on {base_filename}")
+                warnings.warn(
+                    "No tables found on {}".format(os.path.basename(self.rootname))
+                )
             return []
 
         self._generate_table_bbox()

camelot/plotting.py

@@ -35,11 +35,11 @@ class PlotMethods(object):
 
         if table.flavor == "lattice" and kind in ["textedge"]:
             raise NotImplementedError(
-                f"Lattice flavor does not support kind='{kind}'"
+                "Lattice flavor does not support kind='{}'".format(kind)
             )
         elif table.flavor == "stream" and kind in ["joint", "line"]:
             raise NotImplementedError(
-                f"Stream flavor does not support kind='{kind}'"
+                "Stream flavor does not support kind='{}'".format(kind)
             )
 
         plot_method = getattr(self, kind)

camelot/utils.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import division
 
-import os
 import re
+import os
+import sys
 import random
 import shutil
 import string
@@ -27,9 +29,16 @@ from pdfminer.layout import (
     LTImage,
 )
 
-from urllib.request import Request, urlopen
-from urllib.parse import urlparse as parse_url
-from urllib.parse import uses_relative, uses_netloc, uses_params
+
+PY3 = sys.version_info[0] >= 3
+if PY3:
+    from urllib.request import urlopen
+    from urllib.parse import urlparse as parse_url
+    from urllib.parse import uses_relative, uses_netloc, uses_params
+else:
+    from urllib2 import urlopen
+    from urlparse import urlparse as parse_url
+    from urlparse import uses_relative, uses_netloc, uses_params
 
 
 _VALID_URLS = set(uses_relative + uses_netloc + uses_params)
@@ -79,12 +88,13 @@ def download_url(url):
         Temporary filepath.
 
     """
-    filename = f"{random_string(6)}.pdf"
+    filename = "{}.pdf".format(random_string(6))
     with tempfile.NamedTemporaryFile("wb", delete=False) as f:
-        headers = {"User-Agent": "Mozilla/5.0"}
-        request = Request(url, None, headers)
-        obj = urlopen(request)
-        content_type = obj.info().get_content_type()
+        obj = urlopen(url)
+        if PY3:
+            content_type = obj.info().get_content_type()
+        else:
+            content_type = obj.info().getheader("Content-Type")
         if content_type != "application/pdf":
             raise NotImplementedError("File format not supported")
         f.write(obj.read())
@@ -113,7 +123,9 @@ def validate_input(kwargs, flavor="lattice"):
         isec = set(parser_kwargs).intersection(set(input_kwargs.keys()))
         if isec:
             raise ValueError(
-                f"{','.join(sorted(isec))} cannot be used with flavor='{flavor}'"
+                "{} cannot be used with flavor='{}'".format(
+                    ",".join(sorted(isec)), flavor
+                )
             )
 
     if flavor == "lattice":
@@ -411,7 +423,7 @@ def text_strip(text, strip=""):
         return text
 
     stripped = re.sub(
-        fr"[{''.join(map(re.escape, strip))}]", "", text, re.UNICODE
+        r"[{}]".format("".join(map(re.escape, strip))), "", text, re.UNICODE
     )
     return stripped
 
@@ -648,7 +660,9 @@ def get_table_index(
                 text_range = (t.x0, t.x1)
                 col_range = (table.cols[0][0], table.cols[-1][1])
                 warnings.warn(
-                    f"{text} {text_range} does not lie in column range {col_range}"
+                    "{} {} does not lie in column range {}".format(
+                        text, text_range, col_range
+                    )
                 )
             r_idx = r
             c_idx = lt_col_overlap.index(max(lt_col_overlap))

docs/index.rst

@@ -33,9 +33,6 @@ Release v\ |version|. (:ref:`Installation <install>`)
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
 
-.. image:: https://img.shields.io/badge/continous%20quality-deepsource-lightgrey
-    :target: https://deepsource.io/gh/camelot-dev/camelot/?ref=repository-badge
-
 **Camelot** is a Python library that makes it easy for *anyone* to extract tables from PDF files!
 
 .. note:: You can also check out `Excalibur`_, which is a web interface for Camelot!

docs/user/how-it-works.rst

@@ -20,7 +20,7 @@ Stream can be used to parse tables that have whitespaces between cells to simula
 
 3. The number of columns inside each table area are then guessed. This is done by calculating the mode of number of words in each text row. Based on this mode, words in each text row are chosen to calculate a list of column *x* ranges.
 
-4. Words that lie inside/outside the current column *x* ranges are then used to extend the current list of columns.
+4. Words that lie inside/outside the current column *x* ranges are then used to extend extend the current list of columns.
 
 5. Finally, a table is formed using the text rows' *y* ranges and column *x* ranges and words found on the page are assigned to the table's cells based on their *x* and *y* coordinates.
 

setup.py

@@ -71,9 +71,10 @@ def setup_package():
                         # Trove classifiers
                         # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
                         'License :: OSI Approved :: MIT License',
+                        'Programming Language :: Python :: 2.7',
+                        'Programming Language :: Python :: 3.5',
                         'Programming Language :: Python :: 3.6',
-                        'Programming Language :: Python :: 3.7',
-                        'Programming Language :: Python :: 3.8'
+                        'Programming Language :: Python :: 3.7'
                     ])
 
     try:

tests/__init__.py

@@ -1,3 +1,2 @@
 import matplotlib
-
-matplotlib.use("agg")
+matplotlib.use('agg')

tests/data.py

@@ -1,7 +1,19 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
 
 data_stream = [
+    [
+        "",
+        "Table: 5            Public Health Outlay 2012-13 (Budget Estimates)        (Rs. in 000)",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+    ],
     ["States-A", "Revenue", "", "Capital", "", "Total", "Others(1)", "Total"],
     ["", "", "", "", "", "Revenue &", "", ""],
     ["", "Medical &", "Family", "Medical &", "Family", "", "", ""],
@@ -817,6 +829,18 @@ data_stream_table_rotated = [
 ]
 
 data_stream_two_tables_1 = [
+    [
+        "[In thousands (11,062.6 represents 11,062,600) For year ending December 31. Based on Uniform Crime Reporting (UCR)",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+    ],
     [
         "Program. Represents arrests reported (not charged) by 12,910 agencies with a total population of 247,526,916 as estimated",
         "",
@@ -891,7 +915,7 @@ data_stream_two_tables_1 = [
         "2,330 .9",
     ],
     [
-        "Violent crime   .  .  .  .  .  .  .  .\n .  .\n .  .\n .  .\n .  .\n .  .",
+        "Violent crime  .  .  .  .  .  .  .  .\n .  .\n .  .\n .  .\n .  .\n .  .",
         "467 .9",
         "69 .1",
         "398 .8",
@@ -1276,10 +1300,29 @@ data_stream_two_tables_1 = [
         "",
         "",
     ],
+    [
+        "",
+        "Source: U.S. Department of Justice, Federal Bureau of Investigation, Uniform Crime Reports, Arrests Master Files.",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+        "",
+    ],
 ]
 
-
 data_stream_two_tables_2 = [
+    [
+        "",
+        "Source: U.S. Department of Justice, Federal Bureau of Investigation, Uniform Crime Reports, Arrests Master Files.",
+        "",
+        "",
+        "",
+        "",
+    ],
     ["Table 325. Arrests by Race: 2009", "", "", "", "", ""],
     [
         "[Based on Uniform Crime Reporting (UCR) Program. Represents arrests reported (not charged) by 12,371 agencies",
@@ -1309,7 +1352,7 @@ data_stream_two_tables_2 = [
         "123,656",
     ],
     [
-        "Violent crime   .  .  .  .  .  .  .  .\n .  .\n .  .\n .  .\n .  .\n .\n .\n .  .\n .  .\n .\n .\n .\n .\n .  .",
+        "Violent crime  .  .  .  .  .  .  .  .\n .  .\n .  .\n .  .\n .  .\n .\n .\n .  .\n .  .\n .\n .\n .\n .\n .  .",
         "456,965",
         "268,346",
         "177,766",
@@ -1557,9 +1600,16 @@ data_stream_two_tables_2 = [
         "3,950",
     ],
     ["1 Except forcible rape and prostitution.", "", "", "", "", ""],
+    [
+        "",
+        "Source: U.S. Department of Justice, Federal Bureau of Investigation, “Crime in the United States, Arrests,” September 2010,",
+        "",
+        "",
+        "",
+        "",
+    ],
 ]
 
-
 data_stream_table_areas = [
     ["", "One Withholding"],
     ["Payroll Period", "Allowance"],
@@ -1726,7 +1776,18 @@ data_stream_columns = [
 ]
 
 data_stream_split_text = [
-    ["FEB", "RUAR", "Y 2014 M27 (BUS)", "", "", "", "", "", "", ""],
+    [
+        "FEB",
+        "RUAR",
+        "Y 2014 M27 (BUS)",
+        "",
+        "ALPHABETIC LISTING BY T",
+        "YPE",
+        "",
+        "",
+        "",
+        "ABLPDM27",
+    ],
     ["", "", "", "", "OF ACTIVE LICENSES", "", "", "", "", "3/19/2014"],
     ["", "", "", "", "OKLAHOMA ABLE COMMIS", "SION", "", "", "", ""],
     ["LICENSE", "", "", "", "PREMISE", "", "", "", "", ""],
@@ -2060,7 +2121,6 @@ data_stream_split_text = [
     ],
 ]
 
-
 data_stream_flag_size = [
     [
         "States",
@@ -2760,7 +2820,7 @@ data_arabic = [
 ]
 
 data_stream_layout_kwargs = [
-    ["V i n s   a u   V e r r e", ""],
+    ["V i n s   a u   Ve r r e", ""],
     ["Les Blancs", "12.5CL"],
     ["A.O.P Côtes du Rhône", ""],
     ["Domaine de la Guicharde «  Autour de la chapelle » 2016", "8 €"],

tests/test_cli.py

@@ -114,35 +114,31 @@ def test_cli_password():
 def test_cli_output_format():
     with TemporaryDirectory() as tempdir:
         infile = os.path.join(testdir, "health.pdf")
-
+        outfile = os.path.join(tempdir, "health.{}")
         runner = CliRunner()
 
         # json
-        outfile = os.path.join(tempdir, "health.json")
         result = runner.invoke(
             cli,
-            ["--format", "json", "--output", outfile, "stream", infile],
+            ["--format", "json", "--output", outfile.format("json"), "stream", infile],
         )
         assert result.exit_code == 0
 
         # excel
-        outfile = os.path.join(tempdir, "health.xlsx")
         result = runner.invoke(
             cli,
-            ["--format", "excel", "--output", outfile, "stream", infile],
+            ["--format", "excel", "--output", outfile.format("xlsx"), "stream", infile],
         )
         assert result.exit_code == 0
 
         # html
-        outfile = os.path.join(tempdir, "health.html")
         result = runner.invoke(
             cli,
-            ["--format", "html", "--output", outfile, "stream", infile],
+            ["--format", "html", "--output", outfile.format("html"), "stream", infile],
         )
         assert result.exit_code == 0
 
         # zip
-        outfile = os.path.join(tempdir, "health.csv")
         result = runner.invoke(
             cli,
             [
@@ -150,7 +146,7 @@ def test_cli_output_format():
                 "--format",
                 "csv",
                 "--output",
-                outfile,
+                outfile.format("csv"),
                 "stream",
                 infile,
             ],

tests/test_common.py

@@ -3,11 +3,9 @@
 import os
 
 import pandas as pd
-from pandas.testing import assert_frame_equal
 
 import camelot
 from camelot.core import Table, TableList
-from camelot.__version__ import generate_version
 
 from .data import *
 
@@ -28,10 +26,10 @@ def test_password():
 
     filename = os.path.join(testdir, "health_protected.pdf")
     tables = camelot.read_pdf(filename, password="ownerpass", flavor="stream")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
     tables = camelot.read_pdf(filename, password="userpass", flavor="stream")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream():
@@ -39,7 +37,7 @@ def test_stream():
 
     filename = os.path.join(testdir, "health.pdf")
     tables = camelot.read_pdf(filename, flavor="stream")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_table_rotated():
@@ -47,11 +45,11 @@ def test_stream_table_rotated():
 
     filename = os.path.join(testdir, "clockwise_table_2.pdf")
     tables = camelot.read_pdf(filename, flavor="stream")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
     filename = os.path.join(testdir, "anticlockwise_table_2.pdf")
     tables = camelot.read_pdf(filename, flavor="stream")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_two_tables():
@@ -73,7 +71,7 @@ def test_stream_table_regions():
     tables = camelot.read_pdf(
         filename, flavor="stream", table_regions=["320,460,573,335"]
     )
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_table_areas():
@@ -83,7 +81,7 @@ def test_stream_table_areas():
     tables = camelot.read_pdf(
         filename, flavor="stream", table_areas=["320,500,573,335"]
     )
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_columns():
@@ -93,7 +91,7 @@ def test_stream_columns():
     tables = camelot.read_pdf(
         filename, flavor="stream", columns=["67,180,230,425,475"], row_tol=10
     )
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_split_text():
@@ -106,7 +104,7 @@ def test_stream_split_text():
         columns=["72,95,209,327,442,529,566,606,683"],
         split_text=True,
     )
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_flag_size():
@@ -114,7 +112,7 @@ def test_stream_flag_size():
 
     filename = os.path.join(testdir, "superscript.pdf")
     tables = camelot.read_pdf(filename, flavor="stream", flag_size=True)
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_strip_text():
@@ -122,7 +120,7 @@ def test_stream_strip_text():
 
     filename = os.path.join(testdir, "detect_vertical_false.pdf")
     tables = camelot.read_pdf(filename, flavor="stream", strip_text=" ,\n")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_edge_tol():
@@ -130,7 +128,7 @@ def test_stream_edge_tol():
 
     filename = os.path.join(testdir, "edge_tol.pdf")
     tables = camelot.read_pdf(filename, flavor="stream", edge_tol=500)
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_stream_layout_kwargs():
@@ -140,7 +138,7 @@ def test_stream_layout_kwargs():
     tables = camelot.read_pdf(
         filename, flavor="stream", layout_kwargs={"detect_vertical": False}
     )
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_lattice():
@@ -150,7 +148,7 @@ def test_lattice():
         testdir, "tabula/icdar2013-dataset/competition-dataset-us/us-030.pdf"
     )
     tables = camelot.read_pdf(filename, pages="2")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_lattice_table_rotated():
@@ -158,11 +156,11 @@ def test_lattice_table_rotated():
 
     filename = os.path.join(testdir, "clockwise_table_1.pdf")
     tables = camelot.read_pdf(filename)
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
     filename = os.path.join(testdir, "anticlockwise_table_1.pdf")
     tables = camelot.read_pdf(filename)
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_lattice_two_tables():
@@ -181,7 +179,7 @@ def test_lattice_table_regions():
 
     filename = os.path.join(testdir, "table_region.pdf")
     tables = camelot.read_pdf(filename, table_regions=["170,370,560,270"])
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_lattice_table_areas():
@@ -189,7 +187,7 @@ def test_lattice_table_areas():
 
     filename = os.path.join(testdir, "twotables_2.pdf")
     tables = camelot.read_pdf(filename, table_areas=["80,693,535,448"])
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_lattice_process_background():
@@ -197,7 +195,7 @@ def test_lattice_process_background():
 
     filename = os.path.join(testdir, "background_lines_1.pdf")
     tables = camelot.read_pdf(filename, process_background=True)
-    assert_frame_equal(df, tables[1].df)
+    assert df.equals(tables[1].df)
 
 
 def test_lattice_copy_text():
@@ -205,7 +203,7 @@ def test_lattice_copy_text():
 
     filename = os.path.join(testdir, "row_span_1.pdf")
     tables = camelot.read_pdf(filename, line_scale=60, copy_text="v")
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_lattice_shift_text():
@@ -273,7 +271,7 @@ def test_arabic():
 
     filename = os.path.join(testdir, "tabula/arabic.pdf")
     tables = camelot.read_pdf(filename)
-    assert_frame_equal(df, tables[0].df)
+    assert df.equals(tables[0].df)
 
 
 def test_table_order():
@@ -299,18 +297,3 @@ def test_table_order():
         (1, 2),
         (1, 1),
     ]
-
-
-def test_version_generation():
-    version = (0, 7, 3)
-    assert generate_version(version, prerelease=None, revision=None) == "0.7.3"
-
-
-def test_version_generation_with_prerelease_revision():
-    version = (0, 7, 3)
-    prerelease = "alpha"
-    revision = 2
-    assert (
-        generate_version(version, prerelease=prerelease, revision=revision)
-        == "0.7.3-alpha.2"
-    )

tests/test_errors.py

@@ -10,93 +10,88 @@ import camelot
 
 testdir = os.path.dirname(os.path.abspath(__file__))
 testdir = os.path.join(testdir, "files")
-filename = os.path.join(testdir, "foo.pdf")
+filename = os.path.join(testdir, 'foo.pdf')
 
 
 def test_unknown_flavor():
-    message = "Unknown flavor specified." " Use either 'lattice' or 'stream'"
+    message = ("Unknown flavor specified."
+               " Use either 'lattice' or 'stream'")
     with pytest.raises(NotImplementedError, match=message):
-        tables = camelot.read_pdf(filename, flavor="chocolate")
+        tables = camelot.read_pdf(filename, flavor='chocolate')
 
 
 def test_input_kwargs():
     message = "columns cannot be used with flavor='lattice'"
     with pytest.raises(ValueError, match=message):
-        tables = camelot.read_pdf(filename, columns=["10,20,30,40"])
+        tables = camelot.read_pdf(filename, columns=['10,20,30,40'])
 
 
 def test_unsupported_format():
-    message = "File format not supported"
-    filename = os.path.join(testdir, "foo.csv")
+    message = 'File format not supported'
+    filename = os.path.join(testdir, 'foo.csv')
     with pytest.raises(NotImplementedError, match=message):
         tables = camelot.read_pdf(filename)
 
 
 def test_stream_equal_length():
-    message = "Length of table_areas and columns" " should be equal"
+    message = ("Length of table_areas and columns"
+               " should be equal")
     with pytest.raises(ValueError, match=message):
-        tables = camelot.read_pdf(
-            filename,
-            flavor="stream",
-            table_areas=["10,20,30,40"],
-            columns=["10,20,30,40", "10,20,30,40"],
-        )
+        tables = camelot.read_pdf(filename, flavor='stream',
+            table_areas=['10,20,30,40'], columns=['10,20,30,40', '10,20,30,40'])
 
 
 def test_image_warning():
-    filename = os.path.join(testdir, "image.pdf")
+    filename = os.path.join(testdir, 'image.pdf')
     with warnings.catch_warnings():
-        warnings.simplefilter("error")
+        warnings.simplefilter('error')
         with pytest.raises(UserWarning) as e:
             tables = camelot.read_pdf(filename)
-            assert (
-                str(e.value)
-                == "page-1 is image-based, camelot only works on text-based pages."
-            )
+        assert str(e.value) == 'page-1 is image-based, camelot only works on text-based pages.'
 
 
 def test_no_tables_found():
-    filename = os.path.join(testdir, "blank.pdf")
+    filename = os.path.join(testdir, 'blank.pdf')
     with warnings.catch_warnings():
-        warnings.simplefilter("error")
+        warnings.simplefilter('error')
         with pytest.raises(UserWarning) as e:
             tables = camelot.read_pdf(filename)
-        assert str(e.value) == "No tables found on page-1"
+        assert str(e.value) == 'No tables found on page-1'
 
 
 def test_no_tables_found_logs_suppressed():
-    filename = os.path.join(testdir, "foo.pdf")
+    filename = os.path.join(testdir, 'foo.pdf')
     with warnings.catch_warnings():
         # the test should fail if any warning is thrown
-        warnings.simplefilter("error")
+        warnings.simplefilter('error')
         try:
             tables = camelot.read_pdf(filename, suppress_stdout=True)
         except Warning as e:
             warning_text = str(e)
-            pytest.fail(f"Unexpected warning: {warning_text}")
+            pytest.fail('Unexpected warning: {}'.format(warning_text))
 
 
 def test_no_tables_found_warnings_suppressed():
-    filename = os.path.join(testdir, "blank.pdf")
+    filename = os.path.join(testdir, 'blank.pdf')
     with warnings.catch_warnings():
         # the test should fail if any warning is thrown
-        warnings.simplefilter("error")
+        warnings.simplefilter('error')
         try:
             tables = camelot.read_pdf(filename, suppress_stdout=True)
         except Warning as e:
             warning_text = str(e)
-            pytest.fail(f"Unexpected warning: {warning_text}")
+            pytest.fail('Unexpected warning: {}'.format(warning_text))
 
 
 def test_no_password():
-    filename = os.path.join(testdir, "health_protected.pdf")
-    message = "file has not been decrypted"
+    filename = os.path.join(testdir, 'health_protected.pdf')
+    message = 'file has not been decrypted'
     with pytest.raises(Exception, match=message):
         tables = camelot.read_pdf(filename)
 
 
 def test_bad_password():
-    filename = os.path.join(testdir, "health_protected.pdf")
-    message = "file has not been decrypted"
+    filename = os.path.join(testdir, 'health_protected.pdf')
+    message = 'file has not been decrypted'
     with pytest.raises(Exception, match=message):
-        tables = camelot.read_pdf(filename, password="wrongpass")
+        tables = camelot.read_pdf(filename, password='wrongpass')

tests/test_plotting.py

@@ -11,50 +11,57 @@ testdir = os.path.dirname(os.path.abspath(__file__))
 testdir = os.path.join(testdir, "files")
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_text_plot():
     filename = os.path.join(testdir, "foo.pdf")
     tables = camelot.read_pdf(filename)
-    return camelot.plot(tables[0], kind="text")
+    return camelot.plot(tables[0], kind='text')
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_grid_plot():
     filename = os.path.join(testdir, "foo.pdf")
     tables = camelot.read_pdf(filename)
-    return camelot.plot(tables[0], kind="grid")
+    return camelot.plot(tables[0], kind='grid')
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_lattice_contour_plot():
     filename = os.path.join(testdir, "foo.pdf")
     tables = camelot.read_pdf(filename)
-    return camelot.plot(tables[0], kind="contour")
+    return camelot.plot(tables[0], kind='contour')
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_stream_contour_plot():
     filename = os.path.join(testdir, "tabula/12s0324.pdf")
-    tables = camelot.read_pdf(filename, flavor="stream")
-    return camelot.plot(tables[0], kind="contour")
+    tables = camelot.read_pdf(filename, flavor='stream')
+    return camelot.plot(tables[0], kind='contour')
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_line_plot():
     filename = os.path.join(testdir, "foo.pdf")
     tables = camelot.read_pdf(filename)
-    return camelot.plot(tables[0], kind="line")
+    return camelot.plot(tables[0], kind='line')
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_joint_plot():
     filename = os.path.join(testdir, "foo.pdf")
     tables = camelot.read_pdf(filename)
-    return camelot.plot(tables[0], kind="joint")
+    return camelot.plot(tables[0], kind='joint')
 
 
-@pytest.mark.mpl_image_compare(baseline_dir="files/baseline_plots", remove_text=True)
+@pytest.mark.mpl_image_compare(
+    baseline_dir="files/baseline_plots", remove_text=True)
 def test_textedge_plot():
     filename = os.path.join(testdir, "tabula/12s0324.pdf")
-    tables = camelot.read_pdf(filename, flavor="stream")
-    return camelot.plot(tables[0], kind="textedge")
+    tables = camelot.read_pdf(filename, flavor='stream')
+    return camelot.plot(tables[0], kind='textedge')