Add Python 3 compatibility (#109)

* Add python3 compat

* Update .gitignore

* Update .gitignore again

* Remove debugging return

* Add unicode_literals import

* Bump version

* Add python3-tk note

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 build/
 dist/
 *.egg-info/
+.eggs/
 .coverage
 coverage.xml
 

.travis.yml

@@ -1,8 +1,9 @@
 language: python
 python:
   - "2.7"
+  - "3.6"
 before_install:
-  - sudo apt-get install python-tk ghostscript
+  - sudo apt-get install python-tk python3-tk ghostscript
 install:
   - pip install ".[dev]"
 script:

README.md

@@ -17,7 +17,7 @@
 >>> import camelot
 >>> tables = camelot.read_pdf('foo.pdf')
 >>> tables
-<TableList tables=1>
+<TableList n=1>
 >>> tables.export('foo.csv', f='csv', compress=True) # json, excel, html
 >>> tables[0]
 <Table shape=(7, 7)>
@@ -43,6 +43,8 @@
 
 There's a [command-line interface](https://camelot-py.readthedocs.io/en/latest/user/cli.html) too!
 
+**Note:** Camelot only works with text-based PDFs and not scanned documents. If you can click-and-drag to select text in your table in a PDF viewer, then your PDF is text-based.
+
 ## Why Camelot?
 
 - **You are in control**: Unlike other libraries and tools which either give a nice output or fail miserably (with no in-between), Camelot gives you the power to tweak table extraction. (Since everything in the real world, including PDF table extraction, is fuzzy.)
@@ -75,7 +77,7 @@ $ cd camelot
 $ pip install .
 </pre>
 
-Note: Use a [virtualenv](https://virtualenv.pypa.io/en/stable/) if you don't want to affect your global Python installation.
+**Note:** Use a [virtualenv](https://virtualenv.pypa.io/en/stable/) if you don't want to affect your global Python installation.
 
 ## Documentation
 

camelot/__version__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-VERSION = (0, 1, 2)
+VERSION = (0, 2, 0)
 
 __title__ = 'camelot-py'
 __description__ = 'PDF Table Extraction for Humans.'

camelot/core.py

@@ -438,8 +438,8 @@ class TableList(object):
         self._tables = tables
 
     def __repr__(self):
-        return '<{} tables={}>'.format(
-            self.__class__.__name__, len(self._tables))
+        return '<{} n={}>'.format(
+            self.__class__.__name__, self.n)
 
     def __len__(self):
         return len(self._tables)

camelot/parsers/lattice.py

@@ -13,7 +13,7 @@ from .base import BaseParser
 from ..core import Table
 from ..utils import (scale_image, scale_pdf, segments_in_bbox, text_in_bbox,
                      merge_close_lines, get_table_index, compute_accuracy,
-                     compute_whitespace, setup_logging, encode_)
+                     compute_whitespace, setup_logging)
 from ..image_processing import (adaptive_threshold, find_lines,
                                 find_table_contours, find_table_joints)
 
@@ -177,7 +177,7 @@ class Lattice(BaseParser):
         gs_call = [
             "-q", "-sDEVICE=png16m", "-o", self.imagename, "-r600", self.filename
         ]
-        if "ghostscript" in subprocess.check_output(["gs", "-version"]).lower():
+        if "ghostscript" in subprocess.check_output(["gs", "-version"]).decode('utf-8').lower():
             gs_call.insert(0, "gs")
         else:
             gs_call.insert(0, "gsc")
@@ -284,7 +284,6 @@ class Lattice(BaseParser):
             table = Lattice._copy_spanning_text(table, copy_text=self.copy_text)
 
         data = table.data
-        data = encode_(data)
         table.df = pd.DataFrame(data)
         table.shape = table.df.shape
 

camelot/parsers/stream.py

@@ -10,7 +10,7 @@ import pandas as pd
 from .base import BaseParser
 from ..core import Table
 from ..utils import (text_in_bbox, get_table_index, compute_accuracy,
-                     compute_whitespace, setup_logging, encode_)
+                     compute_whitespace, setup_logging)
 
 
 logger = setup_logging(__name__)
@@ -323,7 +323,6 @@ class Stream(BaseParser):
         accuracy = compute_accuracy([[100, pos_errors]])
 
         data = table.data
-        data = encode_(data)
         table.df = pd.DataFrame(data)
         table.shape = table.df.shape
 

camelot/utils.py

@@ -560,7 +560,7 @@ def get_table_index(table, t, direction, split_text=False, flag_size=False):
                     lt_col_overlap.append(abs(left - right) / abs(c[0] - c[1]))
                 else:
                     lt_col_overlap.append(-1)
-            if len(filter(lambda x: x != -1, lt_col_overlap)) == 0:
+            if len(list(filter(lambda x: x != -1, lt_col_overlap))) == 0:
                 text = t.get_text().strip('\n')
                 text_range = (t.x0, t.x1)
                 col_range = (table.cols[0][0], table.cols[-1][1])
@@ -669,22 +669,6 @@ def remove_empty(d):
     return d
 
 
-def encode_(ar):
-    """Encodes two-dimensional list into unicode.
-
-    Parameters
-    ----------
-    ar : list
-
-    Returns
-    -------
-    ar : list
-
-    """
-    ar = [[r.encode('utf-8') for r in row] for row in ar]
-    return ar
-
-
 def get_page_layout(filename, char_margin=1.0, line_margin=0.5, word_margin=0.1,
                detect_vertical=True, all_texts=True):
     """Returns a PDFMiner LTPage object and page dimension of a single
@@ -709,7 +693,7 @@ def get_page_layout(filename, char_margin=1.0, line_margin=0.5, word_margin=0.1,
         Dimension of pdf page in the form (width, height).
 
     """
-    with open(filename, 'r') as f:
+    with open(filename, 'rb') as f:
         parser = PDFParser(f)
         document = PDFDocument(parser)
         if not document.is_extractable:

docs/index.rst

@@ -25,15 +25,7 @@ Release v\ |version|. (:ref:`Installation <install>`)
 
 **Camelot** is a Python library which makes it easy for *anyone* to extract tables from PDF files!
 
-.. note:: Camelot only works with:
-
-          - Python 2, with **Python 3** support `on the way`_.
-          - Text-based PDFs and not scanned documents. If you can click-and-drag to select text in your table in a PDF viewer, then your PDF is text-based. Support for image-based PDFs using **OCR** is `planned`_.
-
-.. _on the way: https://github.com/socialcopsdev/camelot/issues/81
-.. _planned: https://github.com/socialcopsdev/camelot/issues/101
-
-------------------------
+----
 
 **Here's how you can extract tables from PDF files.** Check out the PDF used in this example, `here`_.
 
@@ -44,7 +36,7 @@ Release v\ |version|. (:ref:`Installation <install>`)
     >>> import camelot
     >>> tables = camelot.read_pdf('foo.pdf')
     >>> tables
-    <TableList tables=1>
+    <TableList n=1>
     >>> tables.export('foo.csv', f='csv', compress=True) # json, excel, html
     >>> tables[0]
     <Table shape=(7, 7)>
@@ -63,6 +55,8 @@ Release v\ |version|. (:ref:`Installation <install>`)
 
 There's a :ref:`command-line interface <cli>` too!
 
+.. note:: Camelot only works with text-based PDFs and not scanned documents. If you can click-and-drag to select text in your table in a PDF viewer, then your PDF is text-based.
+
 Why Camelot?
 ------------
 

docs/user/install.rst

@@ -14,6 +14,8 @@ For Ubuntu::
 
     $ apt install python-tk ghostscript
 
+.. note:: For Python 3, install python3-tk.
+
 For macOS::
 
     $ brew install tcl-tk ghostscript

docs/user/intro.rst

@@ -32,10 +32,13 @@ Here is a `comparison`_ of Camelot's output with outputs from other open-source
 What's in a name?
 -----------------
 
-As you can already guess, this library is named after `The Camelot Project`_. Fun fact, "Camelot" is the name of the castle in `Monty Python and the Holy Grail`_, where Arthur leads his men, the Knights of the Round Table, and then sets off elsewhere after deciding that it is "a silly place". Interestingly, the language in which this library is written (Python) was named after Monty Python.
+As you can already guess, this library is named after `The Camelot Project`_.
+
+Fun fact: "Camelot" is the name of the castle in the British comedy film `Monty Python and the Holy Grail`_ (and in the `Arthurian legend`_, which the film depicts), where Arthur leads his men, the Knights of the Round Table, and then sets off elsewhere after deciding that it is "a silly place". Interestingly, the language in which this library is written (Python) was named after Monty Python.
 
 .. _The Camelot Project: http://www.planetpdf.com/planetpdf/pdfs/warnock_camelot.pdf
 .. _Monty Python and the Holy Grail: https://en.wikipedia.org/wiki/Monty_Python_and_the_Holy_Grail
+.. _Arthurian legend: https://en.wikipedia.org/wiki/King_Arthur
 
 Camelot License
 ---------------

requirements.txt

@@ -3,5 +3,5 @@ matplotlib==2.2.3
 numpy==1.13.3
 opencv-python==3.4.2.17
 pandas==0.23.4
-pdfminer==20140328
+pdfminer.six==20170720
 PyPDF2==1.26.0

setup.py

@@ -48,7 +48,8 @@ def setup_package():
                         # Trove classifiers
                         # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
                         'License :: OSI Approved :: MIT License',
-                        'Programming Language :: Python :: 2.7'
+                        'Programming Language :: Python :: 2.7',
+                        'Programming Language :: Python :: 3.6'
                     ])
 
     try:

tests/test_data.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
+
 data_stream_table_rotated = [
-    ["","","Table 21  Current use of contraception by background characteristics—Continued","","","","","","","","","","","","","","",""],
+    ["", "", "Table 21  Current use of contraception by background characteristics\u2014Continued", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""],
     ["", "", "", "", "", "", "Modern method", "", "", "", "", "", "", "Traditional method", "", "", "", ""],
     ["", "", "", "Any", "", "", "", "", "", "", "Other", "Any","", "", "", "Not", "", "Number"],
     ["", "", "Any", "modern", "Female", "Male", "", "", "", "Condom/", "modern", "traditional", "", "With-", "Folk", "currently", "", "of"],
@@ -18,9 +21,10 @@ data_stream_table_rotated = [
     ["", "Fourth", "73.9", "52.3", "32.0", "0.5", "12.5", "0.6", "0.2", "6.3", "0.2", "21.6", "11.5", "9.9", "0.2", "26.1", "100.0", "908"],
     ["", "Highest", "78.3", "44.4", "19.5", "1.0", "9.7", "1.4", "0.0", "12.7", "0.0", "33.8", "18.2", "15.6", "0.0", "21.7", "100.0", "733"],
     ["", "Number of living children", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""],
-    ["","No children","25.1","7.6","0.3","0.5","2.0","0.0","0.0","4.8","0.0","17.5","9.0","8.5","0.0","74.9","100.0","563"],
+    ["", "No children", "25.1", "7.6", "0.3", "0.5", "2.0", "0.0",
+    "0.0", "4.8", "0.0", "17.5", "9.0", "8.5", "0.0", "74.9", "100.0", "563"],
     ["", "1 child", "66.5", "32.1", "3.7", "0.7", "20.1", "0.7", "0.1", "6.9", "0.0", "34.3", "18.9", "15.2", "0.3", "33.5", "100.0", "1,190"],
-    ["","1 son","66.8","33.2","4.1","0.7","21.1","0.5","0.3","6.6","0.0","33.5","21.2","12.3","0.0","33.2","100.0","672"],
+    ["\x18\x18", "1 son", "66.8", "33.2", "4.1", "0.7", "21.1", "0.5", "0.3", "6.6", "0.0", "33.5", "21.2", "12.3", "0.0", "33.2", "100.0", "672"],
     ["", "No sons", "66.1", "30.7", "3.1", "0.6", "18.8", "0.8", "0.0", "7.3", "0.0", "35.4", "15.8", "19.0", "0.6", "33.9", "100.0", "517"],
     ["", "2 children", "81.6", "60.5", "41.8", "0.9", "11.6", "0.8", "0.3", "4.8", "0.2", "21.1", "12.2", "8.3", "0.6", "18.4", "100.0", "1,576"],
     ["", "1 or more sons", "83.7", "64.2", "46.4", "0.9", "10.8", "0.8", "0.4", "4.8", "0.1", "19.5", "11.1", "7.6", "0.7", "16.3", "100.0", "1,268"],
@@ -31,7 +35,8 @@ data_stream_table_rotated = [
     ["", "4+ children", "74.3", "58.1", "45.1", "0.6", "8.7", "0.6", "0.7", "2.4", "0.0", "16.1", "9.9", "5.4", "0.8", "25.7", "100.0", "944"],
     ["", "1 or more sons", "73.9", "58.2", "46.0", "0.7", "8.3", "0.7", "0.7", "1.9", "0.0", "15.7", "9.4", "5.5", "0.8", "26.1", "100.0", "901"],
     ["", "No sons", "(82.1)", "(57.3)", "(25.6)", "(0.0)", "(17.8)", "(0.0)", "(0.0)", "(13.9)", "(0.0)", "(24.8)", "(21.3)", "(3.5)", "(0.0)", "(17.9)", "100.0", "43"],
-    ["","Total","71.2","49.9","32.2","0.7","11.7","0.6","0.3","4.3","0.1","21.3","12.3","8.4","0.5","28.8","100.0","5,234"],
+    ["", "Total", "71.2", "49.9", "32.2",
+    "0.7", "11.7", "0.6", "0.3", "4.3", "0.1", "21.3", "12.3", "8.4", "0.5", "28.8", "100.0", "5,234"],
     ["", "NFHS-2 (1998-99)", "66.6", "47.3", "32.0", "1.8", "9.2", "1.4", "na", "2.9", "na", "na", "8.7", "9.8", "na", "33.4", "100.0", "4,116"],
     ["", "NFHS-1 (1992-93)", "57.7", "37.6", "26.5", "4.3", "3.6", "1.3", "0.1", "1.9", "na", "na", "11.3", "8.3", "na", "42.3", "100.0", "3,970"],
     ["", "", "Note: If more than one method is used, only the most effective method is considered in this tabulation. Total includes women for whom caste/tribe was not known or is missing, who are", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""],
@@ -42,7 +47,6 @@ data_stream_table_rotated = [
     ["", "", "", "", "", "", "", "", "54", "", "", "", "", "", "", "", "", ""]
 ]
 
-
 data_stream_table_area_single = [
     ["","One Withholding"],
     ["Payroll Period","Allowance"],
@@ -57,7 +61,6 @@ data_stream_table_area_single = [
     ["(each day of the payroll period)",""]
 ]
 
-
 data_stream_columns = [
     ["Clave", "Nombre Entidad", "Clave", "Nombre Municipio", "Clave", "Nombre Localidad"],
     ["Entidad", "", "Municipio", "", "Localidad", ""],
@@ -67,15 +70,15 @@ data_stream_columns = [
     ["01", "Aguascalientes", "001", "Aguascalientes", "0102", "Los Arbolitos [Rancho]"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0104", "Ardillas de Abajo (Las Ardillas)"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0106", "Arellano"],
-    ["01","Aguascalientes","001","Aguascalientes","0112","Bajío los Vázquez"],
-    ["01","Aguascalientes","001","Aguascalientes","0113","Bajío de Montoro"],
-    ["01","Aguascalientes","001","Aguascalientes","0114","Residencial San Nicolás [Baños la Cantera]"],
-    ["01","Aguascalientes","001","Aguascalientes","0120","Buenavista de Peñuelas"],
-    ["01","Aguascalientes","001","Aguascalientes","0121","Cabecita 3 Marías (Rancho Nuevo)"],
-    ["01","Aguascalientes","001","Aguascalientes","0125","Cañada Grande de Cotorina"],
-    ["01","Aguascalientes","001","Aguascalientes","0126","Cañada Honda [Estación]"],
-    ["01","Aguascalientes","001","Aguascalientes","0127","Los Caños"],
-    ["01","Aguascalientes","001","Aguascalientes","0128","El Cariñán"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0112", "Baj\xedo los V\xe1zquez"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0113", "Baj\xedo de Montoro"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0114", "Residencial San Nicol\xe1s [Ba\xf1os la Cantera]"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0120", "Buenavista de Pe\xf1uelas"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0121", "Cabecita 3 Mar\xedas (Rancho Nuevo)"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0125", "Ca\xf1ada Grande de Cotorina"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0126", "Ca\xf1ada Honda [Estaci\xf3n]"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0127", "Los Ca\xf1os"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0128", "El Cari\xf1\xe1n"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0129", "El Carmen [Granja]"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0135", "El Cedazo (Cedazo de San Antonio)"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0138", "Centro de Arriba (El Taray)"],
@@ -86,25 +89,24 @@ data_stream_columns = [
     ["01", "Aguascalientes", "001", "Aguascalientes", "0157", "Cotorina de Abajo"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0162", "Coyotes"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0166", "La Huerta (La Cruz)"],
-    ["01","Aguascalientes","001","Aguascalientes","0170","Cuauhtémoc (Las Palomas)"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0170", "Cuauht\xe9moc (Las Palomas)"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0171", "Los Cuervos (Los Ojos de Agua)"],
-    ["01","Aguascalientes","001","Aguascalientes","0172","San José [Granja]"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0172", "San Jos\xe9 [Granja]"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0176", "La Chiripa"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0182", "Dolores"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0183", "Los Dolores"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0190", "El Duraznillo"],
-    ["01","Aguascalientes","001","Aguascalientes","0191","Los Durón"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0191", "Los Dur\xf3n"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0197", "La Escondida"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0201", "Brande Vin [Bodegas]"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0207", "Valle Redondo"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0209", "La Fortuna"],
-    ["01","Aguascalientes","001","Aguascalientes","0212","Lomas del Gachupín"],
-    ["01","Aguascalientes","001","Aguascalientes","0213","El Carmen (Gallinas Güeras) [Rancho]"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0212", "Lomas del Gachup\xedn"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0213", "El Carmen (Gallinas G\xfceras) [Rancho]"],
     ["01", "Aguascalientes", "001", "Aguascalientes", "0216", "La Gloria"],
-    ["01","Aguascalientes","001","Aguascalientes","0226","Hacienda Nueva"],
+    ["01", "Aguascalientes", "001", "Aguascalientes", "0226", "Hacienda Nueva"]
 ]
 
-
 data_lattice = [
     ["Cycle Name","KI (1/km)","Distance (mi)","Percent Fuel Savings","","",""],
     ["","","","Improved Speed","Decreased Accel","Eliminate Stops","Decreased Idle"],
@@ -115,7 +117,6 @@ data_lattice = [
     ["4171_1","0.07","173.9","58.1%","1.6%","2.1%","0.5%"]
 ]
 
-
 data_lattice_table_rotated = [
     ["State","Nutritional Assessment  (No. of individuals)","","","","IYCF Practices  (No. of mothers: 2011-12)","Blood Pressure  (No. of adults: 2011-12)","","Fasting  Blood Sugar (No. of adults:2011-12)",""],
     ["","1975-79","1988-90","1996-97","2011-12","","Men","Women","Men","Women"],
@@ -132,7 +133,6 @@ data_lattice_table_rotated = [
     ["Pooled","38742","53618","60601","86898","4459","21918","27041","14312","18519"]
 ]
 
-
 data_lattice_process_background = [
     ["State","Date","Halt stations","Halt days","Persons directly reached(in lakh)","Persons trained","Persons counseled","Persons testedfor HIV"],
     ["Delhi","1.12.2009","8","17","1.29","3,665","2,409","1,000"],
@@ -144,7 +144,6 @@ data_lattice_process_background = [
     ["Total","","47","92","11.81","22,455","19,584","10,644"]
 ]
 
-
 data_lattice_copy_text = [
     ["Plan Type","County","Plan Name","Totals"],
     ["GMC","Sacramento","Anthem Blue Cross","164,380"],