{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Common import and setup\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": "'/Users/francoishuet/Code/camelot/camelot/__init__.py'" }, "metadata": {}, "execution_count": 1 } ], "source": [ "import os, sys, time, pytest\n", "\n", "import matplotlib.pyplot as plt\n", "from matplotlib import patches, lines\n", "import numpy as np\n", "import pandas as pd\n", "from pandas.testing import assert_frame_equal\n", "\n", "import pdfminer\n", "\n", "from IPython.display import display\n", "\n", "# Make sure we use the local version of camelot if it is here\n", "sys.path.insert(0, os.path.abspath(''))\n", "\n", "import camelot\n", "from camelot.core import Table, TableList, TextEdges\n", "from camelot.__version__ import generate_version\n", "from camelot.utils import get_text_objects, text_in_bbox\n", "from camelot.parsers.stream import Stream\n", "from camelot.parsers.hybrid import Hybrid\n", "from camelot.handlers import PDFHandler\n", "from camelot.plotting import draw_pdf\n", "from tests.data import *\n", "\n", "testdir = os.path.dirname(os.path.abspath('.'))\n", "testdir = os.path.join(testdir, \"camelot/tests/files\")\n", "\n", "# To check which library we're using\n", "camelot.__file__\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "kwargs = {}\n", "data = None\n", "# pdf_file = \"vertical_header.pdf\"\n", "# pdf_file, kwargs, data = \"superscript.pdf\", {\"flag_size\": True}, data_stream_flag_size # test_hybrid_flag_size\n", "# pdf_file = \"health.pdf\" # test_hybrid\n", "# pdf_file = \"clockwise_table_2.pdf\"\n", "# pdf_file = \"tabula/12s0324.pdf\" # interesting because contains two separate tables\n", "# pdf_file = \"clockwise_table_2.pdf\" # test_hybrid_table_rotated / test_stream_table_rotated\n", "# pdf_file, kwargs = \"tabula/us-007.pdf\", {\"table_regions\": [\"320,335,573,505\"]} # test_hybrid_table_regions\n", "# pdf_file, kwargs = \"detect_vertical_false.pdf\", {\"strip_text\": \" ,\\n\"} # data_stream_strip_text\n", "# pdf_file, kwargs, data = \"tabula/m27.pdf\", {\"columns\": [\"72,95,209,327,442,529,566,606,683\"], \"split_text\": True, }, data_stream_split_text # data_stream_split_text\n", "pdf_file = \"vertical_header.pdf\"\n", "# pdf_file = \"twotables_2.pdf\"\n", "# pdf_file = \"camelot-issue-132-multiple-tables.pdf\"\n", "# pdf_file, kwargs, data = \"edge_tol.pdf\", {\"edge_tol\": 500}, data_stream_edge_tol\n", "# pdf_file, kwargs, data = \"edge_tol.pdf\", {}, data_stream_edge_tol\n", "# pdf_file, kwargs = \"tabula/icdar2013-dataset/competition-dataset-us/us-030.pdf\", {\"pages\": \"2\"} # test_lattice\n", "# pdf_file, kwargs = \"background_lines_1.pdf\", {} # {\"process_background\": True} # test_lattice_process_background\n", "\n", "filename = os.path.join(testdir, pdf_file)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "output_type": "stream", "name": "stdout", "text": "Showing table #0 found by stream:\n" }, { "output_type": "display_data", "data": { "text/plain": " 0 1 2 3 4 \\\n0 Alcona 963 439 Brian Calley\\n55 Patrick Colbeck\\n26 \n1 Caledonia 923 393 40 23 \n2 Curtis 1026 349 30 30 \n3 Greenbush 1212 423 56 26 \n4 Gustin 611 180 22 35 \n5 Harrisville 1142 430 45 90 \n6 Hawes 884 293 38 36 \n7 Haynes 626 275 31 20 \n8 Mikado 781 208 19 39 \n9 Millen 353 139 7 16 \n10 Mitchell 327 96 12 17 \n11 City Harrisville 389 171 16 15 \n12 Totals 9237 3396 371 373 \n\n 5 6 7 8 9 \\\n0 Jim Hines\\n47 Bill Schuette\\n164 John James\\n173 Sandy Pensler\\n111 \n1 45 158 150 103 \n2 25 102 95 84 \n3 40 126 104 131 \n4 17 55 73 45 \n5 29 101 155 94 \n6 27 109 121 84 \n7 32 104 121 53 \n8 17 81 90 63 \n9 13 38 49 19 \n10 7 29 41 17 \n11 18 35 49 31 \n12 317 1102 1221 835 0 \n\n ... 15 16 17 18 19 \\\n0 ... Al Scully\\n271 Daniel G. Gauthier\\n224 Craig M. Clemens\\n76 \n1 ... 244 139 143 \n2 ... 157 \n3 ... 208 \n4 ... 109 \n5 ... 226 \n6 ... 184 \n7 ... 152 \n8 ... 143 \n9 ... 62 \n10 ... 56 \n11 ... 77 \n12 ... 0 1889 0 363 219 \n\n 20 21 22 23 24 \n0 Craig Johnston Adam Brege David Bielusiak \n1 \n2 \n3 208 \n4 81 42 \n5 232 \n6 118 87 \n7 76 69 31 \n8 113 \n9 \n10 \n11 73 \n12 381 321 268 160 0 \n\n[13 rows x 25 columns]", "text/html": "
| \n | 0 | \n1 | \n2 | \n3 | \n4 | \n5 | \n6 | \n7 | \n8 | \n9 | \n... | \n15 | \n16 | \n17 | \n18 | \n19 | \n20 | \n21 | \n22 | \n23 | \n24 | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \nAlcona | \n963 | \n439 | \nBrian Calley\\n55 | \nPatrick Colbeck\\n26 | \nJim Hines\\n47 | \nBill Schuette\\n164 | \nJohn James\\n173 | \nSandy Pensler\\n111 | \n\n | ... | \n\n | Al Scully\\n271 | \n\n | Daniel G. Gauthier\\n224 | \nCraig M. Clemens\\n76 | \nCraig Johnston | \n\n | Adam Brege | \nDavid Bielusiak | \n\n |
| 1 | \nCaledonia | \n923 | \n393 | \n40 | \n23 | \n45 | \n158 | \n150 | \n103 | \n\n | ... | \n\n | 244 | \n\n | 139 | \n143 | \n\n | \n | \n | \n | \n |
| 2 | \nCurtis | \n1026 | \n349 | \n30 | \n30 | \n25 | \n102 | \n95 | \n84 | \n\n | ... | \n\n | 157 | \n\n | \n | \n | \n | \n | \n | \n | \n |
| 3 | \nGreenbush | \n1212 | \n423 | \n56 | \n26 | \n40 | \n126 | \n104 | \n131 | \n\n | ... | \n\n | 208 | \n\n | \n | \n | \n | 208 | \n\n | \n | \n |
| 4 | \nGustin | \n611 | \n180 | \n22 | \n35 | \n17 | \n55 | \n73 | \n45 | \n\n | ... | \n\n | 109 | \n\n | \n | \n | \n | \n | 81 | \n42 | \n\n |
| 5 | \nHarrisville | \n1142 | \n430 | \n45 | \n90 | \n29 | \n101 | \n155 | \n94 | \n\n | ... | \n\n | 226 | \n\n | \n | \n | 232 | \n\n | \n | \n | \n |
| 6 | \nHawes | \n884 | \n293 | \n38 | \n36 | \n27 | \n109 | \n121 | \n84 | \n\n | ... | \n\n | 184 | \n\n | \n | \n | \n | \n | 118 | \n87 | \n\n |
| 7 | \nHaynes | \n626 | \n275 | \n31 | \n20 | \n32 | \n104 | \n121 | \n53 | \n\n | ... | \n\n | 152 | \n\n | \n | \n | 76 | \n\n | 69 | \n31 | \n\n |
| 8 | \nMikado | \n781 | \n208 | \n19 | \n39 | \n17 | \n81 | \n90 | \n63 | \n\n | ... | \n\n | 143 | \n\n | \n | \n | \n | 113 | \n\n | \n | \n |
| 9 | \nMillen | \n353 | \n139 | \n7 | \n16 | \n13 | \n38 | \n49 | \n19 | \n\n | ... | \n\n | 62 | \n\n | \n | \n | \n | \n | \n | \n | \n |
| 10 | \nMitchell | \n327 | \n96 | \n12 | \n17 | \n7 | \n29 | \n41 | \n17 | \n\n | ... | \n\n | 56 | \n\n | \n | \n | \n | \n | \n | \n | \n |
| 11 | \nCity Harrisville | \n389 | \n171 | \n16 | \n15 | \n18 | \n35 | \n49 | \n31 | \n\n | ... | \n\n | 77 | \n\n | \n | \n | 73 | \n\n | \n | \n | \n |
| 12 | \nTotals | \n9237 | \n3396 | \n371 | \n373 | \n317 | \n1102 | \n1221 | \n835 | \n0 | \n... | \n0 | \n1889 | \n0 | \n363 | \n219 | \n381 | \n321 | \n268 | \n160 | \n0 | \n
13 rows × 25 columns
\n| \n | 0 | \n1 | \n2 | \n3 | \n4 | \n5 | \n6 | \n7 | \n8 | \n9 | \n... | \n16 | \n17 | \n18 | \n19 | \n20 | \n21 | \n22 | \n23 | \n24 | \n25 | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n\n | \n | STATE | \n\n | \n | \n | CONGRESSIONAL | \n\n | \n | \n | ... | \n\n | COUNTY | \n\n | County Commissioner | \n\n | \n | \n | \n | \n | \n |
| 1 | \n\n | \n | Governor | \n\n | \n | \n | U.S. Senator | \n\n | \n | Congress-\\n1st Dist | \n... | \n\n | Road \\nCommission | \n\n | District #1 | \n\n | Distri\\nct #2 | \nDist \\n#3 | \nDist #4 | \n\n | Dist \\n#5 | \n
| 2 | \nNumber of Registered voters | \nPoll Book Totals | \nBrian Calley | \nPatrick Colbeck | \nJim Hines | \nBill Schuette | \nJohn James | \nSandy Pensler | \n\n | Jack Bergman | \n... | \n\n | Al Scully | \n\n | Daniel G. Gauthier | \nCraig M. Clemens | \nCraig Johnston | \nCarolyn Brummund | \nAdam Brege | \nDavid Bielusiak | \n\n |
| 3 | \n963 | \n439 | \n55 | \n26 | \n47 | \n164 | \n173 | \n111 | \n\n | 268 | \n... | \n\n | 271 | \n\n | 224 | \n76 | \n\n | \n | \n | \n | \n |
| 4 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 5 | \n923 | \n393 | \n40\\n23\\n45\\n158 | \n\n | \n | \n | 150\\n103 | \n\n | \n | 244 | \n... | \n\n | 244 | \n\n | 139 | \n143 | \n\n | \n | \n | \n | \n |
| 6 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 7 | \n1026 | \n349 | \n30\\n30\\n25\\n102 | \n\n | \n | \n | 95\\n84 | \n\n | \n | 159 | \n... | \n\n | 157 | \n\n | \n | \n | \n | \n | \n | \n | \n |
| 8 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 9 | \n1212 | \n423 | \n56\\n26\\n40\\n126 | \n\n | \n | \n | 104\\n131 | \n\n | \n | 208 | \n... | \n\n | 208 | \n\n | \n | \n | \n | 208 | \n\n | \n | \n |
| 10 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 11 | \n611 | \n180 | \n22\\n35\\n17\\n55 | \n\n | \n | \n | 73\\n45 | \n\n | \n | 108 | \n... | \n\n | 109 | \n\n | \n | \n | \n | \n | 81\\n42 | \n\n | \n |
| 12 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 13 | \n1142 | \n430 | \n45\\n90\\n29\\n101 | \n\n | \n | \n | 155\\n94 | \n\n | \n | 226 | \n... | \n\n | 226 | \n\n | \n | \n | 232 | \n\n | \n | \n | \n |
| 14 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 15 | \n884 | \n293 | \n38\\n36\\n27\\n109 | \n\n | \n | \n | 121\\n84 | \n\n | \n | 192 | \n... | \n\n | 184 | \n\n | \n | \n | \n | \n | 118\\n87 | \n\n | \n |
| 16 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 17 | \n626 | \n275 | \n31\\n20\\n32\\n104 | \n\n | \n | \n | 121\\n53 | \n\n | \n | 163 | \n... | \n\n | 152 | \n\n | \n | \n | 76 | \n\n | 69\\n31 | \n\n | \n |
| 18 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 19 | \n781 | \n208 | \n19\\n39\\n17\\n81 | \n\n | \n | \n | 90\\n63 | \n\n | \n | 149 | \n... | \n\n | 143 | \n\n | \n | \n | \n | 113 | \n\n | \n | \n |
| 20 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 21 | \n353 | \n139 | \n7\\n16\\n13\\n38 | \n\n | \n | \n | 49\\n19 | \n\n | \n | 62 | \n... | \n\n | 62 | \n\n | \n | \n | \n | \n | \n | \n | \n |
| 22 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 23 | \n327 | \n96 | \n12\\n17\\n7\\n29 | \n\n | \n | \n | 41\\n17 | \n\n | \n | 57 | \n... | \n\n | 56 | \n\n | \n | \n | \n | \n | \n | \n | \n |
| 24 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 25 | \n389 | \n171 | \n16\\n15\\n18\\n35 | \n\n | \n | \n | 49\\n31 | \n\n | \n | 78 | \n... | \n\n | 77 | \n\n | \n | \n | 73 | \n\n | \n | \n | \n |
26 rows × 26 columns
\n| \n | 0 | \n1 | \n2 | \n3 | \n4 | \n5 | \n6 | \n7 | \n8 | \n9 | \n... | \n14 | \n15 | \n16 | \n17 | \n18 | \n19 | \n20 | \n21 | \n22 | \n23 | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | Reg. of | \nRoad | \n\n | \n | \n | Distri | \nDist | \n\n | \n |
| 1 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | ... | \n\n | Deeds | \n\n | Commission | \n\n | District #1 | \nct #2 | \n#3 | \n\n | Dist #4 | \n
| 2 | \n\n | \n | \n | \n | \n | Governor | \n\n | \n | U.S. Senator | \n\n | ... | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 3 | \n\n | Number of Registered voters | \nPoll Book Totals | \nBrian Calley | \nPatrick Colbeck | \nJim Hines | \nBill Schuette | \nJohn James | \nSandy Pensler | \n\n | ... | \nMelissa A. Cordes | \n\n | Al Scully | \n\n | Daniel G. Gauthier | \nCraig M. Clemens | \nCraig Johnston | \nCarolyn Brummund | \nAdam Brege | \nDavid Bielusiak | \n
| 4 | \nAlcona | \n963 | \n439 | \n55 | \n26 | \n47 | \n164 | \n173 | \n111 | \n\n | ... | \n269 | \n\n | 271 | \n\n | 224 | \n76 | \n\n | \n | \n | \n |
| 5 | \nCaledonia | \n923 | \n393 | \n40 | \n23 | \n45 | \n158 | \n150 | \n103 | \n\n | ... | \n255 | \n\n | 244 | \n\n | 139 | \n143 | \n\n | \n | \n | \n |
| 6 | \nCurtis | \n1026 | \n349 | \n30 | \n30 | \n25 | \n102 | \n95 | \n84 | \n\n | ... | \n161 | \n\n | 157 | \n\n | \n | \n | \n | \n | \n | \n |
| 7 | \nGreenbush | \n1212 | \n423 | \n56 | \n26 | \n40 | \n126 | \n104 | \n131 | \n\n | ... | \n215 | \n\n | 208 | \n\n | \n | \n | \n | 208 | \n\n | \n |
| 8 | \nGustin | \n611 | \n180 | \n22 | \n35 | \n17 | \n55 | \n73 | \n45 | \n\n | ... | \n111 | \n\n | 109 | \n\n | \n | \n | \n | \n | 81 | \n42 | \n
| 9 | \nHarrisville | \n1142 | \n430 | \n45 | \n90 | \n29 | \n101 | \n155 | \n94 | \n\n | ... | \n244 | \n\n | 226 | \n\n | \n | \n | 232 | \n\n | \n | \n |
| 10 | \nHawes | \n884 | \n293 | \n38 | \n36 | \n27 | \n109 | \n121 | \n84 | \n\n | ... | \n193 | \n\n | 184 | \n\n | \n | \n | \n | \n | 118 | \n87 | \n
| 11 | \nHaynes | \n626 | \n275 | \n31 | \n20 | \n32 | \n104 | \n121 | \n53 | \n\n | ... | \n161 | \n\n | 152 | \n\n | \n | \n | 76 | \n\n | 69 | \n31 | \n
| 12 | \nMikado | \n781 | \n208 | \n19 | \n39 | \n17 | \n81 | \n90 | \n63 | \n\n | ... | \n147 | \n\n | 143 | \n\n | \n | \n | \n | 113 | \n\n | \n |
| 13 | \nMillen | \n353 | \n139 | \n7 | \n16 | \n13 | \n38 | \n49 | \n19 | \n\n | ... | \n66 | \n\n | 62 | \n\n | \n | \n | \n | \n | \n | \n |
| 14 | \nMitchell | \n327 | \n96 | \n12 | \n17 | \n7 | \n29 | \n41 | \n17 | \n\n | ... | \n60 | \n\n | 56 | \n\n | \n | \n | \n | \n | \n | \n |
| 15 | \nCity Harrisville | \n389 | \n171 | \n16 | \n15 | \n18 | \n35 | \n49 | \n31 | \n\n | ... | \n81 | \n\n | 77 | \n\n | \n | \n | 73 | \n\n | \n | \n |
| 16 | \nTotals | \n9237 | \n3396 | \n371 | \n373 | \n317 | \n1102 | \n1221 | \n835 | \n0 | \n... | \n1963 | \n0 | \n1889 | \n0 | \n363 | \n219 | \n381 | \n321 | \n268 | \n160 | \n
17 rows × 24 columns
\n