Packaging

.travis.yml

@@ -4,29 +4,30 @@ language: python
 python:
   - 3.6
   - 3.7
+  - 3.8
 
 before_install:
-  # Install node.js following instructions from
-  #   https://nodejs.org/en/download/package-manager/
-  - sudo apt install curl
-  - curl -sL https://deb.nodesource.com/setup_11.x | sudo bash -
-  - sudo apt install nodejs
+  # Use the pre-installed nvm version to update node
+  # Apparently Travis comes with nvm even on the python containers
+  - nvm install v12.18.1
+  - nvm use v12.18.1
+  - node -v
+  - npm version
 
 install:
-  # Install node.js dependencies
-  - npm install
-  # Install python dependencies
-  - pip install -r requirements-dev.txt
+  # Install package, while updating dependencies
+  - pip install --upgrade --upgrade-strategy eager .[dev]
 
 script:
   # Run all pytest unit tests
-  - python -m pytest -v tests --cov readabilipy --cov-report term-missing --benchmark-disable
+  - cd tests && python -m pytest -v . --cov readabilipy --cov-report term-missing --benchmark-disable && cd ..
   # Run pyflakes for error detection
   - pyflakes *.py readabilipy tests
   # Check PEP8 compliance (ignoring long lines)
   - pycodestyle --statistics --ignore=E501 --count *.py readabilipy tests
   # Run pylint for stricter error checking
-  - pylint readabilipy tests
+  - pylint readabilipy
+  - pylint ./tests/*.py
 
 after_success:
   # Upload results to coveralls.io

CHANGELOG.md

@@ -0,0 +1,5 @@
+# Change Log
+
+## Version 0.1.0
+
+* Initial release of Python package to PyPI.

MANIFEST.in

@@ -0,0 +1,2 @@
+include readabilipy/javascript/*.js
+include readabilipy/javascript/package.json

Makefile

@@ -0,0 +1,108 @@
+# Makefile for easier installation and cleanup.
+#
+# Uses self-documenting macros from here:
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+MAKEFLAGS += --warn-undefined-variables --no-builtin-rules
+
+PACKAGE=readabilipy
+DOC_DIR=./docs
+VENV_DIR=/tmp/rdpy_venv
+TEST_DIR=./tests
+
+.PHONY: help
+
+.DEFAULT_GOAL := help
+
+# Display a help message when called without target, using the ## comments
+help:
+	@grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) |\
+		 awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m\
+		 %s\n", $$1, $$2}'
+
+################
+# Installation #
+################
+
+.PHONY: install
+
+install: ## Install for the current user using the default python command
+	python setup.py build_ext --inplace
+	python setup.py install --user
+
+
+################
+# Distribution #
+################
+
+.PHONY: release dist
+
+release: ## Make a release
+	python make_release.py
+
+dist: ## Make Python source distribution
+	python setup.py sdist bdist_wheel
+
+
+###########
+# Testing #
+###########
+
+.PHONY: test
+
+test: venv ## Run unit tests
+	source $(VENV_DIR)/bin/activate && cd $(TEST_DIR) && \
+		python -m pytest -v . --cov readabilipy \
+		--cov-report term-missing --benchmark-disable
+	source $(VENV_DIR)/bin/activate && pyflakes *.py readabilipy $(TEST_DIR)
+	source $(VENV_DIR)/bin/activate && pycodestyle --statistics \
+		--ignore=E501 --count *.py readabilipy $(TEST_DIR)
+	source $(VENV_DIR)/bin/activate && pylint readabilipy $(TEST_DIR)/*.py
+
+#################
+# Documentation #
+#################
+
+.PHONY: docs
+
+docs: install ## Build documentation with Sphinx
+	exit; # not implemented
+	source $(VENV_DIR)/bin/activate && m2r README.md && mv README.rst $(DOC_DIR)
+	source $(VENV_DIR)/bin/activate && m2r CHANGELOG.md && mv CHANGELOG.rst $(DOC_DIR)
+	cd $(DOC_DIR) && \
+		rm source/* && \
+		source $(VENV_DIR)/bin/activate && \
+		sphinx-apidoc -H 'ReadabiliPy API Documentation' -o source ../$(PACKAGE) && \
+		touch source/AUTOGENERATED
+	$(MAKE) -C $(DOC_DIR) html
+
+#######################
+# Virtual environment #
+#######################
+
+.PHONY: venv
+
+venv: $(VENV_DIR)/bin/activate
+
+$(VENV_DIR)/bin/activate: setup.py
+	test -d $(VENV_DIR) || python -m venv $(VENV_DIR)
+	source $(VENV_DIR)/bin/activate && pip install .[dev]
+	touch $(VENV_DIR)/bin/activate
+
+############
+# Clean up #
+############
+
+.PHONY: clean
+
+clean: ## Clean build dist and egg directories left after install
+	rm -rf ./dist
+	rm -rf ./build
+	rm -rf ./$(PACKAGE).egg-info
+	rm -rf $(VENV_DIR)
+	rm -f MANIFEST
+	rm -rf $(PACKAGE)/javascript/node_modules
+	find . -type f -iname '*.pyc' -delete
+	find . -type d -name '__pycache__' -empty -delete

README.md

@@ -1,67 +1,115 @@
-ReadabiliPy
+# ReadabiliPy
+
 [![Build Status](https://travis-ci.org/alan-turing-institute/ReadabiliPy.svg?branch=master)](https://travis-ci.org/alan-turing-institute/ReadabiliPy)
 [![Coverage Status](https://coveralls.io/repos/github/alan-turing-institute/ReadabiliPy/badge.svg?branch=master)](https://coveralls.io/github/alan-turing-institute/ReadabiliPy?branch=master)
-===========
-A Node and filesystem dependent Python wrapper for Mozilla's [Readability.js](https://github.com/mozilla/readability) Node.js package.
-
-This package also augments the output of `Readability.js` to also return a list of plain text representations of article paragraphs.
-
-## Package contents
-- `javascript` folder
-  - `Readability.js`: Taken unaltered from commit [876c81f](https://github.com/mozilla/readability/tree/876c81f710711ba2afb36dd83889d4c5b4fc2743) of Mozilla's [Readability.js](https://github.com/mozilla/readability) Node.js package.
-
-  - `ExtractArticle.js`: A Node.js script that reads a file containing a HTML snippet (does not have to be a full document), parses it using [jsdom](https://github.com/jsdom/jsdom), attempts to extract an article using `Readability.parse()` and writes the output to a JSON file.
-    - Usage: `node ExtractArticle.js -i <input_file> -o <output_file>`
-    - Arguments:
-      - `-i`: The path to an input file containing full or partial HTML as text.
-      - `-o`: The path to the file the output article JSON data should be written to.
-
-- `readabilipy` folder
-  - `simple_json.py` file containing the function `simple_json_from_html_string()`.
-    - Usage:
-    ```python
-    from readabilipy import simple_json_from_html_string
-
-    article = simple_json_from_html_string(html_string, content_digests=False, node_indexes=False, use_readability=False)
-    ```
-    - The function returns a dictionary with the following fields:
-      - `title`: The article title
-      - `byline`: Author information
-      - `content`: A simplified HTML representation of the article, with all article text contained in paragraph elements.
-      - `plain_content`: A "plain" version of the simplified `Readability.js` article HTML present in the `content` field. This attempts to retain only the plain text content of the article, while preserving the HTML structure.
-      - `plain_text`: A list containing plain text representations of each paragraph (`<p>`) or list (`<ol>` or `<ul>`) present in the simplified `Readability.js` article HTML in the `content` field. Each paragraph or list is represented as a single string. List strings look like `"* item 1, * item 2, * item 3,"` for both ordered and unordered lists (note the trailing `,`).
-    - All fields are guaranteed to be present. If individual fields are missing from the output of `Readability.js`, the value of these fields will be `None`. If no article data is returned by `Readability.js`, the value of all fields will be `None`.
-    - All text in the `plain_content` and `plain_text` fields is encoded as unicode normalised using the "NFKC" normal form. This normal form is used to try and ensure as much as possible that things that appear visually the same are encoded with the same unicode representation (the K part) and characters are represented as a single composite character where possible (the C part).
-      - An optional `content_digests` flag can be passed to the Python wrapper. When this is set to `True`, each HTML element in the `plain_content` field has a `data-content-digest` attribute, which holds the SHA-256 hash of its plain text content. For "leaf" nodes (containing only plain text in the output), this is the SHA-256 hash of their plain text content. For nodes containing other nodes, this is the SHA-256 hash of the concatenated SHA-256 hashes of their child nodes.
-      - An optional `node_indexes` flag can be passed to the Python wrapper. When this is set to `True`, each HTML element in the `plain_content` field has a `data-node-indexes` attribute, which holds a hierarchical index describing the location of element within the `plain_content` HTML structure.
-      - An optional `use_readability` flag can be passed to the Python wrapper. When this is set to `True`, Mozilla's `Readability.js` will be used as the parser. If it is set to `False` then the pure-python parser in `plain_html.py` will be used instead.
-
-  - `simple_tree.py` file containing the function `simple_tree_from_html_string()`.
-    - Usage:
-    ```python
-    from readabilipy import simple_tree_from_html_string
-
-    article = simple_tree_from_html_string(html_string)
-    ```
-    - The function returns a `bs4.BeautifulSoup` object containing a cleaned, parsed HTML tree.
-
-
-- `extract_article.py`: A Python script that uses `readabilipy.simple_json_from_html_string()` to extract the augmented readable article data.
-  - Usage: `python extract_article.py -i <input_file> -o <output_file> [-c] [-n] [-p]`
-  - All arguments have long and short form versions.
-    - `-i` / `--input-file`: The path to an input file containing full or partial HTML as text.
-    - `-o` / `--output-file`: The path to the file the output article JSON data should be written to.
-    - `-c` / `--content-digests` (optional): If this flag is provided, then `simple_json_from_html_string()` is called with `content_digests=True`
-    - `-n` / `--node-indexes` (optional): If this flag is provided, then `simple_json_from_html_string()` is called with `node_indexes=True`
-    - `-p` / `--use-python-parser` (optional): If this flag is provided then `simple_json_from_html_string()` is called with `use_readability=False`
 
+`ReadabiliPy` contains a Python wrapper for Mozilla's [Readability.js](https://github.com/mozilla/readability) Node.js package, as well as article extraction routines written in pure Python.
+
+This package augments the output of `Readability.js` to also return a list of plain text representations of article paragraphs.
+
+`ReadabiliPy` comes with a handy command line application: ``readabilipy``.
 
 ## Installation
-1. [Install Node.js](https://nodejs.org/en/download/)
-2. Install the node part of this package by running `npm install`.
-3. Install the requirements for the Python part of this package by running:
-  - As a user of this project `pip install -r requirements.txt`
-  - As a developer of this project `pip install -r requirements-dev.txt`
-
-## Testing
-1. Run the tests with `python -m pytest -vv`
+
+To use the `Readability.js` wrapper you need to have a working [Node.js](https://nodejs.org/en/download/) installation of version 10 or higher.
+Make sure to install Node.js before installing this package, as this ensures Readability.js will be installed.
+If you only want to use the Python-based article extraction, you **do not need** to install Node.js.
+
+`ReadabiliPy` can be installed simply from PyPI:
+
+```
+$ pip install readabilipy
+```
+
+Note that to update to a new version of `Readability.js` you can simply reinstall `ReadabiliPy`.
+
+## Usage
+
+`ReadabiliPy` can be used either as a command line application or as a Python library.
+
+### Command line application
+
+The ``readabilipy`` command line application can be used to extract an article from an HTML source file.
+
+For example, if you have the article saved as ``input.html`` in the current directory then you can run:
+
+```
+$ readabilipy -i ./input.html -o article.json
+```
+
+The extracted article can then be found in the ``article.json`` file. By default ReadabiliPy will use the Readability.js functionality to extract the article, provided this is available. If instead you'd like to use the Python-based extraction, run:
+
+```
+$ readabilipy -p -i ./input.html -o article.json
+```
+
+The complete help text of the command line application is as follows:
+
+```
+$ readabilipy -h
+usage: readabilipy [-h] -i INPUT_FILE -o OUTPUT_FILE [-c] [-n] [-p] [-V]
+
+Extract article data from a HTML file using either Mozilla's Readability.js
+package or a simplified python-only alternative.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -i INPUT_FILE, --input-file INPUT_FILE
+                        Path to input file containing HTML.
+  -o OUTPUT_FILE, --output-file OUTPUT_FILE
+                        Path to file to output the article data to as JSON.
+  -c, --content-digests
+                        Add a 'data-content-digest' attribute containing a
+                        SHA256-based digest of the element's contents to each
+                        HTML element in the plain_content output.
+  -n, --node-indexes    Add a 'data-node-index' attribute containing a
+                        hierarchical representation of the element's position
+                        in the HTML structure each HTML element in the
+                        plain_content output.
+  -p, --use-python-parser
+                        Use the pure-python 'plain_html' parser included in
+                        this project rather than Mozilla's Readability.js.
+  -V, --version         Show version and exit
+```
+
+## Library
+
+ReadabiliPy can also be used as a Python package.
+The main routine is called ``simple_json_from_html_string`` and expects the HTML article as a string.
+Here is an example of extracting an article after downloading the page using [requests](https://requests.readthedocs.io/en/master/):
+
+```python
+>>> import requests
+>>> from readabilipy import simple_json_from_html_string
+>>> req = requests.get('https://en.wikipedia.org/wiki/Readability')
+>>> article = simple_json_from_html_string(req.text, use_readability=True)
+```
+
+Note that you need to use the flag ``use_readability=True`` to use Readability.js, otherwise the Python-based extraction is used.
+
+The ``simple_json_from_html_string`` function returns a dictionary with the following fields:
+
+ - `title`: The article title
+ - `byline`: Author information
+ - `content`: A simplified HTML representation of the article, with all article text contained in paragraph elements.
+ - `plain_content`: A "plain" version of the simplified `Readability.js` article HTML present in the `content` field. This attempts to retain only the plain text content of the article, while preserving the HTML structure.
+ - `plain_text`: A list containing plain text representations of each paragraph (`<p>`) or list (`<ol>` or `<ul>`) present in the simplified `Readability.js` article HTML in the `content` field. Each paragraph or list is represented as a single string. List strings look like `"* item 1, * item 2, * item 3,"` for both ordered and unordered lists (note the trailing `,`).
+
+Note further that:
+
+- All fields are guaranteed to be present. If individual fields are missing from the output of `Readability.js`, the value of these fields will be `None`. If no article data is returned by `Readability.js`, the value of all fields will be `None`.
+- All text in the `plain_content` and `plain_text` fields is encoded as unicode normalised using the "NFKC" normal form. This normal form is used to try and ensure as much as possible that things that appear visually the same are encoded with the same unicode representation (the K part) and characters are represented as a single composite character where possible (the C part).
+- An optional `content_digests` flag can be passed to the Python wrapper. When this is set to `True`, each HTML element in the `plain_content` field has a `data-content-digest` attribute, which holds the SHA-256 hash of its plain text content. For "leaf" nodes (containing only plain text in the output), this is the SHA-256 hash of their plain text content. For nodes containing other nodes, this is the SHA-256 hash of the concatenated SHA-256 hashes of their child nodes.
+- An optional `node_indexes` flag can be passed to the Python wrapper. When this is set to `True`, each HTML element in the `plain_content` field has a `data-node-indexes` attribute, which holds a hierarchical index describing the location of element within the `plain_content` HTML structure.
+- An optional `use_readability` flag can be passed to the Python wrapper. When this is set to `True`, Mozilla's `Readability.js` will be used as the parser. If it is set to `False` then the pure-python parser in `plain_html.py` will be used instead.
+
+The second top-level function exported by ReadabiliPy is ``simple_tree_from_html_string``. This returns a cleaned, parsed HTML tree of the article as a [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) object.
+
+## Notes
+
+License: MIT License, see the `LICENSE` file.
+
+Copyright (c) 2018, The Alan Turing Institute
+
+If you encounter any issues or have any suggestions for improvement, please open an issue [on Github](https://github.com/alan-turing-institute/ReadabiliPy).
+You're helping to make this project better for everyone!