Add compilation timeout

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 2023-06-12-1
+
+* Timeout long LaTeX compilations and prevent zombie processes (the timeout is of 100 seconds for now)
+
 ## 2022-12-07-1
 
 * Add clearer input spec (payload) validation

Pipfile

@@ -16,7 +16,7 @@ flask = "*"
 fclist-cffi = "*"
 gunicorn = "*"
 texlivemetadata = ">=0.1.3"
-hy = ">=1.0a4"
+hy = ">=0.26.0"
 urllib3 = ">=1.26"
 pyzmq = "*"
 msgpack = "*"

TODO.md

@@ -33,6 +33,7 @@
         * https://tug.org/FontCatalogue/
     * Tell font types: T1, OpenType, TrueType, etc.
     * Add usage instructions for each? (in the generated doc?)
+    * TODO Dynamically pull fonts from https://fonts.google.com/?
 * Tmp site https://nicedoc.io/ytotech/latex-on-http
 * Find/create a std Hy formatter (like black for Python)
     * Uniformize Hy code formatting

container/python-debian.Dockerfile

@@ -19,7 +19,8 @@ RUN apt-get update -qq && apt-get install -y \
     && apt-get autoremove --purge -y && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Update pip and install Pipenv.
-RUN pip3 install -U \
+# Yes --break-system-packages, we don't care your EXTERNALLY-MANAGED.
+RUN pip3 install -U --break-system-packages \
   pip \
   pipenv
 

container/tl-distrib-alpine.Dockerfile

@@ -11,7 +11,7 @@
 #--------------------------------
 # Start from our docker-texlive distribution.
 # https://hub.docker.com/r/yoant/docker-texlive
-FROM yoant/docker-texlive:alpine
+FROM yoant/docker-texlive:alpine-2023
 LABEL maintainer="Yoan Tournade <[email protected]>"
 
 

container/tl-distrib-debian.Dockerfile

@@ -12,15 +12,15 @@
 #--------------------------------
 # Start from our docker-texlive distribution.
 # https://hub.docker.com/r/yoant/docker-texlive
-FROM yoant/docker-texlive:debian
+FROM yoant/docker-texlive:debian-2023
 LABEL maintainer="Yoan Tournade <[email protected]>"
 
 
 #--------------------------------
 # Install fonts.
 #--------------------------------
 
-RUN echo "deb http://deb.debian.org/debian stretch contrib non-free" >> /etc/apt/sources.list
+RUN echo "deb http://deb.debian.org/debian bookworm contrib non-free" >> /etc/apt/sources.list
 
 # Accepts Microsoft EULA.
 RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections

latexonhttp/api/builds.py

@@ -12,7 +12,7 @@
 import json
 import glom
 import cerberus
-from flask import Blueprint, request, jsonify, Response
+from flask import Blueprint, request
 from latexonhttp.compiler import (
     latexToPdf,
     AVAILABLE_LATEX_COMPILERS,
@@ -135,7 +135,7 @@ def compiler_latex():
             request.args.to_dict(True), request.args.to_dict(False)
         )
         if error:
-            return jsonify(error), 400
+            return error, 400
 
     # Support for multipart/form-data requests.
     if request.content_type and "multipart/form-data" in request.content_type:
@@ -145,16 +145,16 @@ def compiler_latex():
         logger.info(pprint.pformat(request.form))
         input_spec, error = parse_multipart_resources_spec(request.form, request.files)
         if error:
-            return jsonify(error), 400
+            return error, 400
 
     if not input_spec:
         input_spec_mode = "json"
         input_spec, error = parse_json_resources_spec(request.get_json())
         if error:
-            return jsonify(error), 400
+            return error, 400
 
     if not input_spec:
-        return jsonify({"error": "MISSING_COMPILATION_SPECIFICATION"}), 400
+        return {"error": "MISSING_COMPILATION_SPECIFICATION"}, 400
 
     # Payload validations.
     logger.info(request.content_type)
@@ -164,19 +164,17 @@ def compiler_latex():
 
     if not input_spec_validator.validate(input_spec):
         return (
-            Response(
-                json.dumps(
-                    {
-                        "error": "INVALID_PAYLOAD_SHAPE",
-                        "shape_errors": input_spec_validator.errors,
-                        "input_spec_mode": input_spec_mode,
-                        "input_spec": input_spec,
-                    },
-                    cls=JSONInputSpecEncoderForDebug,
-                ),
-                content_type="application/json",
+            json.dumps(
+                {
+                    "error": "INVALID_PAYLOAD_SHAPE",
+                    "shape_errors": input_spec_validator.errors,
+                    "input_spec_mode": input_spec_mode,
+                    "input_spec": input_spec,
+                },
+                cls=JSONInputSpecEncoderForDebug,
             ),
             400,
+            {"Content-Type": "application/json"},
         )
 
     # High-level normalizsation.
@@ -204,19 +202,17 @@ def compiler_latex():
 
     # - resources (mandatory, must be an array).
     if not "resources" in input_spec:
-        return jsonify({"error": "MISSING_RESOURCES"}), 400
+        return {"error": "MISSING_RESOURCES"}, 400
     if type(input_spec["resources"]) != list:
-        return jsonify({"error": "RESOURCES_SPEC_MUST_BE_A_LIST"}), 400
+        return {"error": "RESOURCES_SPEC_MUST_BE_A_LIST"}, 400
 
     # - compiler
     if compilerName not in AVAILABLE_LATEX_COMPILERS:
         return (
-            jsonify(
-                {
-                    "error": "INVALID_COMPILER",
-                    "available_compilers": AVAILABLE_LATEX_COMPILERS,
-                }
-            ),
+            {
+                "error": "INVALID_COMPILER",
+                "available_compilers": AVAILABLE_LATEX_COMPILERS,
+            },
             400,
         )
 
@@ -226,12 +222,10 @@ def compiler_latex():
         not in AVAILABLE_BIBLIOGRAPHY_COMMANDS
     ):
         return (
-            jsonify(
-                {
-                    "error": "INVALID_BILIOGRAPHY_COMMAND",
-                    "available_commands": AVAILABLE_BIBLIOGRAPHY_COMMANDS,
-                }
-            ),
+            {
+                "error": "INVALID_BILIOGRAPHY_COMMAND",
+                "available_commands": AVAILABLE_BIBLIOGRAPHY_COMMANDS,
+            },
             400,
         )
 
@@ -245,7 +239,7 @@ def compiler_latex():
     # - Prefetch checks (paths, main document, ...);
     errors = check_resources_prefetch(normalized_resources)
     if errors:
-        return jsonify(errors[0]), 400
+        return errors[0], 400
 
     # -------------
     # Fetching, post-fetch normalization and checks, filesystem creation.
@@ -274,7 +268,7 @@ def on_fetched(resource, data):
             normalized_resources, on_fetched, get_from_cache=get_resource_from_cache
         )
         if error:
-            return jsonify(error), 400
+            return error, 400
         # TODO
         # - Process build global signature/hash (compiler, resource hashes, other options...)
 
@@ -302,9 +296,7 @@ def on_fetched(resource, data):
         if not latexToPdfOutput["pdf"]:
             error_compilation = latexToPdfOutput["logs"]
             return (
-                jsonify(
-                    {"error": "COMPILATION_ERROR", "logs": latexToPdfOutput["logs"]}
-                ),
+                {"error": "COMPILATION_ERROR", "logs": latexToPdfOutput["logs"]},
                 400,
             )
         # TODO Also return compilation logs here.
@@ -319,10 +311,10 @@ def on_fetched(resource, data):
 
         # TODO Output cache management.
 
-        return Response(
+        return (
             latexToPdfOutput["pdf"],
-            status="201",
-            headers={
+            201,
+            {
                 "Content-Type": "application/pdf",
                 # TODO Pass an option for returning as attachment (instead of inline, which is the default).
                 "Content-Disposition": "inline;filename={}".format(
@@ -339,14 +331,17 @@ def on_fetched(resource, data):
         # TODO Report error to Sentry (create a hook for custom code?).
 
         error_in_try_block = e
+        logger.exception(e)
+        return ({"error": "SERVER_ERROR"}, 500)
 
     finally:
         # -------------
         # Cleanup.
         # -------------
 
-        # TODO Option to let workspace on failure.
-        let_workspace_on_error = True
+        # TODO Option to let workspace on failure
+        # from env.
+        let_workspace_on_error = False
 
         if let_workspace_on_error is False or (
             error_in_try_block is None and error_compilation is None

latexonhttp/api/caches.py

@@ -9,7 +9,7 @@
     :license: AGPL, see LICENSE for more details.
 """
 import logging
-from flask import Blueprint, request, jsonify
+from flask import Blueprint, request
 from latexonhttp.caching.resources import (
     get_cache_metadata_snapshot,
     are_resources_in_cache,
@@ -33,40 +33,38 @@
 def resources_metadata():
     is_ok, cache_response = get_cache_metadata_snapshot()
     if not is_ok:
-        return (jsonify(cache_response), 500)
-    return (jsonify(map_cache_metadata_for_public(cache_response)), 200)
+        return (cache_response, 500)
+    return (map_cache_metadata_for_public(cache_response), 200)
 
 
 @caches_app.route("/resources", methods=["DELETE"])
 def resources_reset_cache():
     is_ok, cache_response = reset_cache()
     if not is_ok:
-        return (jsonify(cache_response), 500)
+        return (cache_response, 500)
     return "", 204
 
 
 @caches_app.route("/resources/check_cached", methods=["POST"])
 def resources_check_cached():
     payload = request.get_json()
     if not payload:
-        return jsonify("MISSING_PAYLOAD"), 400
+        return {"error": "MISSING_PAYLOAD"}, 400
     if not "resources" in payload:
-        return jsonify("MISSING_RESOURCES"), 400
+        return {"error": "MISSING_RESOURCES"}, 400
     for resource in payload["resources"]:
         if not "hash" in resource:
-            return jsonify("MISSING_RESOURCE_HASH"), 400
+            return {"error": "MISSING_RESOURCE_HASH"}, 400
     is_ok, cache_response = are_resources_in_cache(payload["resources"])
     if not is_ok:
-        return (jsonify(cache_response), 500)
+        return (cache_response, 500)
     return (
-        jsonify(
-            {
-                "resources": {
-                    resource["hash"]: {"hit": resource["hit"]}
-                    for resource in cache_response
-                }
+        {
+            "resources": {
+                resource["hash"]: {"hit": resource["hit"]}
+                for resource in cache_response
             }
-        ),
+        },
         200,
     )
 

latexonhttp/api/fonts.py

@@ -7,7 +7,7 @@
     :copyright: (c) 2019 Yoan Tournade.
     :license: AGPL, see LICENSE for more details.
 """
-from flask import Blueprint, jsonify
+from flask import Blueprint
 from fclist import fclist
 
 fonts_app = Blueprint("fonts", __name__)
@@ -21,4 +21,4 @@ def fonts_list():
             {"family": font.family, "name": font.fullname, "styles": list(font.style)}
         )
     # TODO Group by families?
-    return (jsonify({"fonts": fonts}), 200)
+    return ({"fonts": fonts}, 200)

latexonhttp/api/packages.py

@@ -7,7 +7,7 @@
     :copyright: (c) 2019 Yoan Tournade.
     :license: AGPL, see LICENSE for more details.
 """
-from flask import Blueprint, jsonify, url_for
+from flask import Blueprint, url_for
 from texlivemetadata import list_installed_packages, get_package_info, get_ctan_link
 
 packages_app = Blueprint("packages", __name__)
@@ -30,15 +30,15 @@ def packages_list():
         }
         for package in list_installed_packages()
     ]
-    return (jsonify({"packages": packages}), 200)
+    return ({"packages": packages}, 200)
 
 
 @packages_app.route("/<package_name>", methods=["GET"])
 def packages_info(package_name):
     package_info = get_package_info(package_name)
     if not package_info:
-        return (jsonify("Package not found"), 404)
+        return ({"error": "Package not found"}, 404)
     package_info = {**package_info, "url_ctan": get_ctan_link(package_info["package"])}
     if "cat-date" in package_info:
         package_info["cat-date"] = (package_info["cat-date"].isoformat(),)
-    return (jsonify({"package": package_info}), 200)
+    return ({"package": package_info}, 200)

latexonhttp/api/texlive.py

@@ -7,12 +7,12 @@
     :copyright: (c) 2019 Yoan Tournade.
     :license: AGPL, see LICENSE for more details.
 """
-from flask import Blueprint, jsonify
+from flask import Blueprint
 from latexonhttp.utils.texlive import get_texlive_version_spec
 
 texlive_app = Blueprint("texlive", __name__)
 
 
 @texlive_app.route("information", methods=["GET"])
 def texlive_installation_information():
-    return (jsonify(get_texlive_version_spec()), 200)
+    return (get_texlive_version_spec(), 200)

latexonhttp/app.py

@@ -12,7 +12,7 @@
 import logging.config
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
-from flask import Flask, request, jsonify
+from flask import Flask, request
 from flask_cors import CORS
 from latexonhttp.api.builds import builds_app
 from latexonhttp.api.fonts import fonts_app
@@ -72,14 +72,12 @@ def hello():
     # TODO Return an OpenAPI specification
     # https://github.com/OAI/OpenAPI-Specification
     return (
-        jsonify(
-            {
-                "message": "Welcome to the LaTeX-On-HTTP API",
-                "version": get_api_version(),
-                "source": "https://github.com/YtoTech/latex-on-http",
-                "documentation": "https://github.com/YtoTech/latex-on-http",
-                "texlive_version": get_texlive_version_spec()["texlive"]["version"],
-            }
-        ),
+        {
+            "message": "Welcome to the LaTeX-On-HTTP API",
+            "version": get_api_version(),
+            "source": "https://github.com/YtoTech/latex-on-http",
+            "documentation": "https://github.com/YtoTech/latex-on-http",
+            "texlive_version": get_texlive_version_spec()["texlive"]["version"],
+        },
         200,
     )