Refactor: LaTeX processing in external Docker container

.github/workflows/push.yml

@@ -27,6 +27,8 @@ env:
   DF_ENCRYPTION_DETERMINISTIC_KEY: "anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG"
   DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10"
   DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0"
+  LATEX_CONTAINER_NAME: doubtfire-texlive
+  LATEX_BUILD_PATH: /texlive/shell/latex_build.sh
 
 jobs:
   unit-tests:
@@ -49,21 +51,53 @@ jobs:
         uses: actions/checkout@v4
       - name: Set up docker buildx
         uses: docker/setup-buildx-action@v3
+      - name: Build TexLive image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: texlive.Dockerfile
+          push: false
+          load: true
+          tags: doubtfire-texlive-development:local
+          cache-from: type=gha,scope=texlive
+          cache-to: type=gha,mode=max,scope=texlive
       - name: Build base doubtfire-api development image
         uses: docker/build-push-action@v5
         with:
           context: .
           push: false
           load: true
           tags: doubtfire-api-development:local
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=doubtfire-api
+          cache-to: type=gha,mode=max,scope=doubtfire-api
+      - name: Start TexLive service
+        uses: addnab/docker-run-action@v3
+        with:
+          image: doubtfire-texlive-development:local
+          options: >
+            --name ${{ env.LATEX_CONTAINER_NAME }}
+            -v ${{ github.workspace }}/student-work:/student-work
+            -v ${{ github.workspace }}/public/assets/images:/doubtfire/public/assets/images
+            -v ${{ github.workspace }}/test_files:/doubtfire/test_files
+            -v ${{ github.workspace }}/tmp/rails-latex:/workdir/texlive-latex
+            --detach
+          run: /bin/sh -c "while sleep 5000; do :; done"
+      - name: Test TexLive container
+        uses: addnab/docker-run-action@v3
+        with:
+          image: doubtfire-api-development:local
+          options: >
+            -v ${{ github.workspace }}:/doubtfire
+            -v /var/run/docker.sock:/var/run/docker.sock
+          run: docker exec -t ${{ env.LATEX_CONTAINER_NAME }} lualatex -v
       - name: Populate database
         uses: addnab/docker-run-action@v3
         with:
           image: doubtfire-api-development:local
           options: >
             -v ${{ github.workspace }}:/doubtfire
+            -v ${{ github.workspace }}/student-work:/student-work
+            -v /var/run/docker.sock:/var/run/docker.sock
             -e RAILS_ENV
             -e DF_STUDENT_WORK_DIR
             -e DF_INSTITUTION_HOST
@@ -81,13 +115,17 @@ jobs:
             -e DF_ENCRYPTION_DETERMINISTIC_KEY
             -e DF_ENCRYPTION_KEY_DERIVATION_SALT
             -e DF_REDIS_SIDEKIQ_URL
+            -e LATEX_CONTAINER_NAME
+            -e LATEX_BUILD_PATH
           run: bundle exec rake db:populate
       - name: Run unit tests
         uses: addnab/docker-run-action@v3
         with:
           image: doubtfire-api-development:local
           options: >
             -v ${{ github.workspace }}:/doubtfire
+            -v ${{ github.workspace }}/student-work:/student-work
+            -v /var/run/docker.sock:/var/run/docker.sock
             -e RAILS_ENV
             -e DF_STUDENT_WORK_DIR
             -e DF_INSTITUTION_HOST
@@ -105,4 +143,8 @@ jobs:
             -e DF_ENCRYPTION_DETERMINISTIC_KEY
             -e DF_ENCRYPTION_KEY_DERIVATION_SALT
             -e DF_REDIS_SIDEKIQ_URL
+            -e LATEX_CONTAINER_NAME
+            -e LATEX_BUILD_PATH
           run: TERM=xterm bundle exec rails test
+      - name: Stop TexLive service
+        run: docker rm -f ${{ env.LATEX_CONTAINER_NAME }}

Dockerfile

@@ -34,8 +34,6 @@ RUN apt-get update \
 WORKDIR /doubtfire
 
 COPY ./.ci-setup/ /doubtfire/.ci-setup/
-RUN ./.ci-setup/texlive-install.sh
-ENV PATH /tmp/texlive/bin/x86_64-linux:/tmp/texlive/bin/aarch64-linux:$PATH
 
 RUN gem install bundler -v '2.4.5'
 

app/helpers/latex_helper.rb

@@ -0,0 +1,7 @@
+module LatexHelper
+  def generate_pdf(template:)
+    raise "LATEX_CONTAINER_NAME is not set" if ENV['LATEX_CONTAINER_NAME'].nil?
+    raise "LATEX_BUILD_PATH is not set" if ENV['LATEX_BUILD_PATH'].nil?
+    render_to_string(template: template, layout: true)
+  end
+end

app/models/pdf_generation/project_compile_portfolio_module.rb

@@ -35,6 +35,8 @@ def compress_portfolio
 
     # This class scaffolds the creation of the portfolio - mapping the required data into the erb template
     class ProjectAppController < ApplicationController
+      include LatexHelper
+
       attr_accessor :student,
                     :project,
                     :base_path,
@@ -66,8 +68,10 @@ def init(project, is_retry)
       end
 
       def make_pdf
-        render_to_string(template: '/portfolio/portfolio_pdf', layout: true)
+        logger.debug "Running make_pdf: (portfolio)"
+        generate_pdf(template: "/portfolio/portfolio_pdf")
       end
+
     end
 
     # Create the portfolio for this project

app/models/task.rb

@@ -1062,6 +1062,8 @@ def in_process_files_for_task(is_retry)
   end
 
   class TaskAppController < ApplicationController
+    include LatexHelper
+
     attr_accessor :task
     attr_accessor :files
     attr_accessor :base_path
@@ -1087,7 +1089,7 @@ def make_pdf
         end
       end
       logger.debug "Preprocessing complete, rendering file."
-      render_to_string(template: '/task/task_pdf', layout: true)
+      generate_pdf(template: "/task/task_pdf")
     end
   end
 

app/views/layouts/application.pdf.erbtex

@@ -1,4 +1,17 @@
-<% @latex_config={ :recipe => [ {:command => "lualatex",:runs => 2} ], :supporting => Rails.root.join('app', 'views', 'layouts', 'jupynotex.py') } %>
+<%
+@latex_config = {
+  :supporting => [
+    Rails.root.join('app', 'views', 'layouts', 'jupynotex.py'),
+    Rails.root.join('lib', 'shell', 'latex_run.sh')
+  ],
+  :recipe => [
+    { :command => "./latex_run.sh", :runs => 1 }
+  ],
+  :preservework => false,
+  :workdir => -> { "#{Process.pid}-#{Thread.current.object_id}-#{Time.now.to_i}" }
+}
+%>
+
 \DocumentMetadata{uncompress}
 \documentclass[11pt,a4paper]{article}
 \usepackage[T1]{fontenc}

deployAppSvr.Dockerfile

@@ -37,10 +37,6 @@ RUN apt-get update \
 # Setup the folder where we will deploy the code
 WORKDIR /doubtfire
 
-# Install LaTex
-COPY ./.ci-setup /doubtfire/.ci-setup
-RUN /doubtfire/.ci-setup/texlive-install.sh
-
 # Install bundler
 RUN gem install bundler -v '2.4.5'
 RUN bundle config set --global without development test staging
@@ -49,9 +45,6 @@ RUN bundle config set --global without development test staging
 COPY ./Gemfile ./Gemfile.lock /doubtfire/
 RUN bundle install
 
-# Setup path
-ENV PATH /tmp/texlive/bin/x86_64-linux:/tmp/texlive/bin/aarch64-linux:$PATH
-
 # Copy doubtfire-api source
 COPY . /doubtfire/
 

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - ./:/doubtfire
       - ../data/tmp:/doubtfire/tmp
       - ../data/student-work:/student-work
+      - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
       - dev-db
     environment:
@@ -61,6 +62,10 @@ services:
       # RABBITMQ_USERNAME: secure_credentials
       # RABBITMQ_PASSWORD: secure_credentials
 
+      # Latex details
+      LATEX_CONTAINER_NAME: doubtfire-texlive
+      LATEX_BUILD_PATH: /texlive/shell/latex_build.sh
+
   dev-db:
     container_name: doubtfire-dev-db
     image: mariadb
@@ -71,3 +76,17 @@ services:
       MYSQL_PASSWORD: pwd
     volumes:
       - ../data/database:/var/lib/mysql
+
+  texlive:
+    container_name: doubtfire-texlive
+    build:
+      context: .
+      dockerfile: texlive.Dockerfile
+    volumes:
+      - ../data/student-work:/student-work
+      - ./public/assets/images:/doubtfire/public/assets/images
+      - ./test_files:/doubtfire/test_files
+      - ./tmp/rails-latex:/workdir/texlive-latex
+    depends_on:
+      - df-api
+    command: /bin/sh -c "while sleep 5000; do :; done"

lib/shell/latex_build.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# This script is copied into the TeX Live container and remotely executed by latex_run.sh
+
+OUTPUT_DIR=$1
+
+cd /workdir/texlive-latex/${OUTPUT_DIR}
+
+# Initialise work subfolder
+mkdir -p work
+cp *.tex *.py work/
+cd work
+
+# Compile PDF
+lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex
+echo "Running lualatex a second time to remove temporary last page..."
+lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex
+
+# Copy PDF to parent directory and cleanup
+cp *.log ../
+cp *.pdf ../
+cd ..
+rm -rf work

lib/shell/latex_run.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+# This script is copied into tmp/rails-latex/$WORK_DIR/ and executed by rails-latex
+
+WORK_DIR=$(basename "$PWD")
+docker exec -t $LATEX_CONTAINER_NAME $LATEX_BUILD_PATH $WORK_DIR

test/models/task_test.rb

@@ -369,7 +369,12 @@ def test_ipynb_to_pdf
 
     # Test if latex math was rendered properly
     reader = PDF::Reader.new(task.final_pdf_path)
+
+    # PDF-reader incorrectly parses "weight (kg) / height (m)^2" as "weight (2g) / height (m)", misplacing the ^2
+    # Detecting "height" and "weight" confirms correct LaTeX rendering
     assert reader.pages.last.text.include?("BMI: bmi ="), reader.pages.last.text
+    assert reader.pages.last.text.include?("weight")
+    assert reader.pages.last.text.include?("height (m)")
 
     # ensure the notice is not included when the notebook doesn't have long lines source code cells
     # and no errors

test_files/submissions/vectorial_graph.ipynb

@@ -101,7 +101,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -951,7 +951,7 @@
    "source": [
     "Formula for calculating\n",
     "\n",
-    "BMI: $\\text{bmi}=\\frac{\\text{weight}}{\\text{height}^2}$"
+    "BMI: $\\text{bmi}=\\frac{\\text{weig}\\text{ht (kg)}}{\\text{hei} \\text{ght (m)}^2}$"
    ]
   }
  ],

texlive.Dockerfile

@@ -0,0 +1,14 @@
+FROM texlive/texlive:latest
+
+RUN apt-get update \
+    && apt-get install -y \
+    imagemagick \
+    inkscape \
+    librsvg2-bin \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY ./lib/shell/latex_build.sh /texlive/shell/latex_build.sh
+RUN chmod +x /texlive/shell/latex_build.sh
+
+CMD ["sh", "-c", "while sleep 5000; do :; done"]