feat: improve executor capabilities

executor/.dockerignore

@@ -6,4 +6,5 @@
 !sitecustomize.py
 !requirements.txt
 !requirements-skip.txt
-!pandoc-wrapper
+!pandoc-wrapper
+!ffmpeg-wrapper

executor/Dockerfile

@@ -35,24 +35,48 @@ RUN apk add --no-cache git make gcc g++ musl-dev && \
     make install
 
 FROM docker.io/alpine:${ALPINE_VERSION} AS runtime
-RUN apk add --no-cache \
+RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing \
+    bash \
     coreutils \
     ffmpeg \
     font-dejavu \
+    ghostscript-fonts \
     imagemagick \
+    opencv \
     pandoc-cli \
     python3 \
+    py3-ffmpeg \
+    py3-ffmpeg-pyc \
     py3-jinja2 \
+    py3-jinja2-pyc \
     py3-matplotlib \
+    py3-matplotlib-pyc \
     py3-matplotlib-venn \
+    py3-matplotlib-venn-pyc \
+    py3-moviepy \
+    py3-moviepy-pyc \
     py3-numpy \
+    py3-numpy-pyc \
+    py3-opencv \
+    py3-opencv-pyc \
     py3-pandas \
+    py3-pandas-pyc \
+    py3-pikepdf \
+    py3-pikepdf-pyc \
     py3-pillow \
+    py3-pillow-pyc \
     py3-pip \
+    py3-pip-pyc \
+    py3-pypandoc \
+    py3-pypandoc-pyc \
     py3-scipy \
+    py3-scipy-pyc \
     py3-sympy \
+    py3-sympy-pyc \
     py3-tabulate \
+    py3-tabulate-pyc \
     py3-xarray \
+    py3-xarray-pyc \
     weasyprint
 RUN mkdir /workspace && chmod 777 /workspace
 WORKDIR /workspace
@@ -61,13 +85,21 @@ USER 1001050000
 # Python
 COPY requirements.txt requirements-skip.txt /
 RUN python -m venv --system-site-packages /tmp/venv && /tmp/venv/bin/pip install --no-cache-dir -r /requirements.txt
-COPY sitecustomize.py /venv/lib/python3.12/site-packages
+COPY sitecustomize.py /tmp/venv/lib/python3.12/site-packages
 ENV PATH="/tmp/venv/bin:$PATH" \
-    MPLCONFIGDIR="/tmp/mplconfigdir"
+    MPLCONFIGDIR="/tmp/mplconfigdir" \
+    XDG_CACHE_HOME="/tmp/.cache" \
+    HOME="/tmp/home"
 
 # Pandoc
 COPY pandoc-wrapper /usr/local/bin/pandoc
 
+# ffmpeg
+COPY ffmpeg-wrapper /usr/local/bin/ffmpeg
+
+# xonsh
+ENV XONSH_SHOW_TRACEBACK=True
+
 # UPM
 COPY --from=build-upm /go/bin/upm /usr/local/bin/upm
 COPY --from=build-upm /workdir/upm/internal/backends/python/pypi_map.sqlite /pypi_map.sqlite

executor/ffmpeg-wrapper

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/bin/ffmpeg -hide_banner -loglevel error "$@"

executor/requirements-skip.txt

@@ -1 +1,2 @@
 ffmpeg # actually ffmpeg-python
+opencv-python # installed through OS package

executor/requirements.txt

@@ -3,14 +3,15 @@ jinja2
 matplotlib
 matplotlib-venn
 numpy
+moviepy
 pandas[excel,plot,output-formatting,computation]
 pillow
-pip
 pypandoc
-PyPDF2
+pikepdf
 python-docx
 python-pptx
 scipy
 sympy
 tabulate
-xarray
+xarray
+xonsh

executor/server.rs

@@ -129,7 +129,7 @@ async fn get_file_hashes(dir: &str) -> HashMap<String, String> {
     hashes
 }
 
-async fn execute_python(payload: web::Json<ExecuteRequest>) -> Result<HttpResponse, Error> {
+async fn execute(payload: web::Json<ExecuteRequest>) -> Result<HttpResponse, Error> {
     let workspace = env::var("APP_WORKSPACE").unwrap_or_else(|_| "/workspace".to_string());
     let before_hashes = get_file_hashes(&workspace).await;
     let source_dir = TempDir::new()?;
@@ -161,7 +161,7 @@ async fn execute_python(payload: web::Json<ExecuteRequest>) -> Result<HttpRespon
     let timeout = Duration::from_secs(payload.timeout.unwrap_or(60));
     let (stdout, stderr, exit_code) = tokio::time::timeout(
         timeout,
-        Command::new("python")
+        Command::new("xonsh") // TODO: manually switch between python and shell for ~80ms perf gain
             .arg(source_dir.path().join("script.py"))
             .output(),
     )
@@ -214,7 +214,7 @@ async fn web() -> std::io::Result<()> {
             .wrap(Logger::default())
             .route("/workspace/{path:.*}", web::put().to(upload_file))
             .route("/workspace/{path:.*}", web::get().to(download_file))
-            .route("/execute", web::post().to(execute_python))
+            .route("/execute", web::post().to(execute))
     })
     .bind(&listen_addr)?
     .run()

executor/sitecustomize.py

@@ -1,4 +1,13 @@
-import matplotlib.pyplot
-
 # LLM tends to generate `.show()` which does not work in a headless environment
+import matplotlib.pyplot
 matplotlib.pyplot.show = lambda *_args, **_kwargs: matplotlib.pyplot.savefig("plot.png")
+
+# Disable progressbar for MoviePy which fills up the context window
+import moviepy.editor
+
+old_moviepy_editor_VideoClip_write_videofile = moviepy.editor.VideoClip.write_videofile
+moviepy.editor.VideoClip.write_videofile = (
+    lambda self, *args, **kwargs: old_moviepy_editor_VideoClip_write_videofile(
+        self, *args, verbose=False, logger=None, **kwargs
+    )
+)