Update the jar to Apache POI 5.5.1 and keep its Java source in java/

.gitattributes

@@ -0,0 +1,5 @@
+# The expected output of the jar is compared byte for byte (the .doc text
+# has CRLF line ends); never convert line ends.
+java/expected/*.txt -text
+# Third-party license texts are kept exactly as copied from upstream.
+java/licenses/* -text

.github/workflows/plone-package.yml

@@ -2,6 +2,7 @@ name: Plone package
 
 # Every action is pinned to a full commit SHA of a release that has been out
 # for at least two weeks (cooldown). The comment after each SHA is the tag.
+# Container images are pinned by digest in the same way.
 
 on:
   push:
@@ -47,6 +48,33 @@ jobs:
       - name: Mypy
         run: tox -e mypy
 
+  jar:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      # java/build.sh uses the pinned Maven image, not the runner's Maven
+      # and JDK, so the jar must be the same as the committed one.
+      - name: Build the jar
+        run: java/build.sh "$RUNNER_TEMP/msdoc2txt.jar"
+      - name: Compare with the committed jar
+        run: |
+          sha256sum src/c2/transform/msoffice/msdoc2txt.jar "$RUNNER_TEMP/msdoc2txt.jar"
+          cmp src/c2/transform/msoffice/msdoc2txt.jar "$RUNNER_TEMP/msdoc2txt.jar"
+      # eclipse-temurin:11.0.32_9-jre-noble
+      - name: Check the output with Java 11
+        run: >-
+          docker run --rm --user "$(id -u):$(id -g)" -v "$PWD":/src:ro -w /src
+          eclipse-temurin@sha256:e69ed7d96d8e2a174057ee33cc3a4bb54c5b172dd24d41c22c032789d7bdef9b
+          java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar
+      # eclipse-temurin:21.0.12_8-jre-noble
+      - name: Check the output with Java 21
+        run: >-
+          docker run --rm --user "$(id -u):$(id -g)" -v "$PWD":/src:ro -w /src
+          eclipse-temurin@sha256:7739f0ffce786528961eea6bf46d9610ee968ac6127c9b2e93494757bdecce9f
+          java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar
+
   test:
     runs-on: ubuntu-latest
     needs: lint

AGENTS.md

@@ -16,11 +16,16 @@
   - `transform.py`: the `msoffice_to_text` transform that runs the jar.
   - `config.py`: MIME types; importing it adds them to `mimetypes`.
   - `setuphandlers.py`: install and uninstall (`post_handler`s).
-  - `msdoc2txt.jar`: the bundled Java program (Apache POI 3.17).
+  - `msdoc2txt.jar`: the bundled Java program (Apache POI 5.5.1), built
+    from `java/` with `java/build.sh`; do not change it by hand.
 - `tests/`: pytest tests (not shipped in the wheel).
   - `test_transform.py`, `test_config.py`: unit tests; `java` is replaced.
   - `test_setup.py`: install and uninstall in a Plone site.
   - `test_java.py`: runs the real jar on the files in `tests/samples/`.
+- `java/`: the source of `msdoc2txt.jar` (see `java/README.md`).
+  - `build.sh`: builds the jar reproducibly in a pinned container.
+  - `check-output.sh`: compares the output of a jar for the samples with
+    `java/expected/`, byte for byte.
 
 ## Development
 
@@ -41,21 +46,35 @@ Extra arguments go to pytest, for example
 skipped; set `C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1` to make them fail
 instead, as CI does. Write expected values in tests as literals.
 
+After a change in `java/`, run `java/build.sh` (needs Docker) and
+`java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar`, and commit
+the jar with the change; CI rebuilds it and fails if it differs. If the
+output changes, find out why before you change a file in `java/expected/`.
+
 ## Test samples
 
 All files in `tests/samples/` were made for this package; do not add
-documents of unknown origin. The `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx`
-files were converted with LibreOffice in headless mode from short text and
-CSV files (`soffice --headless --convert-to docx document.txt`, the CSV
-with `--infilter="CSV:44,34,76,1"`, and the presentation by converting a PDF
-of the text with `--infilter=impress_pdf_import`); `<cp:revision>` was then
-removed from `docProps/core.xml` of the OOXML files, because the jar prints
-it as text. `empty.docx` is a converted empty text file.
-
-The `.ppt` file was written with the Apache POI classes in the bundled jar
-(`HSLFSlideShow` with one text box), because it is much smaller than the one
-LibreOffice writes; the jar cannot read either of them (see the known issues
-in `README.md`). POI's empty template names a person as the last editor in
-the Current User stream, so the name was cleared with
-`getSlideShowImpl().getCurrentUserAtom().setLastEditUsername("")`. Check new
-samples for names in their metadata before adding them.
+documents of unknown origin. The `.docx`, `.doc`, `.xlsx`, `.xls` and
+`presentation.pptx` files were converted with LibreOffice in headless mode
+from short text and CSV files (`soffice --headless --convert-to docx
+document.txt`, the CSV with `--infilter="CSV:44,34,76,1"`, and the
+presentation by converting a PDF of the text with
+`--infilter=impress_pdf_import`); `<cp:revision>` was then removed from
+`docProps/core.xml` of the OOXML files, because the jar prints it as text.
+`empty.docx` is a converted empty text file.
+
+`presentation.ppt` was written with the Apache POI 3.17 classes of the jar
+bundled at the time (`HSLFSlideShow` with one text box), because it is much
+smaller than the one LibreOffice writes. POI's empty template names a person
+as the last editor in the Current User stream, so the name was cleared with
+`getSlideShowImpl().getCurrentUserAtom().setLastEditUsername("")`.
+
+`presentation-notes.pptx` was written with the same POI classes
+(`XMLSlideShow` with one text box and speaker notes; on the notes slide only
+the body placeholder was kept), and `<cp:revision>` was removed as above.
+`presentation-notes.ppt` is that file converted with LibreOffice in headless
+mode (`soffice --headless --convert-to ppt`), because POI cannot create
+notes in a `.ppt`.
+
+Check new samples for names in their metadata before adding them, and add
+their output to `java/expected/`.

README.md

@@ -8,13 +8,13 @@ index the text of uploaded File objects, so a search finds words inside the
 documents.
 
 Text is extracted by a bundled Java program (`msdoc2txt.jar`, built on
-[Apache POI](https://poi.apache.org/) 3.17).
+[Apache POI](https://poi.apache.org/) 5.5.1). Its source is in `java/`.
 
 ## Supported versions
 
 - Plone 6.0, 6.1 and 6.2
 - Python 3.10 to 3.14 (as far as each Plone version supports them)
-- Java 8 or later on `PATH` (CI uses Temurin 21)
+- Java 11 or later on `PATH` (CI checks the jar with Temurin 11 and 21)
 - Linux and other POSIX systems. Windows is not supported.
 
 ## Accepted MIME types
@@ -36,9 +36,9 @@ The transform is registered for these MIME types:
 
 ### Tested formats
 
-The tests extract text from `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx`
-files. `.ppt` files give no text (see the known issues). The other formats
-are passed to the jar but are not covered by the tests.
+The tests extract text from `.docx`, `.doc`, `.xlsx`, `.xls`, `.pptx` and
+`.ppt` files, including the speaker notes of presentations. The other
+formats are passed to the jar but are not covered by the tests.
 
 ## Installation
 
@@ -59,6 +59,8 @@ reindexed, for example with "Clear and Rebuild" in the catalog.
 
 ### Upgrading from 1.0b6
 
+2.0 needs Java 11 or later; 1.0b6 ran on Java 8.
+
 2.0 has no upgrade step from 1.0b6. After upgrading the package, deactivate
 the add-on and activate it again in the add-ons control panel. This replaces
 the transform that 1.0b6 registered.
@@ -104,12 +106,6 @@ Each transform runs its own JVM. Up to one JVM per Zope thread can run at the
 same time, each with a heap of up to 512 MB (by default) plus the memory of
 the JVM itself. Plan the memory of the server or container for that.
 
-## Known issues
-
-- The bundled jar cannot extract text from `.ppt` files (PowerPoint 97-2003).
-  The transform runs, logs an error and gives empty text. `.pptx` files work.
-  This will be fixed when the jar is updated.
-
 ## Uninstall
 
 Deactivate the add-on in the add-ons control panel. This removes the
@@ -118,7 +114,8 @@ transform. The MIME types are kept, because other add-ons may use them.
 ## Source code
 
 - This package: <https://github.com/cmscom/c2.transform.msoffice>
-- Java program: <https://bitbucket.org/cmscom/msdoc2txt>
+- Java program: `java/` in this repository (moved from
+  <https://bitbucket.org/cmscom/msdoc2txt>)
 
 ## Credits
 
@@ -134,5 +131,11 @@ This code was inspired by Products.OpenXml.
 
 ## License
 
-GNU General Public License, version 2 or later. See `LICENSE.txt` and
-`LICENSE.GPL`.
+c2.transform.msoffice is licensed under the GNU General Public License,
+version 2 or later. See `LICENSE.txt` and `LICENSE.GPL`.
+
+The bundled `msdoc2txt.jar` contains libraries under the Apache License 2.0
+(some of their license files have additional terms, for example for the
+ECMA-376 schemas in Apache POI) and under a BSD license (curvesapi). They
+are compatible with the package when it is used under the GPL version 3.
+The license files are in the jar, under `META-INF/`; see `java/README.md`.

java/.gitignore

@@ -0,0 +1 @@
+target/

java/README.md

@@ -0,0 +1,103 @@
+# msdoc2txt
+
+The Java program behind c2.transform.msoffice. It reads a Microsoft Office
+file on standard input and writes its text to standard output:
+
+```shell
+java -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar msdoc2txt.jar < document.docx
+```
+
+It reads Word, Excel and PowerPoint files, both the binary (`.doc`, `.xls`,
+`.ppt`) and the OOXML (`.docx`, `.xlsx`, `.pptx`) formats, with
+[Apache POI](https://poi.apache.org/) 5.5.1, and RTF files with
+[RTF Parser Kit](https://github.com/joniles/rtfparserkit). The output has
+the non-empty parts in this order, each followed by a line break: document
+properties, headers, footers, notes, text. On success the exit code is 0
+and nothing is written to standard error; otherwise the exit code is not 0.
+
+`--file PATH` reads a file instead of standard input, `--verbose` logs the
+steps to standard error, and `--help` shows the options.
+
+This source was moved here, with its history, from
+<https://bitbucket.org/cmscom/msdoc2txt> (commit 290d060).
+
+## Building
+
+The jar is built in a container, so only Docker is needed:
+
+```shell
+java/build.sh
+```
+
+This writes `src/c2/transform/msoffice/msdoc2txt.jar`, the jar that the
+Python package uses, and prints its sha256. `java/build.sh OUTPUT` writes
+the jar somewhere else. Maven downloads are cached in
+`~/.cache/msdoc2txt-m2` (set `MSDOC2TXT_M2_CACHE` to change it).
+
+The build is reproducible: the Maven image is pinned by digest in
+`build.sh`, every Maven plugin and dependency is pinned in `pom.xml`, and
+`project.build.outputTimestamp` fixes the time stamps in the jar. The same
+source gives the same jar, byte for byte. CI rebuilds the jar and fails if
+it differs from the committed one, so commit the jar together with the
+change of the source.
+
+The bytecode targets Java 11 (`--release 11`); the jar runs on Java 11 or
+later.
+
+## Checking the output
+
+```shell
+java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar
+```
+
+runs the jar on the files in `tests/samples/` and compares the output, byte
+for byte, with `java/expected/<sample>.txt`. It also checks the exit code
+and that standard error is empty. Set `JAVA` to use another `java`. CI runs
+it with Java 11 and Java 21.
+
+When the output changes, find out why before you change an expected file.
+
+## Updating a dependency
+
+1. Change the version in `pom.xml` and set `project.build.outputTimestamp`
+   to the current date.
+2. Run `java/build.sh` and `java/check-output.sh`.
+3. Commit `pom.xml`, the jar and, if the change is intended, the expected
+   files.
+
+`pom.xml` is part of the jar, so any change to it needs a rebuild, even a
+change of a comment.
+
+## How the jar is put together
+
+`pom.xml` builds one jar with all dependencies, with maven-shade-plugin:
+
+- POI finds its extractors with `ServiceLoader`. poi, poi-ooxml and
+  poi-scratchpad each have a `META-INF/services` file with the same name;
+  they are merged.
+- The Log4j API, which POI uses, prints a warning to standard output when
+  no logging implementation is present, and that would end up in the
+  text. `src/main/resources/log4j2.component.properties` selects the
+  simple provider of the API and turns it off.
+- `module-info.class`, `META-INF/versions/` and signature files of the
+  dependencies are left out.
+- The manifest names the main class and the POI version
+  (`Apache-POI-Version`).
+
+## Licenses
+
+The jar contains these libraries. Most are under the
+[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0):
+Apache POI, XMLBeans, Commons (CLI, Codec, Collections, Compress, IO, Lang,
+Math), the Log4j API, SparseBitSet and RTF Parser Kit. The license files of
+POI and Commons Math have additional terms for parts of them, for example
+for the ECMA-376 schemas in POI. curvesapi is under a BSD license.
+
+The license and notice files that the libraries ship are kept in the jar
+(the SparseBitSet and RTF Parser Kit jars have none): `META-INF/LICENSE` and
+`META-INF/LICENSE.txt` have the license files one after another, and
+`META-INF/NOTICE` the merged notices.
+curvesapi has no license file in its jar, so `licenses/curvesapi-LICENSE.txt`
+is a copy of `license.txt` of curvesapi 1.08
+(<https://github.com/virtuald/curvesapi/blob/1.08/license.txt>); it goes
+into the jar as `META-INF/LICENSE-curvesapi.txt`.

java/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Build msdoc2txt.jar reproducibly in a pinned container: the same image
+# and the pinned plugins give the same jar, byte for byte.
+# Usage: java/build.sh [OUTPUT]  (default: src/c2/transform/msoffice/msdoc2txt.jar)
+set -euo pipefail
+here=$(cd "$(dirname "$0")" && pwd)
+out=${1:-$here/../src/c2/transform/msoffice/msdoc2txt.jar}
+# maven:3.9.16-eclipse-temurin-21-noble (2026-09-26)
+image=maven@sha256:99e61abcff91a9b1333463bd8451fb18495d6eba9250ac66a338b518f8278320
+# Downloaded dependencies are kept between builds, outside the repository.
+cache=${MSDOC2TXT_M2_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/msdoc2txt-m2}
+mkdir -p "$cache"
+docker run --rm \
+    --user "$(id -u):$(id -g)" \
+    -e MAVEN_CONFIG=/var/maven/.m2 \
+    -v "$cache":/var/maven/.m2 \
+    -v "$here":/build -w /build \
+    "$image" mvn -B -q -Duser.home=/var/maven clean package
+cp "$here/target/msdoc2txt.jar" "$out"
+sha256sum "$out"

java/check-output.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Run msdoc2txt.jar on the test samples and compare its output, byte for
+# byte, with java/expected/. Usage: java/check-output.sh path/to/msdoc2txt.jar
+# Set JAVA to use another java command.
+set -u
+here=$(cd "$(dirname "$0")" && pwd)
+samples="$here/../tests/samples"
+jar=${1:?usage: check-output.sh path/to/msdoc2txt.jar}
+java=${JAVA:-java}
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+status=0
+for expected in "$here"/expected/*.txt; do
+    name=$(basename "$expected" .txt)
+    rc=0
+    LC_ALL=C "$java" -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar "$jar" \
+        < "$samples/$name" > "$work/out" 2> "$work/err" || rc=$?
+    if [ "$rc" -ne 0 ]; then
+        echo "FAIL $name: exit code $rc"
+        head -c 2000 "$work/err"
+        echo
+        status=1
+        continue
+    fi
+    if [ -s "$work/err" ]; then
+        echo "FAIL $name: stderr is not empty"
+        head -c 2000 "$work/err"
+        echo
+        status=1
+        continue
+    fi
+    if ! cmp -s "$expected" "$work/out"; then
+        echo "FAIL $name: output differs (expected, actual)"
+        diff <(od -c "$expected") <(od -c "$work/out") | head -40
+        status=1
+        continue
+    fi
+    echo "OK   $name"
+done
+exit $status

java/expected/document.doc.txt

@@ -0,0 +1,5 @@
+0
+This is a simple example of a document with a full set of metadata and comments.
+
+これは日本語の文書の例です。
+

java/expected/document.docx.txt

@@ -0,0 +1,3 @@
+This is a simple example of a document with a full set of metadata and comments.
+これは日本語の文書の例です。
+

java/expected/presentation-notes.ppt.txt

@@ -0,0 +1,9 @@
+0
+Speaker notes example
+
+発表者のノートです。
+
+Presentation with notes
+
+ノート付きのスライドです。
+

java/expected/presentation-notes.pptx.txt

@@ -0,0 +1,6 @@
+Speaker notes example
+発表者のノートです。
+
+Presentation with notes
+ノート付きのスライドです。
+

java/expected/presentation.ppt.txt

@@ -0,0 +1,10 @@
+
+Microsoft PowerPoint
+
+0
+
+PowerPoint Presentation
+Presentation slide example
+
+スライドの例です。
+