diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a883777 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# The expected output of the jar is compared byte for byte (the .doc text +# has CRLF line ends); never convert line ends. +java/expected/*.txt -text +# Third-party license texts are kept exactly as copied from upstream. +java/licenses/* -text diff --git a/.github/workflows/plone-package.yml b/.github/workflows/plone-package.yml index 59f7823..09757d0 100644 --- a/.github/workflows/plone-package.yml +++ b/.github/workflows/plone-package.yml @@ -2,6 +2,7 @@ name: Plone package # Every action is pinned to a full commit SHA of a release that has been out # for at least two weeks (cooldown). The comment after each SHA is the tag. +# Container images are pinned by digest in the same way. on: push: @@ -47,6 +48,33 @@ jobs: - name: Mypy run: tox -e mypy + jar: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # java/build.sh uses the pinned Maven image, not the runner's Maven + # and JDK, so the jar must be the same as the committed one. + - name: Build the jar + run: java/build.sh "$RUNNER_TEMP/msdoc2txt.jar" + - name: Compare with the committed jar + run: | + sha256sum src/c2/transform/msoffice/msdoc2txt.jar "$RUNNER_TEMP/msdoc2txt.jar" + cmp src/c2/transform/msoffice/msdoc2txt.jar "$RUNNER_TEMP/msdoc2txt.jar" + # eclipse-temurin:11.0.32_9-jre-noble + - name: Check the output with Java 11 + run: >- + docker run --rm --user "$(id -u):$(id -g)" -v "$PWD":/src:ro -w /src + eclipse-temurin@sha256:e69ed7d96d8e2a174057ee33cc3a4bb54c5b172dd24d41c22c032789d7bdef9b + java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar + # eclipse-temurin:21.0.12_8-jre-noble + - name: Check the output with Java 21 + run: >- + docker run --rm --user "$(id -u):$(id -g)" -v "$PWD":/src:ro -w /src + eclipse-temurin@sha256:7739f0ffce786528961eea6bf46d9610ee968ac6127c9b2e93494757bdecce9f + java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar + test: runs-on: ubuntu-latest needs: lint diff --git a/AGENTS.md b/AGENTS.md index 932a0d2..83fea83 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,11 +16,16 @@ - `transform.py`: the `msoffice_to_text` transform that runs the jar. - `config.py`: MIME types; importing it adds them to `mimetypes`. - `setuphandlers.py`: install and uninstall (`post_handler`s). - - `msdoc2txt.jar`: the bundled Java program (Apache POI 3.17). + - `msdoc2txt.jar`: the bundled Java program (Apache POI 5.5.1), built + from `java/` with `java/build.sh`; do not change it by hand. - `tests/`: pytest tests (not shipped in the wheel). - `test_transform.py`, `test_config.py`: unit tests; `java` is replaced. - `test_setup.py`: install and uninstall in a Plone site. - `test_java.py`: runs the real jar on the files in `tests/samples/`. +- `java/`: the source of `msdoc2txt.jar` (see `java/README.md`). + - `build.sh`: builds the jar reproducibly in a pinned container. + - `check-output.sh`: compares the output of a jar for the samples with + `java/expected/`, byte for byte. ## Development @@ -41,21 +46,35 @@ Extra arguments go to pytest, for example skipped; set `C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1` to make them fail instead, as CI does. Write expected values in tests as literals. +After a change in `java/`, run `java/build.sh` (needs Docker) and +`java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar`, and commit +the jar with the change; CI rebuilds it and fails if it differs. If the +output changes, find out why before you change a file in `java/expected/`. + ## Test samples All files in `tests/samples/` were made for this package; do not add -documents of unknown origin. The `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx` -files were converted with LibreOffice in headless mode from short text and -CSV files (`soffice --headless --convert-to docx document.txt`, the CSV -with `--infilter="CSV:44,34,76,1"`, and the presentation by converting a PDF -of the text with `--infilter=impress_pdf_import`); `` was then -removed from `docProps/core.xml` of the OOXML files, because the jar prints -it as text. `empty.docx` is a converted empty text file. - -The `.ppt` file was written with the Apache POI classes in the bundled jar -(`HSLFSlideShow` with one text box), because it is much smaller than the one -LibreOffice writes; the jar cannot read either of them (see the known issues -in `README.md`). POI's empty template names a person as the last editor in -the Current User stream, so the name was cleared with -`getSlideShowImpl().getCurrentUserAtom().setLastEditUsername("")`. Check new -samples for names in their metadata before adding them. +documents of unknown origin. The `.docx`, `.doc`, `.xlsx`, `.xls` and +`presentation.pptx` files were converted with LibreOffice in headless mode +from short text and CSV files (`soffice --headless --convert-to docx +document.txt`, the CSV with `--infilter="CSV:44,34,76,1"`, and the +presentation by converting a PDF of the text with +`--infilter=impress_pdf_import`); `` was then removed from +`docProps/core.xml` of the OOXML files, because the jar prints it as text. +`empty.docx` is a converted empty text file. + +`presentation.ppt` was written with the Apache POI 3.17 classes of the jar +bundled at the time (`HSLFSlideShow` with one text box), because it is much +smaller than the one LibreOffice writes. POI's empty template names a person +as the last editor in the Current User stream, so the name was cleared with +`getSlideShowImpl().getCurrentUserAtom().setLastEditUsername("")`. + +`presentation-notes.pptx` was written with the same POI classes +(`XMLSlideShow` with one text box and speaker notes; on the notes slide only +the body placeholder was kept), and `` was removed as above. +`presentation-notes.ppt` is that file converted with LibreOffice in headless +mode (`soffice --headless --convert-to ppt`), because POI cannot create +notes in a `.ppt`. + +Check new samples for names in their metadata before adding them, and add +their output to `java/expected/`. diff --git a/README.md b/README.md index 1eeb989..526f921 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ index the text of uploaded File objects, so a search finds words inside the documents. Text is extracted by a bundled Java program (`msdoc2txt.jar`, built on -[Apache POI](https://poi.apache.org/) 3.17). +[Apache POI](https://poi.apache.org/) 5.5.1). Its source is in `java/`. ## Supported versions - Plone 6.0, 6.1 and 6.2 - Python 3.10 to 3.14 (as far as each Plone version supports them) -- Java 8 or later on `PATH` (CI uses Temurin 21) +- Java 11 or later on `PATH` (CI checks the jar with Temurin 11 and 21) - Linux and other POSIX systems. Windows is not supported. ## Accepted MIME types @@ -36,9 +36,9 @@ The transform is registered for these MIME types: ### Tested formats -The tests extract text from `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx` -files. `.ppt` files give no text (see the known issues). The other formats -are passed to the jar but are not covered by the tests. +The tests extract text from `.docx`, `.doc`, `.xlsx`, `.xls`, `.pptx` and +`.ppt` files, including the speaker notes of presentations. The other +formats are passed to the jar but are not covered by the tests. ## Installation @@ -59,6 +59,8 @@ reindexed, for example with "Clear and Rebuild" in the catalog. ### Upgrading from 1.0b6 +2.0 needs Java 11 or later; 1.0b6 ran on Java 8. + 2.0 has no upgrade step from 1.0b6. After upgrading the package, deactivate the add-on and activate it again in the add-ons control panel. This replaces the transform that 1.0b6 registered. @@ -104,12 +106,6 @@ Each transform runs its own JVM. Up to one JVM per Zope thread can run at the same time, each with a heap of up to 512 MB (by default) plus the memory of the JVM itself. Plan the memory of the server or container for that. -## Known issues - -- The bundled jar cannot extract text from `.ppt` files (PowerPoint 97-2003). - The transform runs, logs an error and gives empty text. `.pptx` files work. - This will be fixed when the jar is updated. - ## Uninstall Deactivate the add-on in the add-ons control panel. This removes the @@ -118,7 +114,8 @@ transform. The MIME types are kept, because other add-ons may use them. ## Source code - This package: -- Java program: +- Java program: `java/` in this repository (moved from + ) ## Credits @@ -134,5 +131,11 @@ This code was inspired by Products.OpenXml. ## License -GNU General Public License, version 2 or later. See `LICENSE.txt` and -`LICENSE.GPL`. +c2.transform.msoffice is licensed under the GNU General Public License, +version 2 or later. See `LICENSE.txt` and `LICENSE.GPL`. + +The bundled `msdoc2txt.jar` contains libraries under the Apache License 2.0 +(some of their license files have additional terms, for example for the +ECMA-376 schemas in Apache POI) and under a BSD license (curvesapi). They +are compatible with the package when it is used under the GPL version 3. +The license files are in the jar, under `META-INF/`; see `java/README.md`. diff --git a/java/.gitignore b/java/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/java/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/java/README.md b/java/README.md new file mode 100644 index 0000000..312680f --- /dev/null +++ b/java/README.md @@ -0,0 +1,103 @@ +# msdoc2txt + +The Java program behind c2.transform.msoffice. It reads a Microsoft Office +file on standard input and writes its text to standard output: + +```shell +java -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar msdoc2txt.jar < document.docx +``` + +It reads Word, Excel and PowerPoint files, both the binary (`.doc`, `.xls`, +`.ppt`) and the OOXML (`.docx`, `.xlsx`, `.pptx`) formats, with +[Apache POI](https://poi.apache.org/) 5.5.1, and RTF files with +[RTF Parser Kit](https://github.com/joniles/rtfparserkit). The output has +the non-empty parts in this order, each followed by a line break: document +properties, headers, footers, notes, text. On success the exit code is 0 +and nothing is written to standard error; otherwise the exit code is not 0. + +`--file PATH` reads a file instead of standard input, `--verbose` logs the +steps to standard error, and `--help` shows the options. + +This source was moved here, with its history, from + (commit 290d060). + +## Building + +The jar is built in a container, so only Docker is needed: + +```shell +java/build.sh +``` + +This writes `src/c2/transform/msoffice/msdoc2txt.jar`, the jar that the +Python package uses, and prints its sha256. `java/build.sh OUTPUT` writes +the jar somewhere else. Maven downloads are cached in +`~/.cache/msdoc2txt-m2` (set `MSDOC2TXT_M2_CACHE` to change it). + +The build is reproducible: the Maven image is pinned by digest in +`build.sh`, every Maven plugin and dependency is pinned in `pom.xml`, and +`project.build.outputTimestamp` fixes the time stamps in the jar. The same +source gives the same jar, byte for byte. CI rebuilds the jar and fails if +it differs from the committed one, so commit the jar together with the +change of the source. + +The bytecode targets Java 11 (`--release 11`); the jar runs on Java 11 or +later. + +## Checking the output + +```shell +java/check-output.sh src/c2/transform/msoffice/msdoc2txt.jar +``` + +runs the jar on the files in `tests/samples/` and compares the output, byte +for byte, with `java/expected/.txt`. It also checks the exit code +and that standard error is empty. Set `JAVA` to use another `java`. CI runs +it with Java 11 and Java 21. + +When the output changes, find out why before you change an expected file. + +## Updating a dependency + +1. Change the version in `pom.xml` and set `project.build.outputTimestamp` + to the current date. +2. Run `java/build.sh` and `java/check-output.sh`. +3. Commit `pom.xml`, the jar and, if the change is intended, the expected + files. + +`pom.xml` is part of the jar, so any change to it needs a rebuild, even a +change of a comment. + +## How the jar is put together + +`pom.xml` builds one jar with all dependencies, with maven-shade-plugin: + +- POI finds its extractors with `ServiceLoader`. poi, poi-ooxml and + poi-scratchpad each have a `META-INF/services` file with the same name; + they are merged. +- The Log4j API, which POI uses, prints a warning to standard output when + no logging implementation is present, and that would end up in the + text. `src/main/resources/log4j2.component.properties` selects the + simple provider of the API and turns it off. +- `module-info.class`, `META-INF/versions/` and signature files of the + dependencies are left out. +- The manifest names the main class and the POI version + (`Apache-POI-Version`). + +## Licenses + +The jar contains these libraries. Most are under the +[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0): +Apache POI, XMLBeans, Commons (CLI, Codec, Collections, Compress, IO, Lang, +Math), the Log4j API, SparseBitSet and RTF Parser Kit. The license files of +POI and Commons Math have additional terms for parts of them, for example +for the ECMA-376 schemas in POI. curvesapi is under a BSD license. + +The license and notice files that the libraries ship are kept in the jar +(the SparseBitSet and RTF Parser Kit jars have none): `META-INF/LICENSE` and +`META-INF/LICENSE.txt` have the license files one after another, and +`META-INF/NOTICE` the merged notices. +curvesapi has no license file in its jar, so `licenses/curvesapi-LICENSE.txt` +is a copy of `license.txt` of curvesapi 1.08 +(); it goes +into the jar as `META-INF/LICENSE-curvesapi.txt`. diff --git a/java/build.sh b/java/build.sh new file mode 100755 index 0000000..8ee0652 --- /dev/null +++ b/java/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Build msdoc2txt.jar reproducibly in a pinned container: the same image +# and the pinned plugins give the same jar, byte for byte. +# Usage: java/build.sh [OUTPUT] (default: src/c2/transform/msoffice/msdoc2txt.jar) +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +out=${1:-$here/../src/c2/transform/msoffice/msdoc2txt.jar} +# maven:3.9.16-eclipse-temurin-21-noble (2026-09-26) +image=maven@sha256:99e61abcff91a9b1333463bd8451fb18495d6eba9250ac66a338b518f8278320 +# Downloaded dependencies are kept between builds, outside the repository. +cache=${MSDOC2TXT_M2_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/msdoc2txt-m2} +mkdir -p "$cache" +docker run --rm \ + --user "$(id -u):$(id -g)" \ + -e MAVEN_CONFIG=/var/maven/.m2 \ + -v "$cache":/var/maven/.m2 \ + -v "$here":/build -w /build \ + "$image" mvn -B -q -Duser.home=/var/maven clean package +cp "$here/target/msdoc2txt.jar" "$out" +sha256sum "$out" diff --git a/java/check-output.sh b/java/check-output.sh new file mode 100755 index 0000000..61f52bb --- /dev/null +++ b/java/check-output.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Run msdoc2txt.jar on the test samples and compare its output, byte for +# byte, with java/expected/. Usage: java/check-output.sh path/to/msdoc2txt.jar +# Set JAVA to use another java command. +set -u +here=$(cd "$(dirname "$0")" && pwd) +samples="$here/../tests/samples" +jar=${1:?usage: check-output.sh path/to/msdoc2txt.jar} +java=${JAVA:-java} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +status=0 +for expected in "$here"/expected/*.txt; do + name=$(basename "$expected" .txt) + rc=0 + LC_ALL=C "$java" -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar "$jar" \ + < "$samples/$name" > "$work/out" 2> "$work/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL $name: exit code $rc" + head -c 2000 "$work/err" + echo + status=1 + continue + fi + if [ -s "$work/err" ]; then + echo "FAIL $name: stderr is not empty" + head -c 2000 "$work/err" + echo + status=1 + continue + fi + if ! cmp -s "$expected" "$work/out"; then + echo "FAIL $name: output differs (expected, actual)" + diff <(od -c "$expected") <(od -c "$work/out") | head -40 + status=1 + continue + fi + echo "OK $name" +done +exit $status diff --git a/java/expected/document.doc.txt b/java/expected/document.doc.txt new file mode 100644 index 0000000..911ab90 --- /dev/null +++ b/java/expected/document.doc.txt @@ -0,0 +1,5 @@ +0 +This is a simple example of a document with a full set of metadata and comments. + +これは日本語の文書の例です。 + diff --git a/java/expected/document.docx.txt b/java/expected/document.docx.txt new file mode 100644 index 0000000..a8b9a1e --- /dev/null +++ b/java/expected/document.docx.txt @@ -0,0 +1,3 @@ +This is a simple example of a document with a full set of metadata and comments. +これは日本語の文書の例です。 + diff --git a/java/expected/empty.docx.txt b/java/expected/empty.docx.txt new file mode 100644 index 0000000..e69de29 diff --git a/java/expected/presentation-notes.ppt.txt b/java/expected/presentation-notes.ppt.txt new file mode 100644 index 0000000..0693a81 --- /dev/null +++ b/java/expected/presentation-notes.ppt.txt @@ -0,0 +1,9 @@ +0 +Speaker notes example + +発表者のノートです。 + +Presentation with notes + +ノート付きのスライドです。 + diff --git a/java/expected/presentation-notes.pptx.txt b/java/expected/presentation-notes.pptx.txt new file mode 100644 index 0000000..a14950d --- /dev/null +++ b/java/expected/presentation-notes.pptx.txt @@ -0,0 +1,6 @@ +Speaker notes example +発表者のノートです。 + +Presentation with notes +ノート付きのスライドです。 + diff --git a/java/expected/presentation.ppt.txt b/java/expected/presentation.ppt.txt new file mode 100644 index 0000000..ef308f8 --- /dev/null +++ b/java/expected/presentation.ppt.txt @@ -0,0 +1,10 @@ + +Microsoft PowerPoint + +0 + +PowerPoint Presentation +Presentation slide example + +スライドの例です。 + diff --git a/java/expected/presentation.pptx.txt b/java/expected/presentation.pptx.txt new file mode 100644 index 0000000..f2853a9 --- /dev/null +++ b/java/expected/presentation.pptx.txt @@ -0,0 +1,3 @@ +Presentation slide example +スライドの例です。 + diff --git a/java/expected/spreadsheet.xls.txt b/java/expected/spreadsheet.xls.txt new file mode 100644 index 0000000..8d2282d --- /dev/null +++ b/java/expected/spreadsheet.xls.txt @@ -0,0 +1,8 @@ +0 +spreadsheet +&"Times New Roman,標準"&12&A +Fruit Count +Apple 3 +みかん 5 +&"Times New Roman,標準"&12ページ &P + diff --git a/java/expected/spreadsheet.xlsx.txt b/java/expected/spreadsheet.xlsx.txt new file mode 100644 index 0000000..37b4612 --- /dev/null +++ b/java/expected/spreadsheet.xlsx.txt @@ -0,0 +1,7 @@ +spreadsheet +&"Times New Roman,標準"&12&A +Fruit Count +Apple 3 +みかん 5 +&"Times New Roman,標準"&12ページ &P + diff --git a/java/licenses/curvesapi-LICENSE.txt b/java/licenses/curvesapi-LICENSE.txt new file mode 100644 index 0000000..972b1c1 --- /dev/null +++ b/java/licenses/curvesapi-LICENSE.txt @@ -0,0 +1,28 @@ +Copyright (c) 2005, Graph Builder +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +-Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +-Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +-Neither the name of Graph Builder nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/java/pom.xml b/java/pom.xml new file mode 100644 index 0000000..c79d38b --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,148 @@ + + + 4.0.0 + jp.cmscom.c2.transform.msoffice + msdoc2txt + 2.0.0 + jar + msdoc2txt + Extract plain text from Microsoft Office files (used by c2.transform.msoffice) + + + UTF-8 + 11 + + 2026-10-20T00:00:00Z + 5.5.1 + + + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + org.apache.poi + poi-scratchpad + ${poi.version} + + + com.github.joniles + rtfparserkit + 1.16.0 + + + commons-cli + commons-cli + 1.11.0 + + + + + msdoc2txt + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.5.0 + + + org.apache.maven.plugins + maven-resources-plugin + 3.5.0 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.16.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.6.0 + + + org.apache.maven.plugins + maven-jar-plugin + 3.5.1 + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.2 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + false + + + *:* + + module-info.class + **/module-info.class + META-INF/versions/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + META-INF/LICENSE + + + META-INF/LICENSE.txt + + + + META-INF/LICENSE-curvesapi.txt + licenses/curvesapi-LICENSE.txt + + + + jp.cmscom.c2.transform.msoffice.MSDoc2Txt + + ${poi.version} + + + + + + + + + + diff --git a/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocExtracter.java b/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocExtracter.java new file mode 100644 index 0000000..2fbb7d7 --- /dev/null +++ b/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocExtracter.java @@ -0,0 +1,23 @@ +package jp.cmscom.c2.transform.msoffice; + +public interface DocExtracter { + default public String getProperties() { + return ""; + } + + default public String getNotes() { + return ""; + } + + default public String getHeader() { + return ""; + } + + default public String getFooter() { + return ""; + } + + default public String getContents() { + return ""; + } +} \ No newline at end of file diff --git a/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocPart.java b/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocPart.java new file mode 100644 index 0000000..9202bd4 --- /dev/null +++ b/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocPart.java @@ -0,0 +1,9 @@ +package jp.cmscom.c2.transform.msoffice; + +public enum DocPart { + Properties, + Header, + Footer, + Notes, + Contents; +} \ No newline at end of file diff --git a/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocType.java b/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocType.java new file mode 100644 index 0000000..763ca58 --- /dev/null +++ b/java/src/main/java/jp/cmscom/c2/transform/msoffice/DocType.java @@ -0,0 +1,58 @@ +package jp.cmscom.c2.transform.msoffice; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +import org.apache.poi.openxml4j.exceptions.OpenXML4JException; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.poifs.filesystem.FileMagic; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.xmlbeans.XmlException; + +public enum DocType { + + OLE2(FileMagic.OLE2) { + @Override + public DocExtracter getExtracter(InputStream is) throws IOException, OpenXML4JException, XmlException { + POIFSFileSystem fs = new POIFSFileSystem(is); + return new MSDocExtracter(fs); + } + }, + + OOXML(FileMagic.OOXML) { + @Override + public DocExtracter getExtracter(InputStream is) throws IOException, OpenXML4JException, XmlException { + OPCPackage pkg = OPCPackage.open(is); + return new MSDocExtracter(pkg); + } + }, + + RTF(FileMagic.RTF) { + @Override + public DocExtracter getExtracter(InputStream is) throws IOException, OpenXML4JException, XmlException { + return new RTFExtracter(is); + } + }; + + abstract DocExtracter getExtracter(InputStream is) throws IOException, OpenXML4JException, XmlException; + + private final FileMagic fileType; + + DocType(FileMagic fileType) { + this.fileType = fileType; + } + + public FileMagic getFileType() { + return this.fileType; + } + + public static Optional valueOf(FileMagic fileType) { + for (DocType docType : DocType.values()) { + if (docType.getFileType().equals(fileType)) { + return Optional.ofNullable(docType); + } + } + return Optional.empty(); + } +} \ No newline at end of file diff --git a/java/src/main/java/jp/cmscom/c2/transform/msoffice/MSDoc2Txt.java b/java/src/main/java/jp/cmscom/c2/transform/msoffice/MSDoc2Txt.java new file mode 100644 index 0000000..44d3885 --- /dev/null +++ b/java/src/main/java/jp/cmscom/c2/transform/msoffice/MSDoc2Txt.java @@ -0,0 +1,122 @@ +package jp.cmscom.c2.transform.msoffice; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PushbackInputStream; +import java.util.Arrays; +import java.util.Date; +import java.util.Optional; +import java.util.logging.ConsoleHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; +import java.util.stream.Collectors; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.poi.openxml4j.exceptions.OpenXML4JException; +import org.apache.poi.poifs.filesystem.FileMagic; +import org.apache.xmlbeans.XmlException; + +public class MSDoc2Txt { + + private static Logger logger = Logger.getLogger(MSDoc2Txt.class.getName()); + + private static void prn(String s) { + if (s != null && !s.trim().isEmpty()) { + System.out.println(s); + } + } + + private static CommandLine parseArgument(String[] args) throws ParseException { + CommandLineParser parser = new DefaultParser(); + Options options = new Options(); + options.addOption("h", "help", false, "print this message"); + options.addOption("v", "verbose", false, "verbose mode for debugging"); + options.addOption( + Option.builder("f").longOpt("file").hasArg().desc("set path to file to retreive contents").build()); + + CommandLine cmd = parser.parse(options, args); + if (cmd.hasOption("help")) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("msdoc2txt", options); + } + + return cmd; + } + + private static void configureLogger() { + logger.setLevel(Level.OFF); + logger.setUseParentHandlers(false); + Handler handler = new ConsoleHandler(); + handler.setFormatter(new SimpleFormatter() { + private static final String format = "%1$tF %1$tT [%2$s] %3$s %n"; + + @Override + public synchronized String format(LogRecord lr) { + Date date = new Date(lr.getMillis()); + return String.format(format, date, lr.getLevel().getLocalizedName(), lr.getMessage()); + } + }); + logger.addHandler(handler); + + // TODO: logger configuration + MSDocExtracter.setLogger(logger); + RTFExtracter.setLogger(logger); + } + + public static void main(String[] args) + throws FileNotFoundException, IOException, XmlException, ParseException, OpenXML4JException { + configureLogger(); + + CommandLine cmd = parseArgument(args); + if (cmd.hasOption("help")) { + return; + } + + if (cmd.hasOption("verbose")) { + logger.setLevel(Level.INFO); + } + logger.info("start msdoc2txt for debugging"); + + InputStream is = System.in; + if (cmd.hasOption("file")) { + String path = cmd.getOptionValue("file"); + logger.info("path to file: " + path); + is = new FileInputStream(path); + } + + try (InputStream pis = FileMagic.prepareToCheckMagic(new PushbackInputStream(is, 8))) { + FileMagic fileType = FileMagic.valueOf(pis); + Optional opt = DocType.valueOf(fileType); + if (opt.isPresent()) { + DocType docType = opt.get(); + logger.info("file type: " + fileType.toString()); + + DocExtracter extracter = docType.getExtracter(pis); + prn(extracter.getProperties()); + prn(extracter.getHeader()); + prn(extracter.getFooter()); + prn(extracter.getNotes()); + prn(extracter.getContents()); + } else { + logger.info("file type: " + fileType.toString()); + String types = Arrays.asList(DocType.values()).stream().map(i -> i.toString()) + .collect(Collectors.joining(", ")); + String msg = fileType.toString() + " is not supported, msdoc2txt supports " + types + " format"; + throw new IllegalArgumentException(msg); + } + } + + logger.info("end msdoc2txt for debugging"); + } +} \ No newline at end of file diff --git a/java/src/main/java/jp/cmscom/c2/transform/msoffice/MSDocExtracter.java b/java/src/main/java/jp/cmscom/c2/transform/msoffice/MSDocExtracter.java new file mode 100644 index 0000000..ebd3262 --- /dev/null +++ b/java/src/main/java/jp/cmscom/c2/transform/msoffice/MSDocExtracter.java @@ -0,0 +1,253 @@ +package jp.cmscom.c2.transform.msoffice; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import org.apache.poi.POIDocument; +import org.apache.poi.extractor.ExtractorFactory; +import org.apache.poi.extractor.POITextExtractor; +import org.apache.poi.hdgf.HDGFDiagram; +import org.apache.poi.hdgf.extractor.VisioTextExtractor; +import org.apache.poi.hpsf.SummaryInformation; +import org.apache.poi.hslf.usermodel.HSLFSlideShow; +import org.apache.poi.hslf.usermodel.HSLFSlideShowImpl; +import org.apache.poi.hssf.extractor.ExcelExtractor; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.hwpf.HWPFDocument; +import org.apache.poi.hwpf.extractor.WordExtractor; +import org.apache.poi.openxml4j.exceptions.OpenXML4JException; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.openxml4j.opc.PackagePart; +import org.apache.poi.openxml4j.opc.PackageRelationshipCollection; +import org.apache.poi.ooxml.POIXMLDocument; +import org.apache.poi.ooxml.POIXMLProperties; +import org.apache.poi.ooxml.POIXMLProperties.CoreProperties; +import org.apache.poi.ooxml.extractor.POIXMLExtractorFactory; +import org.apache.poi.poifs.filesystem.DirectoryNode; +import org.apache.poi.poifs.filesystem.Entry; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.sl.extractor.SlideShowExtractor; +import org.apache.poi.xslf.usermodel.XSLFRelation; +import org.apache.poi.xslf.usermodel.XSLFSlideShow; +import org.apache.poi.xssf.usermodel.XSSFRelation; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.apache.poi.xwpf.usermodel.XWPFRelation; +import org.apache.xmlbeans.XmlException; + +public class MSDocExtracter implements DocExtracter { + public static final String CORE_DOCUMENT_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; + + protected static Logger logger; + + protected static void setLogger(Logger l) { + logger = l; + } + + private Map>> document; + + @Override + public String getProperties() { + logger.info("=== properties ==="); + return this.join(document.get(DocPart.Properties)); + } + + @Override + public String getNotes() { + logger.info("=== notes ==="); + return this.join(document.get(DocPart.Notes)); + } + + @Override + public String getHeader() { + logger.info("=== header ==="); + return this.join(document.get(DocPart.Header)); + } + + @Override + public String getFooter() { + logger.info("=== footer ==="); + return this.join(document.get(DocPart.Footer)); + } + + @Override + public String getContents() { + logger.info("=== contents ==="); + return this.join(document.get(DocPart.Contents)); + } + + public MSDocExtracter() { + this.document = new HashMap>>(); + for (DocPart part : DocPart.values()) { + this.document.put(part, new ArrayList>()); + } + } + + public MSDocExtracter(POIFSFileSystem fs) throws IOException, OpenXML4JException, XmlException { + this(); + POIDocument doc = this.getDocument(fs); + this.extractMetadata(doc); + if (doc instanceof HSLFSlideShowImpl) { + // The slide show that was read for the metadata gives the text too. + // The extractor is not closed: that would close the file system. + this.extractContents(new SlideShowExtractor<>(new HSLFSlideShow((HSLFSlideShowImpl) doc))); + } else { + this.extractContents(ExtractorFactory.createExtractor(fs)); + } + } + + private void extractMetadata(POIDocument doc) { + if (doc == null) { + return; + } + + SummaryInformation summary = doc.getSummaryInformation(); + if (summary == null) { + return; + } + + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getAuthor())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getApplicationName())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getComments())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getLastAuthor())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getRevNumber())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getSubject())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getTemplate())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(summary.getTitle())); + } + + public POIDocument getDocument(POIFSFileSystem fs) throws IOException { + DirectoryNode dir = fs.getRoot(); + for (Iterator entries = dir.getEntries(); entries.hasNext();) { + Entry entry = (Entry) entries.next(); + String entryName = entry.getName(); + logger.info("entry name: " + entryName); + + if (entryName.equals("Workbook")) { + logger.info("document type: HSSFWorkbook"); + return new HSSFWorkbook(dir, fs, true); + } + if (entryName.equals("WordDocument")) { + logger.info("document type: HWPFDocument"); + return new HWPFDocument(fs); + } + if (entryName.equals("PowerPoint Document")) { + logger.info("document type: HSLFSlideShowImpl"); + return new HSLFSlideShowImpl(fs); + } + if (entryName.equals("VisioDocument")) { + logger.info("document type: VisioDocument"); + return new HDGFDiagram(fs); + } + } + + logger.info("document type: HSSFWorkbook"); + return new HSSFWorkbook(dir, fs, true); + } + + public MSDocExtracter(OPCPackage pkg) throws IOException, OpenXML4JException, XmlException { + this(); + this.extractMetadata(pkg); + POITextExtractor oleTextExtractor = new POIXMLExtractorFactory().create(pkg); + this.extractContents(oleTextExtractor); + } + + private void extractMetadata(OPCPackage pkg) throws IOException, OpenXML4JException, XmlException { + POIXMLDocument doc = this.getDocument(pkg); + if (doc == null) { + return; + } + + POIXMLProperties prop = doc.getProperties(); + CoreProperties cp = prop.getCoreProperties(); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getCategory())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getDescription())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getSubject())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getTitle())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getContentStatus())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getContentType())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getRevision())); + this.document.get(DocPart.Properties).add(Optional.ofNullable(cp.getCreator())); + } + + public POIXMLDocument getDocument(OPCPackage pkg) throws IOException, OpenXML4JException, XmlException { + PackageRelationshipCollection core = pkg.getRelationshipsByType(CORE_DOCUMENT_REL); + PackagePart corePart = pkg.getPart(core.getRelationship(0)); + + if (corePart.getContentType().equals(XSSFRelation.WORKBOOK.getContentType()) + || corePart.getContentType().equals(XSSFRelation.MACRO_TEMPLATE_WORKBOOK.getContentType()) + || corePart.getContentType().equals(XSSFRelation.MACRO_ADDIN_WORKBOOK.getContentType()) + || corePart.getContentType().equals(XSSFRelation.TEMPLATE_WORKBOOK.getContentType()) + || corePart.getContentType().equals(XSSFRelation.MACROS_WORKBOOK.getContentType())) { + logger.info("document type: XSSFWorkbook"); + return new XSSFWorkbook(pkg); + } + + if (corePart.getContentType().equals(XWPFRelation.DOCUMENT.getContentType()) + || corePart.getContentType().equals(XWPFRelation.TEMPLATE.getContentType()) + || corePart.getContentType().equals(XWPFRelation.MACRO_DOCUMENT.getContentType()) + || corePart.getContentType().equals(XWPFRelation.MACRO_TEMPLATE_DOCUMENT.getContentType())) { + logger.info("document type: XWPFDocument"); + return new XWPFDocument(pkg); + } + + if (corePart.getContentType().equals(XSLFRelation.MAIN.getContentType())) { + logger.info("document type: XSLFSlideShow"); + return new XSLFSlideShow(pkg); + } + + return null; + } + + private void extractContents(POITextExtractor textExtractor) { + // If the embedded object was an Excel spreadsheet. + if (textExtractor instanceof ExcelExtractor) { + logger.info("extracter is ExcelExtractor"); + ExcelExtractor excelExtractor = (ExcelExtractor) textExtractor; + this.document.get(DocPart.Contents).add(Optional.ofNullable(excelExtractor.getText())); + } + // A Word Document + else if (textExtractor instanceof WordExtractor) { + logger.info("extracter is WordExtractor"); + WordExtractor wordExtractor = (WordExtractor) textExtractor; + String[] paragraphText = wordExtractor.getParagraphText(); + for (String paragraph : paragraphText) { + this.document.get(DocPart.Contents).add(Optional.ofNullable(paragraph)); + } + this.document.get(DocPart.Footer).add(Optional.ofNullable(wordExtractor.getFooterText())); + this.document.get(DocPart.Header).add(Optional.ofNullable(wordExtractor.getHeaderText())); + } + // PowerPoint Presentation (.ppt and .pptx). + else if (textExtractor instanceof SlideShowExtractor) { + logger.info("extracter is SlideShowExtractor"); + SlideShowExtractor slideShowExtractor = (SlideShowExtractor) textExtractor; + this.document.get(DocPart.Contents).add(Optional.ofNullable(slideShowExtractor.getText())); + slideShowExtractor.setSlidesByDefault(false); + slideShowExtractor.setNotesByDefault(true); + this.document.get(DocPart.Notes).add(Optional.ofNullable(slideShowExtractor.getText())); + } + // Visio Drawing + else if (textExtractor instanceof VisioTextExtractor) { + logger.info("extracter is VisioTextExtractor"); + VisioTextExtractor visioTextExtractor = (VisioTextExtractor) textExtractor; + this.document.get(DocPart.Contents).add(Optional.ofNullable(visioTextExtractor.getText())); + } + // Others + else { + logger.info("extracter is Others"); + this.document.get(DocPart.Contents).add(Optional.ofNullable(textExtractor.getText())); + } + } + + private String join(List> data) { + return data.stream().filter(o -> o.isPresent()).map(o -> o.get()) + .collect(Collectors.joining(System.lineSeparator())); + } +} \ No newline at end of file diff --git a/java/src/main/java/jp/cmscom/c2/transform/msoffice/RTFExtracter.java b/java/src/main/java/jp/cmscom/c2/transform/msoffice/RTFExtracter.java new file mode 100644 index 0000000..3a1fce1 --- /dev/null +++ b/java/src/main/java/jp/cmscom/c2/transform/msoffice/RTFExtracter.java @@ -0,0 +1,30 @@ +package jp.cmscom.c2.transform.msoffice; + +import java.io.IOException; +import java.io.InputStream; +import java.util.logging.Logger; + +import com.rtfparserkit.converter.text.StringTextConverter; +import com.rtfparserkit.parser.RtfStreamSource; + +public class RTFExtracter implements DocExtracter { + + protected static Logger logger; + + protected static void setLogger(Logger l) { + logger = l; + } + + private String contents; + + @Override + public String getContents() { + return this.contents; + } + + public RTFExtracter(InputStream is) throws IOException { + StringTextConverter converter = new StringTextConverter(); + converter.convert(new RtfStreamSource(is)); + this.contents = converter.getText(); + } +} \ No newline at end of file diff --git a/java/src/main/resources/log4j2.component.properties b/java/src/main/resources/log4j2.component.properties new file mode 100644 index 0000000..e5b69e6 --- /dev/null +++ b/java/src/main/resources/log4j2.component.properties @@ -0,0 +1,5 @@ +# POI uses the Log4j API. Without a logging implementation the API prints a +# warning to stdout, which would end up in the extracted text. Use the simple +# provider of the API and turn it off. +log4j.provider=org.apache.logging.log4j.simple.internal.SimpleProvider +log4j2.simplelogLevel=OFF diff --git a/news/+java11.breaking b/news/+java11.breaking new file mode 100644 index 0000000..484e13d --- /dev/null +++ b/news/+java11.breaking @@ -0,0 +1 @@ +Java 11 or later is required (1.0b6 ran on Java 8). [terapyon] diff --git a/news/+poi5.feature b/news/+poi5.feature new file mode 100644 index 0000000..3d183f0 --- /dev/null +++ b/news/+poi5.feature @@ -0,0 +1 @@ +The bundled jar is built with Apache POI 5.5.1 (was 3.17). Its Java source is now in `java/`, with a reproducible build that CI checks. [terapyon] diff --git a/news/+ppt.bugfix b/news/+ppt.bugfix new file mode 100644 index 0000000..d9d9c50 --- /dev/null +++ b/news/+ppt.bugfix @@ -0,0 +1 @@ +Extract text from `.ppt` files (PowerPoint 97-2003). The bundled jar closed the file before it read the text, so every `.ppt` gave no text. [terapyon] diff --git a/news/+ppt.documentation b/news/+ppt.documentation deleted file mode 100644 index c7a8861..0000000 --- a/news/+ppt.documentation +++ /dev/null @@ -1 +0,0 @@ -Document a known issue: the bundled jar cannot extract text from `.ppt` files (this was already the case in 1.0b6). [terapyon] diff --git a/src/c2/transform/msoffice/javasrc/README.md b/src/c2/transform/msoffice/javasrc/README.md deleted file mode 100644 index 35bfb08..0000000 --- a/src/c2/transform/msoffice/javasrc/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Java packeage - -https://bitbucket.org/cmscom/msdoc2txt - -# build - -Please see `README.md` at msdoc2txt pacackege - -# Usage - -``` -$ java -jar msdoc2txt.jar < DOC_FILE.doc -``` diff --git a/src/c2/transform/msoffice/msdoc2txt.jar b/src/c2/transform/msoffice/msdoc2txt.jar index b81204d..3f4b501 100644 Binary files a/src/c2/transform/msoffice/msdoc2txt.jar and b/src/c2/transform/msoffice/msdoc2txt.jar differ diff --git a/tests/samples/presentation-notes.ppt b/tests/samples/presentation-notes.ppt new file mode 100644 index 0000000..d5a4b92 Binary files /dev/null and b/tests/samples/presentation-notes.ppt differ diff --git a/tests/samples/presentation-notes.pptx b/tests/samples/presentation-notes.pptx new file mode 100644 index 0000000..f1b6754 Binary files /dev/null and b/tests/samples/presentation-notes.pptx differ diff --git a/tests/test_java.py b/tests/test_java.py index 6bb6a38..97ec12b 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -105,6 +105,16 @@ def test_docx_text(self, portal): PPTX, ["Presentation slide example", "スライドの例です。"], ), + ( + "presentation-notes.pptx", + PPTX, + [ + "Presentation with notes", + "ノート付きのスライドです。", + "Speaker notes example", + "発表者のノートです。", + ], + ), ], ) def test_other_formats(self, portal, name, content_type, expected): @@ -117,33 +127,28 @@ def test_document_without_text(self, portal, caplog): assert transformed(portal, "empty.docx", DOCX) == "" assert caplog.records == [] - def test_ppt_gives_no_text_with_the_bundled_jar(self, portal, caplog): - """Known issue: the bundled jar cannot read .ppt files (see the xfail).""" - obj = add_file(portal, "presentation.ppt", PPT) - assert obj.file.data == (SAMPLES / "presentation.ppt").read_bytes() - # Creating the File indexes it once already. - caplog.clear() - with caplog.at_level(logging.ERROR, logger="c2.transform.msoffice"): - text = SearchableText_file(obj)() - assert "Presentation slide example" not in text - assert [r.getMessage() for r in caplog.records] == [ - "msdoc2txt failed with exit code 1" - ] - - @pytest.mark.xfail( - strict=True, - reason=( - "The bundled msdoc2txt.jar (POI 3.17) fails on every .ppt: " - "CurrentUserAtom. -> 'Block 3 not found' / 'Unable to read 512 " - "bytes from 2048 in stream of length -1'. MSDocExtracter seems to close " - "the POIFS before HSLF reads the Current User stream. To be fixed when " - "the jar is updated." - ), + @pytest.mark.parametrize( + ("name", "expected"), + [ + ( + "presentation.ppt", + ["Presentation slide example", "スライドの例です。"], + ), + ( + "presentation-notes.ppt", + [ + "Presentation with notes", + "ノート付きのスライドです。", + "Speaker notes example", + "発表者のノートです。", + ], + ), + ], ) - def test_ppt_text_is_searchable(self, portal): - text = text_of(portal, "presentation.ppt", PPT) - assert "Presentation slide example" in text - assert "スライドの例です。" in text + def test_ppt_text_is_searchable(self, portal, name, expected): + text = text_of(portal, name, PPT) + for part in expected: + assert part in text def test_saving_works_without_java(portal, monkeypatch, tmp_path, caplog):