From 17c9f723c45baa65fbafa362d66a62394755b792 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 18:48:46 +0900 Subject: [PATCH 01/13] Move the package into src/ and drop the pkg_resources namespace Move the code to src/c2/transform/msoffice/ with git mv so that history follows the files, and remove c2/__init__.py and c2/transform/__init__.py to make c2 and c2.transform PEP 420 namespace packages. Also move README.txt to README.md, docs/HISTORY.txt to CHANGELOG.md and the license files to the top level, and remove .hgignore. File contents are unchanged; they are rewritten in later commits. Co-Authored-By: Claude Opus 5.5 --- .hgignore | 18 ------------------ docs/HISTORY.txt => CHANGELOG.md | 0 docs/LICENSE.GPL => LICENSE.GPL | 0 docs/LICENSE.txt => LICENSE.txt | 0 README.txt => README.md | 0 c2/__init__.py | 6 ------ c2/transform/__init__.py | 6 ------ {c2 => src/c2}/transform/msoffice/__init__.py | 0 {c2 => src/c2}/transform/msoffice/config.py | 0 .../c2}/transform/msoffice/configure.zcml | 0 .../c2}/transform/msoffice/javasrc/README.md | 0 .../c2}/transform/msoffice/msdoc2txt.jar | Bin .../profiles/default/import_steps.xml | 0 .../msoffice/profiles/default/metadata.xml | 0 .../c2}/transform/msoffice/setuphandlers.py | 0 .../c2}/transform/msoffice/tests/__init__.py | 0 .../c2}/transform/msoffice/tests/test_main.py | 0 .../msoffice/tests/wordprocessing1.docx | Bin .../c2}/transform/msoffice/transform.py | 0 {c2 => src/c2}/transform/msoffice/version.txt | 0 20 files changed, 30 deletions(-) delete mode 100644 .hgignore rename docs/HISTORY.txt => CHANGELOG.md (100%) rename docs/LICENSE.GPL => LICENSE.GPL (100%) rename docs/LICENSE.txt => LICENSE.txt (100%) rename README.txt => README.md (100%) delete mode 100644 c2/__init__.py delete mode 100644 c2/transform/__init__.py rename {c2 => src/c2}/transform/msoffice/__init__.py (100%) rename {c2 => src/c2}/transform/msoffice/config.py (100%) rename {c2 => src/c2}/transform/msoffice/configure.zcml (100%) rename {c2 => src/c2}/transform/msoffice/javasrc/README.md (100%) rename {c2 => src/c2}/transform/msoffice/msdoc2txt.jar (100%) rename {c2 => src/c2}/transform/msoffice/profiles/default/import_steps.xml (100%) rename {c2 => src/c2}/transform/msoffice/profiles/default/metadata.xml (100%) rename {c2 => src/c2}/transform/msoffice/setuphandlers.py (100%) rename {c2 => src/c2}/transform/msoffice/tests/__init__.py (100%) rename {c2 => src/c2}/transform/msoffice/tests/test_main.py (100%) rename {c2 => src/c2}/transform/msoffice/tests/wordprocessing1.docx (100%) rename {c2 => src/c2}/transform/msoffice/transform.py (100%) rename {c2 => src/c2}/transform/msoffice/version.txt (100%) diff --git a/.hgignore b/.hgignore deleted file mode 100644 index ddf4120..0000000 --- a/.hgignore +++ /dev/null @@ -1,18 +0,0 @@ -syntax: glob -*~ -*.pyc -*.pyo -dist/* -build/* -c2.*.*.egg-info/* -.semantic.cache -.DS_Store -TAGS -.DS_Store -.svn -.hg -.hgsvn -.as2proj -.as3proj -Thumbs.db -.~lock* diff --git a/docs/HISTORY.txt b/CHANGELOG.md similarity index 100% rename from docs/HISTORY.txt rename to CHANGELOG.md diff --git a/docs/LICENSE.GPL b/LICENSE.GPL similarity index 100% rename from docs/LICENSE.GPL rename to LICENSE.GPL diff --git a/docs/LICENSE.txt b/LICENSE.txt similarity index 100% rename from docs/LICENSE.txt rename to LICENSE.txt diff --git a/README.txt b/README.md similarity index 100% rename from README.txt rename to README.md diff --git a/c2/__init__.py b/c2/__init__.py deleted file mode 100644 index f48ad10..0000000 --- a/c2/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/c2/transform/__init__.py b/c2/transform/__init__.py deleted file mode 100644 index f48ad10..0000000 --- a/c2/transform/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/c2/transform/msoffice/__init__.py b/src/c2/transform/msoffice/__init__.py similarity index 100% rename from c2/transform/msoffice/__init__.py rename to src/c2/transform/msoffice/__init__.py diff --git a/c2/transform/msoffice/config.py b/src/c2/transform/msoffice/config.py similarity index 100% rename from c2/transform/msoffice/config.py rename to src/c2/transform/msoffice/config.py diff --git a/c2/transform/msoffice/configure.zcml b/src/c2/transform/msoffice/configure.zcml similarity index 100% rename from c2/transform/msoffice/configure.zcml rename to src/c2/transform/msoffice/configure.zcml diff --git a/c2/transform/msoffice/javasrc/README.md b/src/c2/transform/msoffice/javasrc/README.md similarity index 100% rename from c2/transform/msoffice/javasrc/README.md rename to src/c2/transform/msoffice/javasrc/README.md diff --git a/c2/transform/msoffice/msdoc2txt.jar b/src/c2/transform/msoffice/msdoc2txt.jar similarity index 100% rename from c2/transform/msoffice/msdoc2txt.jar rename to src/c2/transform/msoffice/msdoc2txt.jar diff --git a/c2/transform/msoffice/profiles/default/import_steps.xml b/src/c2/transform/msoffice/profiles/default/import_steps.xml similarity index 100% rename from c2/transform/msoffice/profiles/default/import_steps.xml rename to src/c2/transform/msoffice/profiles/default/import_steps.xml diff --git a/c2/transform/msoffice/profiles/default/metadata.xml b/src/c2/transform/msoffice/profiles/default/metadata.xml similarity index 100% rename from c2/transform/msoffice/profiles/default/metadata.xml rename to src/c2/transform/msoffice/profiles/default/metadata.xml diff --git a/c2/transform/msoffice/setuphandlers.py b/src/c2/transform/msoffice/setuphandlers.py similarity index 100% rename from c2/transform/msoffice/setuphandlers.py rename to src/c2/transform/msoffice/setuphandlers.py diff --git a/c2/transform/msoffice/tests/__init__.py b/src/c2/transform/msoffice/tests/__init__.py similarity index 100% rename from c2/transform/msoffice/tests/__init__.py rename to src/c2/transform/msoffice/tests/__init__.py diff --git a/c2/transform/msoffice/tests/test_main.py b/src/c2/transform/msoffice/tests/test_main.py similarity index 100% rename from c2/transform/msoffice/tests/test_main.py rename to src/c2/transform/msoffice/tests/test_main.py diff --git a/c2/transform/msoffice/tests/wordprocessing1.docx b/src/c2/transform/msoffice/tests/wordprocessing1.docx similarity index 100% rename from c2/transform/msoffice/tests/wordprocessing1.docx rename to src/c2/transform/msoffice/tests/wordprocessing1.docx diff --git a/c2/transform/msoffice/transform.py b/src/c2/transform/msoffice/transform.py similarity index 100% rename from c2/transform/msoffice/transform.py rename to src/c2/transform/msoffice/transform.py diff --git a/c2/transform/msoffice/version.txt b/src/c2/transform/msoffice/version.txt similarity index 100% rename from c2/transform/msoffice/version.txt rename to src/c2/transform/msoffice/version.txt From 25f9af00bd515e37fd0516564b6bbbae1d3ca2bc Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 18:57:03 +0900 Subject: [PATCH 02/13] Port the transform to Python 3 and add pyproject, tox and tests Packaging: - Replace setup.py, setup.cfg and the old MANIFEST.in with pyproject.toml (setuptools, src/ layout, PEP 420 namespaces) and a MANIFEST.in that grafts src/c2, so the jar is shipped. - Version 2.0.0.dev0, Plone 6.0-6.2, Python 3.10-3.14. - tox: ruff-check, py310-313 with Plone 6.0 and 6.1, py310-314 with Plone 6.2, installed with pip against the Plone constraints. pytest runs with -rs. C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA and JAVA_HOME are passed on. - towncrier (news/) and zest.releaser settings. Transform: - Run java with subprocess.run: java, -Xmx512m, -Dfile.encoding=UTF-8, -Dstdout.encoding=UTF-8, -jar, the jar. Timeout 60 seconds. - C2_TRANSFORM_MSOFFICE_TIMEOUT and C2_TRANSFORM_MSOFFICE_JAVA_OPTS change the timeout and the JVM options; the encoding options always come last. - The exit code decides success. Exit code 0 with empty output is a document without text. Output is decoded with errors="replace". - On failure the text is empty and the error is logged with at most 500 characters of stderr, never the document. - str input is logged and gives empty text instead of an assert. - Add application/vnd.ms-powerpoint to inputs. Drop the Windows and Plone 3 code paths. config.py keeps adding the Office types to the standard mimetypes module on import, now also imported from the package's __init__. setuphandlers.py only gets an absolute import and formatting here so that ruff passes; it is rewritten with tests in the next commit. The old PloneTestCase test is removed. RED: - tests/test_imports.py: ModuleNotFoundError: No module named 'config'. - tests/test_transform.py: ImportError: cannot import name 'implements' from 'zope.interface'. - tests/test_config.py on py312-Plone62 with add_type commented out: docm, pptm, pptx, xlsb, xlsm came back as None. Mutations of transform.py, each caught by tests/test_transform.py: dropping the encoding options, putting them before the JVM options, deciding by stderr, strict decoding, dropping the str guard, logging the whole stderr, accepting a timeout of 0, logging empty output as an error. Co-Authored-By: Claude Opus 5.5 --- .editorconfig | 35 +++ .gitignore | 51 ++++ MANIFEST.in | 11 +- constraints_plone60.txt | 2 + constraints_plone61.txt | 2 + constraints_plone62.txt | 2 + docs/INSTALL.txt | 47 ---- news/.changelog_template.jinja | 15 ++ news/.gitkeep | 0 pyproject.toml | 178 ++++++++++++++ requirements_plone60.txt | 2 + requirements_plone61.txt | 2 + requirements_plone62.txt | 2 + setup.cfg | 2 - setup.py | 36 --- src/c2/transform/msoffice/__init__.py | 13 +- src/c2/transform/msoffice/config.py | 80 +++--- src/c2/transform/msoffice/setuphandlers.py | 35 +-- src/c2/transform/msoffice/testing.py | 27 +++ src/c2/transform/msoffice/tests/__init__.py | 2 - src/c2/transform/msoffice/tests/test_main.py | 144 ----------- src/c2/transform/msoffice/transform.py | 243 +++++++++---------- src/c2/transform/msoffice/version.txt | 1 - tests/conftest.py | 14 ++ tests/test_config.py | 47 ++++ tests/test_imports.py | 12 + tests/test_transform.py | 220 +++++++++++++++++ tox.ini | 82 +++++++ 28 files changed, 888 insertions(+), 419 deletions(-) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 constraints_plone60.txt create mode 100644 constraints_plone61.txt create mode 100644 constraints_plone62.txt delete mode 100644 docs/INSTALL.txt create mode 100644 news/.changelog_template.jinja create mode 100644 news/.gitkeep create mode 100644 pyproject.toml create mode 100644 requirements_plone60.txt create mode 100644 requirements_plone61.txt create mode 100644 requirements_plone62.txt delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 src/c2/transform/msoffice/testing.py delete mode 100644 src/c2/transform/msoffice/tests/__init__.py delete mode 100644 src/c2/transform/msoffice/tests/test_main.py delete mode 100644 src/c2/transform/msoffice/version.txt create mode 100644 tests/conftest.py create mode 100644 tests/test_config.py create mode 100644 tests/test_imports.py create mode 100644 tests/test_transform.py create mode 100644 tox.ini diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b3561a7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,35 @@ + +# EditorConfig Configuration file, for more details see: +# https://EditorConfig.org +# EditorConfig is a convention description, that could be interpreted +# by multiple editors to enforce common coding conventions for specific +# file types + +# top-most EditorConfig file: +# Will ignore other EditorConfig files in Home directory or upper tree level. +root = true + + +[*] # For All Files +# Unix-style newlines with a newline ending every file +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +# Set default charset +charset = utf-8 +# Indent style default +indent_style = space + +[*.{py,cfg,ini}] +# 4 space indentation +indent_size = 4 + +[*.{html,dtml,pt,zpt,xml,zcml,js,json,ts,less,scss,css,sass,yml,yaml}] +# 2 space indentation +indent_size = 2 + +[{Makefile,.gitmodules}] +# Tab indentation (no size specified, but view as 4 spaces) +indent_style = tab +indent_size = unset +tab_width = unset diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..461e060 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# python related +*.egg-info +*.pyc +*.pyo + +# tools related +__pycache__/ +.coverage +.coverage.* +.mypy_cache +.pytest_cache +.ruff_cache +/build/ +coverage.xml +dist/ +docs/_build +node_modules/ + +# venv / buildout related +.eggs/ +.installed.cfg +.mr.developer.cfg +.venv/ +bin/ +develop-eggs/ +eggs/ +etc/ +include/ +lib/ +lib64 +parts/ +pyvenv.cfg +var/ + +# mxdev +.installed.txt +.lock +/.mxdev_cache/ +/.make-sentinels/ +/*-mxdev.txt +/instance/ +/reports/ +/sources/ +/venv/ +# mxdev generated +constraints.txt +constraints-mxdev.txt +requirements.txt +requirements-mxdev.txt + +.tox/ diff --git a/MANIFEST.in b/MANIFEST.in index 0355823..09b5ed8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,8 @@ -recursive-include c2 * -recursive-include docs * -global-exclude *pyc *pyo +graft src/c2 +include *.md +include pyproject.toml +exclude tox.ini +exclude *.txt +global-exclude *.pyc +include LICENSE.GPL +include LICENSE.txt diff --git a/constraints_plone60.txt b/constraints_plone60.txt new file mode 100644 index 0000000..693399a --- /dev/null +++ b/constraints_plone60.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.0-latest/constraints.txt +ruff>=0.4.0 diff --git a/constraints_plone61.txt b/constraints_plone61.txt new file mode 100644 index 0000000..d40bf26 --- /dev/null +++ b/constraints_plone61.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.1-latest/constraints.txt +ruff>=0.4.0 diff --git a/constraints_plone62.txt b/constraints_plone62.txt new file mode 100644 index 0000000..c6ba411 --- /dev/null +++ b/constraints_plone62.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.2-latest/constraints.txt +ruff>=0.4.0 diff --git a/docs/INSTALL.txt b/docs/INSTALL.txt deleted file mode 100644 index 6f29fd1..0000000 --- a/docs/INSTALL.txt +++ /dev/null @@ -1,47 +0,0 @@ -c2.transform.msoffice Installation -===================================== - -* When you're reading this you have probably already run - ``easy_install c2.transform.msoffice``. Find out how to install setuptools - (and EasyInstall) here: - http://peak.telecommunity.com/DevCenter/EasyInstall - -* Get `pythonproducts`_ and install it via:: - - python setup.py install --home /path/to/instance - - into your Zope instance. - -* Create a file called ``c2.transform.msoffice-configure.zcml`` in the - ``/path/to/instance/etc/package-includes`` directory. The file - should only contain this:: - - - -.. _pythonproducts: http://plone.org/products/pythonproducts - -Alternatively, if you are using zc.buildout and the plone.recipe.zope2instance -recipe to manage your project, you can do this: - -* Add ``c2.transform.msoffice`` to the list of eggs to install, e.g.:: - - [buildout] - ... - eggs = - ... - c2.transform.msoffice - -* Tell the plone.recipe.zope2instance recipe to install a ZCML slug:: - - [instance] - recipe = plone.recipe.zope2instance - ... - zcml = - c2.transform.msoffice - -* Re-run buildout, e.g. with:: - - $ ./bin/buildout - -You can skip the ZCML slug if you are going to explicitly include the package -from another package's configure.zcml file. diff --git a/news/.changelog_template.jinja b/news/.changelog_template.jinja new file mode 100644 index 0000000..0cf429a --- /dev/null +++ b/news/.changelog_template.jinja @@ -0,0 +1,15 @@ +{% if sections[""] %} +{% for category, val in definitions.items() if category in sections[""] %} + +### {{ definitions[category]['name'] }} + +{% for text, values in sections[""][category].items() %} +- {{ text }} {{ values|join(', ') }} +{% endfor %} + +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} diff --git a/news/.gitkeep b/news/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4d32c39 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,178 @@ +[build-system] +requires = ["setuptools>=69.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "c2.transform.msoffice" +version = "2.0.0.dev0" +description = "Make Microsoft Office files searchable in Plone" +readme = "README.md" +license = {text = "GPL version 2 or later"} +requires-python = ">=3.10,<3.15" +authors = [ + { name = "Manabu TERADA", email = "terada@cmscom.jp" }, +] +keywords = [ + "Plone", + "transform", + "MS Office", + "search", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Plone", + "Framework :: Plone :: 6.0", + "Framework :: Plone :: 6.1", + "Framework :: Plone :: 6.2", + "Framework :: Plone :: Addon", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: POSIX", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "plone.base", + "Products.CMFPlone", + "Products.GenericSetup", + "Products.MimetypesRegistry", + "Products.PortalTransforms", +] + +[project.optional-dependencies] +test = [ + "horse-with-no-namespace", + "plone.app.contenttypes", + "plone.app.testing", + "pytest", + "pytest-cov", + "pytest-plone>=0.5.0", +] +release = [ + "zest.releaser[recommended]", + "zestreleaser.towncrier", + "zest.pocompile", +] + +[project.urls] +Homepage = "https://github.com/cmscom/c2.transform.msoffice" +PyPI = "https://pypi.org/project/c2.transform.msoffice" +Source = "https://github.com/cmscom/c2.transform.msoffice" +Tracker = "https://github.com/cmscom/c2.transform.msoffice/issues" + +[project.entry-points."z3c.autoinclude.plugin"] +target = "plone" + +[tool.setuptools] +include-package-data = true +zip-safe = false + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = true + +[tool.towncrier] +directory = "news/" +filename = "CHANGELOG.md" +start_string = "\n" +title_format = "## {version} ({project_date})" +template = "news/.changelog_template.jinja" +issue_format = "[#{issue}](https://github.com/cmscom/c2.transform.msoffice/issues/{issue})" +underlines = ["", "", ""] + +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking changes:" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "New features:" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug fixes:" +showcontent = true + +[[tool.towncrier.type]] +directory = "internal" +name = "Internal:" +showcontent = true + +[[tool.towncrier.type]] +directory = "documentation" +name = "Documentation:" +showcontent = true + +[[tool.towncrier.type]] +directory = "tests" +name = "Tests" +showcontent = true + +[tool.ruff] +target-version = "py310" +line-length = 88 +fix = true +lint.select = [ + # flake8-2020 + "YTT", + # flake8-bandit + "S", + # flake8-bugbear + "B", + # flake8-builtins + "A", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", + # flake8-simplify + "SIM", + # mccabe + "C90", + # pycodestyle + "E", "W", + # pyflakes + "F", + # pygrep-hooks + "PGH", + # pyupgrade + "UP", + # ruff + "RUF", +] +lint.ignore = [ + # DoNotAssignLambda + "E731", +] + +[tool.ruff.lint.isort] +case-sensitive = false +no-sections = true +force-single-line = true +from-first = true +lines-after-imports = 2 +lines-between-types = 1 +order-by-type = false + +[tool.ruff.format] +preview = true + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["E501", "RUF001", "S101"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.run] +source_pkgs = ["c2.transform.msoffice", "tests"] +branch = true +parallel = true + +[tool.zest-releaser] +python-file-with-version = "src/c2/transform/msoffice/__init__.py" diff --git a/requirements_plone60.txt b/requirements_plone60.txt new file mode 100644 index 0000000..f5fbf6d --- /dev/null +++ b/requirements_plone60.txt @@ -0,0 +1,2 @@ +-c constraints_plone60.txt +setuptools diff --git a/requirements_plone61.txt b/requirements_plone61.txt new file mode 100644 index 0000000..443b65f --- /dev/null +++ b/requirements_plone61.txt @@ -0,0 +1,2 @@ +-c constraints_plone61.txt +setuptools diff --git a/requirements_plone62.txt b/requirements_plone62.txt new file mode 100644 index 0000000..471765c --- /dev/null +++ b/requirements_plone62.txt @@ -0,0 +1,2 @@ +-c constraints_plone62.txt +setuptools diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 21cced6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[egg_info] -tag_svn_revision = false diff --git a/setup.py b/setup.py deleted file mode 100644 index e7ffa45..0000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -from setuptools import setup, find_packages -import os - -version = '1.0b6' - -setup(name='c2.transform.msoffice', - version=version, - description="This product enables to search MS-Office files on Plone.", - long_description=open("README.txt").read() + "\n" + - open(os.path.join("docs", "HISTORY.txt")).read() + "\n" + - open(os.path.join("docs", "INSTALL.txt")).read(), - # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - "Framework :: Plone", - "Programming Language :: Python", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - keywords='Plone transform MS Office search', - author='Manabu Terada', - author_email='terada@cmscom.jp', - url='http://www.cmscom.jp', - license='GPL', - packages=find_packages(exclude=['ez_setup']), - namespace_packages=['c2', 'c2.transform'], - include_package_data=True, - zip_safe=False, - install_requires=[ - 'setuptools', - # -*- Extra requirements: -*- - ], - entry_points=""" - # -*- Entry points: -*- - [z3c.autoinclude.plugin] - target = plone - """, - ) diff --git a/src/c2/transform/msoffice/__init__.py b/src/c2/transform/msoffice/__init__.py index 332b398..dd91d5d 100644 --- a/src/c2/transform/msoffice/__init__.py +++ b/src/c2/transform/msoffice/__init__.py @@ -1,8 +1,13 @@ +"""Make Microsoft Office files searchable in Plone.""" + +# Importing config registers the Office types with the mimetypes module. +from c2.transform.msoffice import config # noqa: F401 import logging -import config -logger = logging.getLogger(config.PROJECTNAME) -# def initialize(context): -# """Initializer called when used as a Zope 2 product.""" +__version__ = "2.0.0.dev0" + +PACKAGE_NAME = "c2.transform.msoffice" + +logger = logging.getLogger(PACKAGE_NAME) diff --git a/src/c2/transform/msoffice/config.py b/src/c2/transform/msoffice/config.py index b5cb74d..d6d659b 100644 --- a/src/c2/transform/msoffice/config.py +++ b/src/c2/transform/msoffice/config.py @@ -1,48 +1,60 @@ -#!/usr/bin/env python -# encoding: utf-8 import mimetypes -PROJECTNAME = 'c2.transform.msoffice' -PROJECT_GLOBALS = globals() -SITE_CHARSET = 'utf-8' -TRANSFORM_NAME = 'msoffice_to_text' + +PROJECTNAME = "c2.transform.msoffice" +SITE_CHARSET = "utf-8" +TRANSFORM_NAME = "msoffice_to_text" office_mimetypes = ( - {'name': 'Office Word 2007 XML macro enabled document', - 'mimetypes': ('application/vnd.ms-word.document.macroEnabled.12',), - 'extensions': ('docm',) + { + "name": "Office Word 2007 XML macro enabled document", + "mimetypes": ("application/vnd.ms-word.document.macroEnabled.12",), + "extensions": ("docm",), + }, + { + "name": "Office Word 2007 XML document", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + "extensions": ("docx",), }, - {'name': 'Office Word 2007 XML document', - 'mimetypes': ('application/vnd.openxmlformats-officedocument.wordprocessingml.document',), - 'extensions': ('docx',) - }, - {'name': 'Office Powerpoint 2007 macro-enabled XML presentation', - 'mimetypes': ('application/vnd.ms-powerpoint.presentation.macroEnabled.12',), - 'extensions': ('pptm',) + { + "name": "Office Powerpoint 2007 macro-enabled XML presentation", + "mimetypes": ("application/vnd.ms-powerpoint.presentation.macroEnabled.12",), + "extensions": ("pptm",), }, - {'name': 'Office Powerpoint 2007 XML presentation', - 'mimetypes': ('application/vnd.openxmlformats-officedocument.presentationml.presentation',), - 'extensions': ('pptx',) + { + "name": "Office Powerpoint 2007 XML presentation", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ), + "extensions": ("pptx",), }, - {'name': 'Office Excel 2007 binary workbook (BIFF12)', - 'mimetypes': ('application/vnd.ms-excel.sheet.binary.macroEnabled.12',), - 'extensions': ('xlsb',) + { + "name": "Office Excel 2007 binary workbook (BIFF12)", + "mimetypes": ("application/vnd.ms-excel.sheet.binary.macroEnabled.12",), + "extensions": ("xlsb",), }, - {'name': 'Office Excel 2007 XML macro-enabled workbook', - 'mimetypes': ('application/vnd.ms-excel.sheet.macroEnabled.12',), - 'extensions': ('xlsm',) + { + "name": "Office Excel 2007 XML macro-enabled workbook", + "mimetypes": ("application/vnd.ms-excel.sheet.macroEnabled.12",), + "extensions": ("xlsm",), }, - {'name': 'Office Excel 2007 XML workbook', - 'mimetypes': ('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',), - 'extensions': ('xlsx',) + { + "name": "Office Excel 2007 XML workbook", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + "extensions": ("xlsx",), }, - ) +) for mt in office_mimetypes: - mt['globs'] = tuple(['*.' + ext for ext in mt['extensions']]) - mt['icon_path'] = 'application.png' - # Adding to standard mimetypes - mimetypes.add_type(mt['mimetypes'][0], '.' + mt['extensions'][0]) + mt["globs"] = tuple("*." + ext for ext in mt["extensions"]) + mt["icon_path"] = "application.png" + # plone.namedfile guesses a missing content type with the standard library. + # Python < 3.14 does not know these types and slim images have no + # /etc/mime.types, so they are added on import, in every process. + mimetypes.add_type(mt["mimetypes"][0], "." + mt["extensions"][0]) del mimetypes - diff --git a/src/c2/transform/msoffice/setuphandlers.py b/src/c2/transform/msoffice/setuphandlers.py index 843f3b9..0601359 100644 --- a/src/c2/transform/msoffice/setuphandlers.py +++ b/src/c2/transform/msoffice/setuphandlers.py @@ -1,37 +1,41 @@ -#!/usr/bin/env python -# encoding: utf-8 from Products.CMFCore.utils import getToolByName -import config +from c2.transform.msoffice import config from c2.transform.msoffice import logger + def setupMSOfficeFile(context): """Add our transform in PT for MS Office""" site = context.getSite() - + # Adding our file types to MTR - mtr = getToolByName(site, 'mimetypes_registry') + mtr = getToolByName(site, "mimetypes_registry") for mt_dict in config.office_mimetypes: - main_mt = mt_dict['mimetypes'][0] - mt_name = mt_dict['name'] + main_mt = mt_dict["mimetypes"][0] + mt_name = mt_dict["name"] if bool(mtr.lookup(main_mt)): # Already installed - logger.info("%s (%s) Mime type already installed, skipped", main_mt, mt_name) + logger.info( + "%s (%s) Mime type already installed, skipped", main_mt, mt_name + ) continue mtr.manage_addMimeType( mt_name, - mt_dict['mimetypes'], - mt_dict['extensions'], - mt_dict['icon_path'], + mt_dict["mimetypes"], + mt_dict["extensions"], + mt_dict["icon_path"], binary=True, - globs=mt_dict['globs']) + globs=mt_dict["globs"], + ) logger.info("%s (%s) Mime type installed", main_mt, mt_name) # Registering our transform in PT - transforms_tool = getToolByName(site, 'portal_transforms') + transforms_tool = getToolByName(site, "portal_transforms") if config.TRANSFORM_NAME not in transforms_tool.objectIds(): # Not already installed - transforms_tool.manage_addTransform(config.TRANSFORM_NAME, 'c2.transform.msoffice.transform') + transforms_tool.manage_addTransform( + config.TRANSFORM_NAME, "c2.transform.msoffice.transform" + ) return @@ -43,7 +47,6 @@ def removeMSOfficeFile(context): site = context # Removing our transform from PT - transforms_tool = getToolByName(site, 'portal_transforms') + transforms_tool = getToolByName(site, "portal_transforms") transforms_tool.unregisterTransform(config.TRANSFORM_NAME) return - diff --git a/src/c2/transform/msoffice/testing.py b/src/c2/transform/msoffice/testing.py new file mode 100644 index 0000000..abc4114 --- /dev/null +++ b/src/c2/transform/msoffice/testing.py @@ -0,0 +1,27 @@ +from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE +from plone.app.testing import FunctionalTesting +from plone.app.testing import IntegrationTesting +from plone.app.testing import PloneSandboxLayer + +import c2.transform.msoffice + + +class Layer(PloneSandboxLayer): + defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,) + + def setUpZope(self, app, configurationContext): + # z3c.autoinclude is disabled in the Plone fixture base layer. + self.loadZCML(package=c2.transform.msoffice) + + +FIXTURE = Layer() + +INTEGRATION_TESTING = IntegrationTesting( + bases=(FIXTURE,), + name="C2.Transform.MsofficeLayer:IntegrationTesting", +) + +FUNCTIONAL_TESTING = FunctionalTesting( + bases=(FIXTURE,), + name="C2.Transform.MsofficeLayer:FunctionalTesting", +) diff --git a/src/c2/transform/msoffice/tests/__init__.py b/src/c2/transform/msoffice/tests/__init__.py deleted file mode 100644 index 6f804bc..0000000 --- a/src/c2/transform/msoffice/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 diff --git a/src/c2/transform/msoffice/tests/test_main.py b/src/c2/transform/msoffice/tests/test_main.py deleted file mode 100644 index 99486de..0000000 --- a/src/c2/transform/msoffice/tests/test_main.py +++ /dev/null @@ -1,144 +0,0 @@ -import unittest - -import os -import StringIO -from Products.CMFCore.utils import getToolByName -from Products.PloneTestCase import PloneTestCase - -from zope.testing import doctestunit -from zope.component import testing -from Testing import ZopeTestCase as ztc - -from Products.Five import zcml -from Products.Five import fiveconfigure -from Products.PloneTestCase import PloneTestCase as ptc -from Products.PloneTestCase.layer import PloneSite -from Products.MimetypesRegistry.interfaces import IMimetype - -from c2.transform.msoffice.config import TRANSFORM_NAME -from c2.transform.msoffice.config import office_mimetypes - -ptc.setupPloneSite() - -import c2.transform.msoffice - -class TestCase(ptc.PloneTestCase): - class layer(PloneSite): - @classmethod - def setUp(cls): - fiveconfigure.debug_mode = True - zcml.load_config('configure.zcml', - c2.transform.msoffice) - fiveconfigure.debug_mode = False - - @classmethod - def tearDown(cls): - pass - -class PTTestCase(TestCase): - """Testing portal_transforms settings""" - - def afterSetUp(self): - qi = self.portal.portal_quickinstaller - qi.installProduct('c2.transform.msoffice') - qi.installProduct('collective.indexing') - - portal = self.getPortal() - self.pt = getToolByName(portal, 'portal_transforms') - - - def testInstallation(self): - """Checking installation of our transform""" - - self.failUnless(TRANSFORM_NAME in self.pt.objectIds(spec='Transform'), - "%s transform expected" % TRANSFORM_NAME) - return - - - def testATfileSearchableText(self): - """Do we index the text of an openxml office file""" - - self.loginAsPortalOwner() - class fakefile(StringIO.StringIO): - pass - this_dir = os.path.dirname(os.path.abspath(__file__)) - test_filename = os.path.join(this_dir, 'wordprocessing1.docx') - fakefile = fakefile(file(test_filename, 'rb').read()) - fakefile.filename = 'wordprocessing1.docx' - file_id = self.portal.invokeFactory('File', fakefile.filename, file=fakefile) - file_item = getattr(self.portal, file_id) - # We sample some words from the file and its metadata - words = ("a", "simple", "example", "of", "document", "with", - "full", "set", "metadata", "comments") - st = file_item.SearchableText() - for word in words: - self.failUnless(word in st, "Expected '%s' in indexable text" % word) - return - - -class MTRTestCase(TestCase): - - def afterSetUp(self): - portal = self.getPortal() - self.mtr = getToolByName(portal, 'mimetypes_registry') - - - def testInstallation(self): - """Checking installation of our Mime types""" - providedBy = getattr(IMimetype, 'providedBy', - getattr(IMimetype, 'isImplementedBy', None)) - mtr = self.mtr - for mt_dict in office_mimetypes: - mt_string = mt_dict['mimetypes'][0] - mimetype = mtr.lookup(mt_string) - self.failUnless( - len(mimetype) > 0, - "Didn't find MimeType obj for %s" % mt_string) - if len(mimetype) > 0: - self.failUnless( - providedBy(mtr.lookup(mt_string)[0]), - "Didn't find MimeType obj for %s" % mt_string) - return - - - def testExtensions(self): - """Finding mimetypes by extension""" - - mtr = self.mtr - for mt_dict in office_mimetypes: - ext = mt_dict['extensions'][0] - expected = mt_dict['mimetypes'][0] - got = mtr.lookupExtension(ext).normalized() - self.failUnlessEqual(expected, got) - return - - - -def test_suite(): - return unittest.TestSuite([ - unittest.makeSuite(PTTestCase), - unittest.makeSuite(MTRTestCase), - - # Unit tests - #doctestunit.DocFileSuite( - # 'README.txt', package='c2.transform.msoffice', - # setUp=testing.setUp, tearDown=testing.tearDown), - - #doctestunit.DocTestSuite( - # module='c2.transform.msoffice.mymodule', - # setUp=testing.setUp, tearDown=testing.tearDown), - - - # Integration tests that use PloneTestCase - #ztc.ZopeDocFileSuite( - # 'README.txt', package='c2.transform.msoffice', - # test_class=TestCase), - - #ztc.FunctionalDocFileSuite( - # 'browser.txt', package='c2.transform.msoffice', - # test_class=TestCase), - - ]) - -if __name__ == '__main__': - unittest.main(defaultTest='test_suite') diff --git a/src/c2/transform/msoffice/transform.py b/src/c2/transform/msoffice/transform.py index 7de3546..c81ba7d 100644 --- a/src/c2/transform/msoffice/transform.py +++ b/src/c2/transform/msoffice/transform.py @@ -1,156 +1,139 @@ -#!/usr/bin/env python -# encoding: utf-8 +"""Extract plain text from Microsoft Office files with the bundled jar.""" + +from c2.transform.msoffice import logger +from c2.transform.msoffice.config import SITE_CHARSET +from c2.transform.msoffice.config import TRANSFORM_NAME +from pathlib import Path +from Products.PortalTransforms.interfaces import ITransform +from zope.interface import implementer + import os -import sys -import signal -import mimetypes +import shlex import subprocess -if sys.platform == 'win32': - isWIN = True - import time - try: - import ctypes - except ImportError: - ctypes = None -else: - isWIN = False - import select - import time - import threading - -from zope.interface import implements -try: - from Products.PortalTransforms.interfaces import ITransform - HAS_PLONE3 = False -except ImportError: - from Products.PortalTransforms.interfaces import itransform - HAS_PLONE3 = True -from config import SITE_CHARSET, TRANSFORM_NAME -from c2.transform.msoffice import logger -scpath = os.path.join(os.path.dirname(__file__), 'msdoc2txt.jar') - -def win_kill(pid): - if ctypes: - h = ctypes.windll.kernel32.OpenProcess(1, False, pid) - ctypes.windll.kernel32.TerminateProcess(h, -1) - ctypes.windll.kernel32.CloseHandle(h) - else: - subprocess.call(["TASKKILL", "/PID", str(pid), "/F"]) - -def win_timeout(p, timeout): - s = time.time() - while time.time() - s < timeout and p.poll() == None: - time.sleep(0.1) - if p.poll() == None: - win_kill(p.pid) - return False - return True - -def msdoc2txt(data, timeout=60): #default 60 sec - assert isinstance(data, str) - p = subprocess.Popen(("java", '-jar', '-Xmx512m', scpath), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - if not isWIN: - # Based on the code from http://betabug.ch/blogs/ch-athens/1093 - kill_check = threading.Event() - def _kill_process_after_a_timeout(pid): - os.kill(pid, signal.SIGTERM) - kill_check.set() # tell the main routine that we had to kill - # use SIGKILL if hard to kill... - return - - pid = p.pid - watchdog = threading.Timer(timeout, _kill_process_after_a_timeout, args=(pid, )) - watchdog.start() - (out,err) = p.communicate(input=data) - watchdog.cancel() # if it's still waiting to run - success = not kill_check.isSet() - kill_check.clear() - - if success: - if err: - status = False - val = err - elif out: - status = True - val = out - else: - status = None - val = "timeout" #if timeout - - return status, val - else: - r = win_timeout(p, timeout) - if r: - if p.poll(): - return False, p.stderr.read() #if fail - else: - return True, p.stdout.read() #if not fail - else: - return None, "timeout" #if timeout + +JAR_PATH = Path(__file__).with_name("msdoc2txt.jar") + +TIMEOUT_ENV = "C2_TRANSFORM_MSOFFICE_TIMEOUT" +JAVA_OPTS_ENV = "C2_TRANSFORM_MSOFFICE_JAVA_OPTS" +DEFAULT_TIMEOUT = 60 +DEFAULT_JAVA_OPTS = ("-Xmx512m",) +# The jar writes text in the JVM's default encoding. JDK 8-17 take it from +# file.encoding and JDK 19+ from stdout.encoding, so both are always set. +# They come after the options from the environment so they always win. +ENCODING_OPTS = ("-Dfile.encoding=UTF-8", "-Dstdout.encoding=UTF-8") +# How much of the jar's stderr (a Java stack trace) goes into the log. +STDERR_LOG_LIMIT = 500 + + +def timeout() -> int: + """Seconds to wait for the jar, from the environment or the default.""" + value = os.environ.get(TIMEOUT_ENV, "").strip() + if not value: + return DEFAULT_TIMEOUT + if value.isdigit() and int(value) > 0: + return int(value) + logger.warning( + "Ignoring %s=%r: expected a positive integer; using %d", + TIMEOUT_ENV, + value, + DEFAULT_TIMEOUT, + ) + return DEFAULT_TIMEOUT + + +def java_command() -> list[str]: + java_opts = shlex.split(os.environ.get(JAVA_OPTS_ENV, "")) or DEFAULT_JAVA_OPTS + return ["java", *java_opts, *ENCODING_OPTS, "-jar", str(JAR_PATH)] + + +def msdoc2txt(data: bytes) -> str | None: + """Return the text of an Office document, or None if extraction failed. + + The document goes to the jar on stdin and the text comes back on stdout. + The exit code decides success: an empty stdout with exit code 0 is a + document without text. Neither the document nor the text is logged. + """ + seconds = timeout() + try: + result = subprocess.run( # noqa: S603 + java_command(), input=data, capture_output=True, timeout=seconds + ) + except subprocess.TimeoutExpired: + # subprocess.run kills the process and waits for it. + logger.error("msdoc2txt timed out after %d seconds", seconds) + return None + except FileNotFoundError: + logger.error("msdoc2txt could not run: java was not found") + return None + except Exception: + logger.exception("msdoc2txt could not run") + return None + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace") + logger.error( + "msdoc2txt failed with exit code %d: %s", + result.returncode, + stderr[:STDERR_LOG_LIMIT], + ) + return None + if not result.stdout: + logger.debug("msdoc2txt returned no text") + return result.stdout.decode("utf-8", errors="replace") +@implementer(ITransform) class msoffice_to_text: - if HAS_PLONE3: - __implements__ = itransform - else: - implements(ITransform) __name__ = TRANSFORM_NAME inputs = ( # MS-Word formats - 'application/msword', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/vnd.ms-word.document.macroEnabled.12', - # MS-Excle formats - 'application/vnd.ms-excel', - 'application/msexcel', - 'application/x-msexcel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.ms-excel.sheet.binary.macroEnabled.12', - 'application/vnd.ms-excel.sheet.macroEnabled.12', + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-word.document.macroEnabled.12", + # MS-Excel formats + "application/vnd.ms-excel", + "application/msexcel", + "application/x-msexcel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-excel.sheet.macroEnabled.12", # MS-PowerPoint formats - 'application/powerpoint', - 'application/mspowerpoint', - 'application/x-mspowerpoint', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - 'application/vnd.ms-powerpoint.presentation.macroEnabled.12', - ) + "application/vnd.ms-powerpoint", + "application/powerpoint", + "application/mspowerpoint", + "application/x-mspowerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + ) - output = 'text/plain' + output = "text/plain" output_encoding = SITE_CHARSET - def __init__(self,name=None): + def __init__(self, name=None): if name: - self.__name__=name - return + self.__name__ = name def name(self): return self.__name__ def convert(self, orig, data, **kwargs): - #orig_file = kwargs.get('filename') or 'unknown.xxx' - mimetype = kwargs.get('mimetype') - filename = kwargs.get('filename') or 'unknown.xxx' - - if mimetype is None: - mimetype = mimetypes.guess_type(filename)[0] - try: - status, text = msdoc2txt(orig) - except Exception, e: - logger.error(e, exc_info=True) - data.setData('') + """Set the text of ``orig`` (bytes) on ``data``. + + On any failure the text is empty, so saving the file still works. + """ + if not isinstance(orig, bytes): + logger.error( + "msoffice_to_text expects bytes, got %s; no text is extracted", + type(orig).__name__, + ) + data.setData("") return data - if status: - data.setData(text) - else: - logger.error(text, exc_info=True) - data.setData('') + text = msdoc2txt(orig) + data.setData(text or "") return data + def register(): return msoffice_to_text() diff --git a/src/c2/transform/msoffice/version.txt b/src/c2/transform/msoffice/version.txt deleted file mode 100644 index eda7b34..0000000 --- a/src/c2/transform/msoffice/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.0b6 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b7b5d61 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +from c2.transform.msoffice.testing import FUNCTIONAL_TESTING +from c2.transform.msoffice.testing import INTEGRATION_TESTING +from pytest_plone import fixtures_factory + + +pytest_plugins = ["pytest_plone"] + + +globals().update( + fixtures_factory(( + (FUNCTIONAL_TESTING, "functional"), + (INTEGRATION_TESTING, "integration"), + )) +) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..8233741 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,47 @@ +from c2.transform.msoffice import config + +import importlib +import mimetypes +import pytest + + +@pytest.fixture +def stdlib_mimetypes_only(monkeypatch): + """Forget the host's mime.types (slim images do not have one). + + mimetypes.init(files=[]) keeps the current database, so it is replaced. + """ + monkeypatch.setattr(mimetypes, "_db", mimetypes.MimeTypes()) + + +@pytest.mark.usefixtures("stdlib_mimetypes_only") +def test_import_adds_office_types_to_stdlib_mimetypes(): + importlib.reload(config) + assert { + ext: mimetypes.guess_type(f"file.{ext}")[0] + for ext in ("docm", "docx", "pptm", "pptx", "xlsb", "xlsm", "xlsx") + } == { + "docm": "application/vnd.ms-word.document.macroEnabled.12", + "docx": ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ), + "pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "pptx": ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation" + ), + "xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + } + + +def test_office_mimetypes_have_globs_and_icon(): + assert config.office_mimetypes[1] == { + "name": "Office Word 2007 XML document", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + "extensions": ("docx",), + "globs": ("*.docx",), + "icon_path": "application.png", + } diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..962975d --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,12 @@ +def test_version(): + from c2.transform.msoffice import __version__ + + assert __version__ == "2.0.0.dev0" + + +def test_c2_is_a_pep420_namespace(): + import c2 + import c2.transform + + assert getattr(c2, "__file__", None) is None + assert getattr(c2.transform, "__file__", None) is None diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..05ae54b --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,220 @@ +"""Unit tests for the transform. ``java`` is replaced; Plone is not started.""" + +from c2.transform.msoffice import transform +from pathlib import Path +from Products.PortalTransforms.data import datastream +from Products.PortalTransforms.interfaces import ITransform + +import logging +import pytest +import subprocess + + +JAR = str(Path(transform.__file__).with_name("msdoc2txt.jar")) +DOCUMENT = b"PK\x03\x04 secret document body" + + +class FakeRun: + """Stands in for subprocess.run and records the calls.""" + + def __init__(self, returncode=0, stdout=b"", stderr=b"", error=None): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + self.error = error + self.calls = [] + + def __call__(self, args, **kwargs): + self.calls.append((args, kwargs)) + if self.error is not None: + raise self.error + return subprocess.CompletedProcess( + args, self.returncode, stdout=self.stdout, stderr=self.stderr + ) + + +@pytest.fixture(autouse=True) +def clean_environment(monkeypatch): + monkeypatch.delenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", raising=False) + monkeypatch.delenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", raising=False) + + +def fake_run(monkeypatch, **kwargs) -> FakeRun: + run = FakeRun(**kwargs) + monkeypatch.setattr(transform.subprocess, "run", run) + return run + + +def convert(orig) -> str: + data = datastream("test") + result = transform.msoffice_to_text().convert(orig, data) + assert result is data + return data.getData() + + +def test_implements_itransform(): + assert ITransform.providedBy(transform.msoffice_to_text()) + + +def test_name_and_mimetypes(): + t = transform.register() + assert t.name() == "msoffice_to_text" + assert t.output == "text/plain" + assert t.output_encoding == "utf-8" + assert t.inputs == ( + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.ms-excel", + "application/msexcel", + "application/x-msexcel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-powerpoint", + "application/powerpoint", + "application/mspowerpoint", + "application/x-mspowerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + ) + + +def test_success(monkeypatch, caplog): + run = fake_run(monkeypatch, stdout="Hello 日本語\n".encode()) + with caplog.at_level(logging.DEBUG, logger="c2.transform.msoffice"): + assert convert(DOCUMENT) == "Hello 日本語\n" + assert caplog.records == [] + args, kwargs = run.calls[0] + assert args == [ + "java", + "-Xmx512m", + "-Dfile.encoding=UTF-8", + "-Dstdout.encoding=UTF-8", + "-jar", + JAR, + ] + assert kwargs == {"input": DOCUMENT, "capture_output": True, "timeout": 60} + + +def test_jar_is_in_the_package(): + assert Path(JAR).is_file() + + +def test_stderr_with_exit_code_zero_is_success(monkeypatch): + fake_run(monkeypatch, stdout=b"text", stderr=b"WARNING: POI warning") + assert convert(DOCUMENT) == "text" + + +def test_empty_stdout_with_exit_code_zero_is_success(monkeypatch, caplog): + """A document without text gives empty output; it is not an error.""" + fake_run(monkeypatch, stdout=b"") + with caplog.at_level(logging.DEBUG, logger="c2.transform.msoffice"): + assert convert(DOCUMENT) == "" + assert [r.levelname for r in caplog.records] == ["DEBUG"] + + +def test_stdout_that_is_not_utf8_does_not_raise(monkeypatch): + fake_run(monkeypatch, stdout=b"ok \xff\xfe end") + assert convert(DOCUMENT) == "ok �� end" + + +def test_non_zero_exit_code(monkeypatch, caplog): + stderr = b"java.lang.IllegalArgumentException: UNKNOWN is not supported" + ( + b"x" * 2000 + ) + fake_run(monkeypatch, returncode=1, stdout=b"partial", stderr=stderr) + assert convert(DOCUMENT) == "" + assert [r.levelname for r in caplog.records] == ["ERROR"] + message = caplog.records[0].getMessage() + assert message == ( + "msdoc2txt failed with exit code 1: " + "java.lang.IllegalArgumentException: UNKNOWN is not supported" + "x" * 440 + ) + assert "secret document body" not in caplog.text + + +def test_timeout(monkeypatch, caplog): + fake_run( + monkeypatch, + error=subprocess.TimeoutExpired(["java"], 60, output=b"partial"), + ) + assert convert(DOCUMENT) == "" + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt timed out after 60 seconds" + ] + + +def test_java_not_found(monkeypatch, caplog): + fake_run(monkeypatch, error=FileNotFoundError(2, "No such file", "java")) + assert convert(DOCUMENT) == "" + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt could not run: java was not found" + ] + + +def test_other_exception(monkeypatch, caplog): + fake_run(monkeypatch, error=OSError("Too many open files")) + assert convert(DOCUMENT) == "" + assert [r.getMessage() for r in caplog.records] == ["msdoc2txt could not run"] + assert caplog.records[0].exc_info is not None + + +def test_str_input_is_not_sent_to_java(monkeypatch, caplog): + run = fake_run(monkeypatch, stdout=b"never") + assert convert("not bytes") == "" + assert run.calls == [] + assert [r.getMessage() for r in caplog.records] == [ + "msoffice_to_text expects bytes, got str; no text is extracted" + ] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, 60), + ("", 60), + ("120", 120), + (" 5 ", 5), + ], +) +def test_timeout_from_environment(monkeypatch, value, expected): + if value is not None: + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", value) + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][1]["timeout"] == expected + + +@pytest.mark.parametrize("value", ["abc", "0", "-3", "1.5"]) +def test_invalid_timeout_falls_back_to_default(monkeypatch, caplog, value): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", value) + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][1]["timeout"] == 60 + assert [r.getMessage() for r in caplog.records] == [ + f"Ignoring C2_TRANSFORM_MSOFFICE_TIMEOUT={value!r}: " + "expected a positive integer; using 60" + ] + + +def test_java_opts_from_environment(monkeypatch): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", "-Xmx1g -XX:+UseSerialGC") + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][0] == [ + "java", + "-Xmx1g", + "-XX:+UseSerialGC", + "-Dfile.encoding=UTF-8", + "-Dstdout.encoding=UTF-8", + "-jar", + JAR, + ] + + +def test_blank_java_opts_use_default(monkeypatch): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", " ") + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][0][1] == "-Xmx512m" diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..05d74b4 --- /dev/null +++ b/tox.ini @@ -0,0 +1,82 @@ +[tox] +envlist = + ruff-check, + py{310,311,312,313}-Plone60, + py{310,311,312,313}-Plone61, + py{310,311,312,313,314}-Plone62, + +skip_missing_interpreters = True + + +[testenv] +skip_install = true + +commands_pre = + {envpython} -m pip install -c {toxinidir}/{env:constraints_file} -e "{toxinidir}[test]" + +commands = + coverage run {envbindir}/pytest -v -rs {posargs:{toxinidir}/tests} + +setenv = + COVERAGE_FILE=.coverage.{envname} + Plone60: constraints_file=constraints_plone60.txt + Plone61: constraints_file=constraints_plone61.txt + Plone62: constraints_file=constraints_plone62.txt + +# Set C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1 to turn the "java not found" skip +# into a failure (CI does this). JAVA_HOME is passed for setup-java. +passenv = + C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA + JAVA_HOME + +deps = + Plone60: -rrequirements_plone60.txt + Plone61: -rrequirements_plone61.txt + Plone62: -rrequirements_plone62.txt + coverage[toml] + + +[testenv:ruff-check] +basepython = python3.12 +skip_install = True +commands_pre = +deps = + -cconstraints_plone62.txt + ruff + +commands = + ruff check --no-fix src tests + ruff format --check --diff src tests + + +[testenv:ruff-fix] +basepython = python3.12 +skip_install = True +commands_pre = +deps = + -cconstraints_plone62.txt + ruff + +commands = + ruff check --fix src tests + ruff format src tests + + +[testenv:coverage-report] +skip_install = true +basepython = python3.12 +commands_pre = + +deps = + coverage[toml] + -cconstraints_plone62.txt + +setenv = + COVERAGE_FILE=.coverage + +commands = + coverage erase + coverage combine + coverage html + coverage xml + coverage report From 53b9630128ec53ff29a7df9484beda911e08d06c Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 18:59:08 +0900 Subject: [PATCH 03/13] Install with a post_handler and add an uninstall profile - Register the default profile with a GenericSetup post_handler instead of the old import step. It registers the Office MIME types that the registry does not know yet, and adds the transform unless portal_transforms already has an object with its name (then it logs and skips, as before). Running it again changes nothing. - The profile version is 2000. There is no upgrade step from 1.0b6; 2.0 is meant to be installed again. - Add an uninstall profile whose post_handler removes the transform if it is there. If unregisterTransform raises KeyError (a MIME type that was never mapped), log a warning and go on. MIME types are kept. - Hide the uninstall profile with INonInstallable. - The test layer now applies the default profile. RED: tests/test_setup.py failed 7 of 11 with the old handler: profile version '1' != '2000', no INonInstallable utility, "no uninstall profile found", and the second install was not logged. Mutations of setuphandlers.py, each caught: no objectIds guard on uninstall, catching the wrong exception, no objectIds guard on install, never adding a missing MIME type. Co-Authored-By: Claude Opus 5.5 --- src/c2/transform/msoffice/configure.zcml | 33 ++++-- .../profiles/default/import_steps.xml | 9 -- .../msoffice/profiles/default/metadata.xml | 4 +- .../msoffice/profiles/uninstall/metadata.xml | 4 + src/c2/transform/msoffice/setuphandlers.py | 74 +++++++----- src/c2/transform/msoffice/testing.py | 4 + tests/test_setup.py | 106 ++++++++++++++++++ 7 files changed, 184 insertions(+), 50 deletions(-) delete mode 100644 src/c2/transform/msoffice/profiles/default/import_steps.xml create mode 100644 src/c2/transform/msoffice/profiles/uninstall/metadata.xml create mode 100644 tests/test_setup.py diff --git a/src/c2/transform/msoffice/configure.zcml b/src/c2/transform/msoffice/configure.zcml index b0f7b09..acdc045 100644 --- a/src/c2/transform/msoffice/configure.zcml +++ b/src/c2/transform/msoffice/configure.zcml @@ -1,14 +1,31 @@ + xmlns:genericsetup="http://namespaces.zope.org/genericsetup" + i18n_domain="c2.transform.msoffice" + > + name="default" + title="c2.transform.msoffice: Install" + description="Register the MS Office to text transform." + provides="Products.GenericSetup.interfaces.EXTENSION" + directory="profiles/default" + post_handler=".setuphandlers.post_install" + /> + + + + + diff --git a/src/c2/transform/msoffice/profiles/default/import_steps.xml b/src/c2/transform/msoffice/profiles/default/import_steps.xml deleted file mode 100644 index 06ba2ea..0000000 --- a/src/c2/transform/msoffice/profiles/default/import_steps.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - C2TransformMsOffice setup for MimeTypesRegistry - - \ No newline at end of file diff --git a/src/c2/transform/msoffice/profiles/default/metadata.xml b/src/c2/transform/msoffice/profiles/default/metadata.xml index d5d3247..106aca5 100644 --- a/src/c2/transform/msoffice/profiles/default/metadata.xml +++ b/src/c2/transform/msoffice/profiles/default/metadata.xml @@ -1,4 +1,4 @@ - + - 1.0b6 + 2000 diff --git a/src/c2/transform/msoffice/profiles/uninstall/metadata.xml b/src/c2/transform/msoffice/profiles/uninstall/metadata.xml new file mode 100644 index 0000000..106aca5 --- /dev/null +++ b/src/c2/transform/msoffice/profiles/uninstall/metadata.xml @@ -0,0 +1,4 @@ + + + 2000 + diff --git a/src/c2/transform/msoffice/setuphandlers.py b/src/c2/transform/msoffice/setuphandlers.py index 0601359..9f7ed5a 100644 --- a/src/c2/transform/msoffice/setuphandlers.py +++ b/src/c2/transform/msoffice/setuphandlers.py @@ -1,23 +1,31 @@ -from Products.CMFCore.utils import getToolByName from c2.transform.msoffice import config from c2.transform.msoffice import logger +from plone.base.interfaces import INonInstallable +from Products.CMFCore.utils import getToolByName +from zope.interface import implementer + + +TRANSFORM_MODULE = "c2.transform.msoffice.transform" + +@implementer(INonInstallable) +class HiddenProfiles: + def getNonInstallableProfiles(self): + """Hide the uninstall profile from site creation and the add-ons list.""" + return ["c2.transform.msoffice:uninstall"] -def setupMSOfficeFile(context): - """Add our transform in PT for MS Office""" - site = context.getSite() +def post_install(context): + """Register the Office MIME types and the transform. - # Adding our file types to MTR - mtr = getToolByName(site, "mimetypes_registry") + ``context`` is the setup tool. Running it again changes nothing. + """ + mtr = getToolByName(context, "mimetypes_registry") for mt_dict in config.office_mimetypes: main_mt = mt_dict["mimetypes"][0] mt_name = mt_dict["name"] - if bool(mtr.lookup(main_mt)): - # Already installed - logger.info( - "%s (%s) Mime type already installed, skipped", main_mt, mt_name - ) + if mtr.lookup(main_mt): + logger.info("%s (%s) MIME type already registered", main_mt, mt_name) continue mtr.manage_addMimeType( mt_name, @@ -27,26 +35,30 @@ def setupMSOfficeFile(context): binary=True, globs=mt_dict["globs"], ) - logger.info("%s (%s) Mime type installed", main_mt, mt_name) - - # Registering our transform in PT - transforms_tool = getToolByName(site, "portal_transforms") - if config.TRANSFORM_NAME not in transforms_tool.objectIds(): - # Not already installed - transforms_tool.manage_addTransform( - config.TRANSFORM_NAME, "c2.transform.msoffice.transform" - ) - return - + logger.info("%s (%s) MIME type registered", main_mt, mt_name) -def removeMSOfficeFile(context): - """Removing various resources from plone site""" + transforms = getToolByName(context, "portal_transforms") + if config.TRANSFORM_NAME in transforms.objectIds(): + logger.info( + "%s is already registered in portal_transforms; skipped", + config.TRANSFORM_NAME, + ) + return + transforms.manage_addTransform(config.TRANSFORM_NAME, TRANSFORM_MODULE) - # At the moment, there's no uninstall support in GenericSetup. So - # this is run by the old style quickinstaller uninstall handler. - site = context - # Removing our transform from PT - transforms_tool = getToolByName(site, "portal_transforms") - transforms_tool.unregisterTransform(config.TRANSFORM_NAME) - return +def uninstall(context): + """Remove the transform. The MIME types stay; others may use them.""" + transforms = getToolByName(context, "portal_transforms") + if config.TRANSFORM_NAME not in transforms.objectIds(): + return + try: + transforms.unregisterTransform(config.TRANSFORM_NAME) + except KeyError as error: + # unregisterTransform unmaps first and raises KeyError when a MIME + # type was never mapped to this transform. + logger.warning( + "Could not unregister %s from portal_transforms: %r", + config.TRANSFORM_NAME, + error, + ) diff --git a/src/c2/transform/msoffice/testing.py b/src/c2/transform/msoffice/testing.py index abc4114..4f2d037 100644 --- a/src/c2/transform/msoffice/testing.py +++ b/src/c2/transform/msoffice/testing.py @@ -1,4 +1,5 @@ from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE +from plone.app.testing import applyProfile from plone.app.testing import FunctionalTesting from plone.app.testing import IntegrationTesting from plone.app.testing import PloneSandboxLayer @@ -13,6 +14,9 @@ def setUpZope(self, app, configurationContext): # z3c.autoinclude is disabled in the Plone fixture base layer. self.loadZCML(package=c2.transform.msoffice) + def setUpPloneSite(self, portal): + applyProfile(portal, "c2.transform.msoffice:default") + FIXTURE = Layer() diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 0000000..6860db2 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,106 @@ +from Acquisition import aq_base +from c2.transform.msoffice import PACKAGE_NAME +from plone.base.interfaces import INonInstallable +from zope.component import getUtility + +import logging +import pytest + + +TRANSFORM_NAME = "msoffice_to_text" +DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +DEFAULT_PROFILE = "profile-c2.transform.msoffice:default" +UNINSTALL_PROFILE = "profile-c2.transform.msoffice:uninstall" + + +@pytest.fixture +def transforms(portal): + return portal.portal_transforms + + +@pytest.fixture +def setup_tool(portal): + return portal.portal_setup + + +def test_addon_installed(installer): + assert installer.is_product_installed(PACKAGE_NAME) is True + + +def test_latest_version(profile_last_version): + assert profile_last_version(f"{PACKAGE_NAME}:default") == "2000" + + +def test_transform_installed(transforms): + assert TRANSFORM_NAME in transforms.objectIds() + transform = transforms[TRANSFORM_NAME] + assert transform.module == "c2.transform.msoffice.transform" + assert "application/vnd.ms-powerpoint" in transform.inputs + + +def test_install_restores_a_missing_mimetype(portal, setup_tool): + mtr = portal.mimetypes_registry + mtr.manage_delObjects([DOCX]) + assert mtr.lookup(DOCX) == () + setup_tool.runAllImportStepsFromProfile(DEFAULT_PROFILE) + [mimetype] = mtr.lookup(DOCX) + assert mimetype.name() == "Office Word 2007 XML document" + assert mimetype.extensions == ("docx",) + assert mimetype.globs == ("*.docx",) + assert mimetype.binary + + +def test_install_twice(transforms, setup_tool, caplog): + transform = transforms[TRANSFORM_NAME] + with caplog.at_level(logging.INFO, logger=PACKAGE_NAME): + setup_tool.runAllImportStepsFromProfile(DEFAULT_PROFILE) + assert transforms.objectIds().count(TRANSFORM_NAME) == 1 + assert aq_base(transforms[TRANSFORM_NAME]) is aq_base(transform) + assert ( + PACKAGE_NAME, + logging.INFO, + "msoffice_to_text is already registered in portal_transforms; skipped", + ) in caplog.record_tuples + + +def test_uninstall_profile_is_hidden(): + hidden = getUtility(INonInstallable, name=PACKAGE_NAME) + assert hidden.getNonInstallableProfiles() == ["c2.transform.msoffice:uninstall"] + + +class TestUninstall: + @pytest.fixture(autouse=True) + def uninstalled(self, installer): + installer.uninstall_product(PACKAGE_NAME) + + def test_addon_uninstalled(self, installer): + assert installer.is_product_installed(PACKAGE_NAME) is False + + def test_transform_removed(self, transforms): + assert TRANSFORM_NAME not in transforms.objectIds() + + def test_mimetypes_are_kept(self, portal): + assert len(portal.mimetypes_registry.lookup(DOCX)) == 1 + + def test_uninstall_twice(self, transforms, setup_tool): + setup_tool.runAllImportStepsFromProfile(UNINSTALL_PROFILE) + assert TRANSFORM_NAME not in transforms.objectIds() + + def test_unmapping_error_is_logged_and_uninstall_continues( + self, installer, transforms, setup_tool, monkeypatch, caplog + ): + installer.install_product(PACKAGE_NAME) + + def broken_unregister(self, name): + raise KeyError("text/plain") + + monkeypatch.setattr( + type(aq_base(transforms)), "unregisterTransform", broken_unregister + ) + with caplog.at_level(logging.WARNING, logger=PACKAGE_NAME): + installer.uninstall_product(PACKAGE_NAME) + assert installer.is_product_installed(PACKAGE_NAME) is False + assert [r.getMessage() for r in caplog.records] == [ + "Could not unregister msoffice_to_text from portal_transforms: " + "KeyError('text/plain')" + ] From 9f4ec31ddd8f8ff3b12c30fc2fc98a533293a30e Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:03:43 +0900 Subject: [PATCH 04/13] Test the transform with a real java and self-made samples Replace wordprocessing1.docx (taken from another project) with samples made for this package: document.docx/.doc with a short English sentence containing the words the old test looked for and a Japanese sentence, spreadsheet.xlsx/.xls, presentation.pptx/.ppt and an empty.docx. tests/test_java.py runs the bundled jar with LC_ALL=C, as in a slim container image: - docx, doc, xls, xlsx and pptx text, including Japanese, ends up in the File's SearchableText. - A document without text gives empty text and no log. - Without java on PATH, creating the File still works and an error is logged. - The tests are skipped without java, and fail instead when C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1. Known issue: the bundled jar (POI 3.17) cannot read any .ppt file, as in 1.0b6. One test pins the current behaviour (empty text, an error log, the File is saved) and a strict xfail test expects the text, so it starts failing once the jar is fixed. Checks: - Without java: 10 skipped; with C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1: 9 errors. - Removing the -Dfile.encoding/-Dstdout.encoding options from the java command makes 6 tests fail (Japanese text becomes "?"). Co-Authored-By: Claude Opus 5.5 --- .../msoffice/tests/wordprocessing1.docx | Bin 7823 -> 0 bytes tests/samples/document.doc | Bin 0 -> 9728 bytes tests/samples/document.docx | Bin 0 -> 4949 bytes tests/samples/empty.docx | Bin 0 -> 4719 bytes tests/samples/presentation.ppt | Bin 0 -> 10752 bytes tests/samples/presentation.pptx | Bin 0 -> 6713 bytes tests/samples/spreadsheet.xls | Bin 0 -> 5632 bytes tests/samples/spreadsheet.xlsx | Bin 0 -> 5463 bytes tests/test_java.py | 158 ++++++++++++++++++ 9 files changed, 158 insertions(+) delete mode 100644 src/c2/transform/msoffice/tests/wordprocessing1.docx create mode 100644 tests/samples/document.doc create mode 100644 tests/samples/document.docx create mode 100644 tests/samples/empty.docx create mode 100644 tests/samples/presentation.ppt create mode 100644 tests/samples/presentation.pptx create mode 100644 tests/samples/spreadsheet.xls create mode 100644 tests/samples/spreadsheet.xlsx create mode 100644 tests/test_java.py diff --git a/src/c2/transform/msoffice/tests/wordprocessing1.docx b/src/c2/transform/msoffice/tests/wordprocessing1.docx deleted file mode 100644 index 82c5a478a8affbeff08e79dc7add8d9f12a34a5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7823 zcmd5>by(B=*B&uKLSjgYC|x6@Q@Xo5N5g0yLb?Q`L!?1c5tNYb5Re>52`C*>()~`K z=l3XkfA9bAu50^l-(C0F*~$BykE#L+DhL1oU;si&2zf<^&V+W60043v0019;*Ai-> z>I!vsW3zy|Lco7HI&7y6#%8eN^_l=(=+WMrNQI^}yq+6)t65`XaI{(z(PdPbARM|p z?XymOL}Z6eW1A$QS-J^)7P8MC;OkF09GSBa@ajWG+2QBh=c=yXWU59^o^W=D8GrBdIGVBr3C)ctu@P^-U|kcXHoA<)c;=2NK{sv%;SaFu?rs*?G)@i$GL?Ut%O> zt^zHpt@$~=+%BKK$nfsYK}2(9hH&W?%hJ6eOcSCoQ|Ag@nAC_yAWradn+ia8X(7tB zyMd6Ctscj**xF(av({12eoeW}M{;@K07m~on(CDp6X|I>w;}pBB|M{5waiZyiP5L_ z^4C;T54U^eZypm^=X3;EA2ali8As8_15>6($-eQaQeeaKrG0zpboE-0<4XIDtm9hB z(1fcs{B-+A1*ulchRGca3hUGNASh8E$$%hQeoV!Zk>qkl~EPS#<-)0Uqc zE4GX1#Van*C{WxkbwB0?DpcXu)Ld1ir4eU5g6=p-(+6WA%}da>)lg29V(IDaBeni8 zFL0=4L6Xplne-}ZGz%G3&x~4J-by01<6K(u{E&7ke}uqV8%)N9kNf^RhBMJ9gQmqK z7M*=JEbY`p4FVxI`K!*?W5H+aN@^JMj78oMi<=~jKrOJE(kryYl5OK&%MNNX>YH~s zbWgHZ_qJ!(W%s`~z2guUFQxTcXOlWqn({8LI{fiE*SBKDH646cP}D~nS7lP?V5i%{ zQ5+;~-r*tH{>YOqggU?GVA zu8Spht+Qi7GefiU#FLcy5xxuswWVZz@7=HB<4B~3Pln!Bd*hz$3=K(k<(-P=W-2ku zOAlbj323{$JV7ejUXE-Bv!{e8tEoY;wPSJOs}yGVTAW}K@6rh-R5>n@y%fHgUBRYr zp1jigTpr9uRgAQuBY1;)+nE9vqQq8kXJVlO0EB-!)562e9qRbgpEXGXKm92cK>I_) zgOYWGNlEI#n_{ya&!$n?ErE6>5>)QS@l_VysMO$h4vmf8@doP&JpZSpc$kY5i5}WefxM?9d+4` zile-oag&nese1aUdapoq-%0ze%&M)4o`8Ux`npY->cczY1Vchw-^bs{xB10lZvCaL z1ElBP+We8PlVkKe1Q{t*8QMC@E{J;r^YHnD5>fbjG93 zM|O>|ZcZ|h>;po0ZZ~MiH2+ned&pX#`ooN@P^Nkkg3C2wlT>Sm3{p- z^oALh)y$C+%N}BqU3?Gi$!Ix_DzUClN!ZdK;%79^`a~L;w7PWoc;Wntu!occnqkVh zF?uq~h~k^7BV?a05obEz_*oHHJpED#gY&OBzI*ze(mMOIVid^ad0P2cwxBVc9P{PQ z!OnGjmQI3VT{@i)Yn<_xb;*>wmDa+HUF-x?@tPqdLd??09;i{BG$Ghh@vJ3uKjro{H0E=T+#xh1t0Y zeJlun#)cC@_{EV1DFBiYrBzAy>MeSrvvd%0-hvY^ei_a=cbH;vXQ>a?$Le&U;8=0D zk9{$xgwIImol0le0Q({JgY#*}P7vnJVV;lSRFTTny$Q{4_~%7ab}$blL>3;{N1t0b zdIx;UIA)V96XJi+!z%yR&>PBzZ=k7Wo{2-U?kEzc-2s6+!iz)6FD_I7=@p4by;hS)S zqr;#_rcADhH$uTX<`Y+*wdrDyyx4TPl_;b_m@CBWCfJ=pubAxPXC{qBfYl9XBSyQM zNNfEpXLZ+a=LLp~L?b`4wv*`2%)40DM^YBJ?cgt5?Zd&g-4I|$5^-G^48_o55F*e+Eu;xyCXe6C^q z29q?+2yL^w*Z+?Dp%qHI{TrtU?wB|z7udd~Cii?%dKOY^wau)gRrCYtyHeN{zYkqC zBkC1u4;YnT{&npBfTK-&FQ{|89R%^CHOAs*dyM~80J*E z2Lq^fPR&`6A+jv!^^LEcD7nYBu~!m(CJ){BhN$a8mT`<^!Hk%&Jp#K;M`bJlwl}S# zqW*3Q#V8MUt;ch=v^;p;C`m=iZ>i3f7!&tR%(C9+F0Kg6{PW;}i_!gW7mQvuEp`OF z79_#A3s#NBv|B!!j!>0t1uLuu zLt7kxbUWT2UkUhsf%j@II4vjmPg8L6a5RUw+B#YP47lwy1$e;0sn;VLD9m$=25%7+ z7@WVI|1qC9m$Cm*2XXkT=E5?omCO&ylyz8Lfh+Xsq?A9dM+--NQPOEr&Qyx{jQ@}qy@Lmc=_`cGK&-> zDdComglaT-tZsQe(P|?QMJ-XGD+7*tKX$RIEw@706C# zRlIKe~-Z7 zyS>Y&kob;PkC3~Z@1WEwgVn?ui_=i>Y1h8k<%QExpS} zUkl>AVJ&g^aa`C1n=1S$Z7!QTy_PYwBm|6W3GaWAoZ$XMv~<+Kl@64`xGJuNgYW1! z#~MVlzkBJyBIw=vzMbBCC22uKf{SuZ(NhzLsn^=xkb=`HNh2HUGcxieNz>V-mHpc} z)wPFvG+_5<^Hb$XHqkFPElx;dgG)t)BwWAd20WSpN3P;+-;;EUeEaK#82(O#1%7Yt z*UQgu=U5|$@DuvNINyD2Sb%N2x~^otr#9L&y$BG!+lmkiJW=0YV8COc4_<%n|0fnq zTpO1CEZOiuhRU_y4PT~l-ZKFy<$}Ixu$_eCplh@j7aZEh;b>WqUO8hop|}rh0azLktG0Jqvke zs&q+8B_!1}?z-o?k1VMn`m3=!#xm%GeiaLg)6_(%s!VI*ej_nORMmQ@UKiveKdcDm zGp+6mJUAsxlNBwqi$=j{sP!EA@GysX&gSNL`<6Mp~`HY|n^yHN||87(80ZxXgA z`0_wE^V!~l3SJa<^yxmAr#n+_9h!hh4r3JP_1g1Y}rv!1FND5G#-H}&`5 zaLq=gX)E~xQRy-CpybKV3%X{*s`lFx>X7GUUf+=qpctH@9A3USuy1|}ed^Mdd>@E2 zAcYA;Q>eib#JSJQ8&^5^=A!K3;**&A_^|lb#+}*x+VYUb-p|WGDeN(OhMI6-QPw8o z1V=)5rc}j|omT2e88^7C-fC3m5?@WC$0pHCo+|k@ck+@U#efD`HpO!y)(u}`ONfU- z2X?j~Tj<6Mv>vV)xz4#xfU%mxx6pB_1FJA$l8SbVPU=!+-XaMZHa9jULJMKCR+0{N zQ(}S69|df!D6AEzFYo94xFp}Hn2zSHAQMILK0vu`r{xXoqbRtYdf-olP`hQVU!nGV zNCCHl9UFDzwd6bu^&W=S)t4Z0$L!nzgS^S!lDIqKc(zp9By~~O=_|0&l1HeyrC}ANzDdu2yG_F! z&BN=cmYpo1aL#QibH4q+YM>g_kCr}b!13L9TqGzSMtu8ERDh@E=HP!|0iH8IR~o;1 zHN?sCUtia#!l%|J*vT68Og+Db4ug!TcD;*pMwySZg{)10?92L82_z_~d@fb`n^^TV$C{CT zg6#F=y3i8!s0iagWl(uUXhTbVE6*@2H-ufTwx{NK<4%rL%Z?$tz~j`v_T2}JKQUky zQ=ONlEoe{PSRAbmwmJk%e(^`XZHqr0O7S0C{9QZ#w8;N!f378TB_zOg{{(;j>rV@) z<4@}KYeafJG=S5=j)yv;{ic}^9T!9D%+RLG_4BjBDf?}K7xR3o1gxg0Kxc}& zYtgyRBBzgwjJVDY`l=+XgCby*@6-3e?9ZQWu!IG2M3-GK7?GL|@_Xn<8*z}>GMK%| zqQ<40{1^@Hax;I=pk80Op)s57!7kDf_bq_Lb;zJ8aZdDZ3&xU_YctDh6=eB1=xrbP z1gud_z;(ZKyZAV{Ynqum{MxY<#<#=AZtQRq+WxGl^?0>qF8-3-h+@ z3%zm3a;?pVYqoyj6d!pIFPGjMhgYD7lRQ~kd0dLmVjypgIZ7O1LM0fKD~t5<(Rk&l zDm*xWd%&2GuH&RZylRI{v9jIq4~r2$;)OzqjEo}IJGVGj7t^7ZKc?$|pH#E5LyVLJ zv%5wMTn?ip>nzXRzEbl=*$14N%#?q7UtN=nD+p>034p8zs^W39&#{9MREKiMAJ7K{$`wY~Y zQ?WaGD3R8Oo26sfv#~;@K**Ko3gKDea&~daP^K@iTTkG(6M5#n6fwa!$2psaXY8aF z@*rM{usJvrEmr0OK2iM_fyDKvO=Su$DUeC=5Y7$%Wo5IJuDi)g;{p&qr7q*)cY&%u z$mVqvYvsOJdez(wvOTu_U=v8~L1;=`sVvJ$2{Endv~Ye~y-4?h+jl*sjx;pR^l1`9 z?&08@;7A+}@Rw`LQXIXhuP7WzZ>Kj)R=G}Yvm+{TgIN}1Xm{X?|UEIXE>#N zf?d*7u-n%gls_CzCyJGmKjA82p%z4zt! z{@@sH{u8u_G@k=LgVs;PUrm?VoOUg6e7t7vL^?&RHwc~~Z1tdhKo0=i7tHWQ5JFM- zbENrI6v~tSaf|t2HXmytmF5R>p1FMEHV-;cyGv?c_0g(#Uu6xEsBK>6_nUuPX?nNd zcfbn3WWy*(ex+-v zg!Ih(JB-3Vg{D(to`!`~0oF@WNYl+p4}7_Kw@XisQ)tyvRXgAV9kw;Z@~e)C{}qDe zI*RONOW~G#hw=9c%|HU`^$>PYm`Rw~){`=-eUi3pyK5&B9 zqe~Dl2n8068rA6!Er_t52I|+Cgpy184!*3m$@jL1GY$w_R2}(uxwTF3dX{6t zr*T=LM&lU2L>c!|D)jzE-jUnO{rsA7dNL29TV2Qs;mV z7IGs1*Htvuxz3V<@ik>@!ez#bJ)r zhE*2+SK(&PlwKt%m(mKKkah9$_JqTmB(Bfa8u8Z?q+1)df+PN5ES)GzDC?8{SpvMr=yKqSq} zMGD*@3FVGl9=>jNvaM^7djS0%7x$fEsu(6#x{uE8PMnpS=$2;)46$QAPH!7tq$l9@ z2@8JxyGQ<=0`STNr-srM1l|k#)l{LXFsCW^2t49Qf?-9;1L|Is6$chLVW29*Q`HOy zH^dyAtxWe{Y|XkhV;LMt8u~?%ttw$fLG$?su zgH}Egwi2W`)X|UbWlrT?u<#OL-HF>`QJy^W)x@UhA?2_kgt547TkUJMJTLoX#G3Zh zd~AI&6DvPU)qS}rOK~VkK5b3&y+s1(T5n9`U?mhi$L$C{dLY3q@PD5V!AF&4y diff --git a/tests/samples/document.doc b/tests/samples/document.doc new file mode 100644 index 0000000000000000000000000000000000000000..0c34e868278fd580d57dacec1954b45563f7e48e GIT binary patch literal 9728 zcmeI2TWnNC7=Zt?-7dS8mhHAw1e8Nr0lBmX5G;b)mI7t5w54c8gmz0?T5Wf+EsBXm zqcL8<1mlA*Mx((eFF}3sMR}3<;^l=0o5W>Q#MNie?Xq0M>BV2 zSWHTBqvOD2C8UlQljX}aEhm3W1DWV7RsY`joz3^JbAf*U5h#IDmT2K^<&{EwB~pVH?C?J3I&tumc*Q37TOiw7@QS2p)!3cm#ICqp$}a zgT1g1w7u=FKh}HKKV(poJO+4`90=)*E^^9#$`MC zf%By)HbWc%^ReIBu)eQ$9KtG-(#dFSVpm;qn6X(`Utx)KmKp{n^1YonMw}^?VZ(?R zCSgA|+AD*0ULW74pR$A4c{$~l8`b2Pfu+agAUS#|KS4|(-){+Z)B4S%ei2&{>w<>9 zs_otzG)%i?tMRqF%8$%$)0IMro!^mBXXmIj^cxDwP*+D=F!7vq+WOe~!ukd+6finC zb=GmAF&07tAtaTefdXn7LR2a5inIfI)Ml~B%(pt*N#7u>jyLhsim8kQH(*+im?g=eQie*ZMP{qC3OuR;@-Fmrf5(={mrV>2}*y)N(?5qtn;jQkh+g^>m(kJ$MEgz~8e4TmXYd(&i%Zhg* zbh_6mkD-mHUtT<}{`gq$+gg%GlDf%9S|?BLnN|Z4D{WHN30@zK6_8OUO}i$~j|rGM zG|Y~}F|6@<)%4E2^zNS*8hgSYz7dq=OFloVHF*|v_Rw@)AL{pNW^=(=qt zbb{`84}v!S*GrDsYFNvv2G3P{0sM5~lq{pc-###m1~Y^v&+q)oiFFj_&U8hxDW2?$ zC#sy-(f2e^>@Sl_d!BTn+Z`|Mb7W+eJ&S7FHSFm@!xP;rcEoqi7c8DXF<0tu&Ew4A zsj=z9Ka*dz{JZOWIp?9?4sRInBD3%>x^4YC!Eo?eKa# zpPaqkB6IR`8pvtjKc#_P(!+K~w?WP`PH2#DKdX*DHby2KW>YlwJIUtGB*tWYl9c@{ z_I2Z`XSXKX)?WM`H@%y*KK``CE<4?MSMv{1LMN!Nr6u@fQkrggz48V*X)BZ#>zI|a ze(73q;GNeJ*2`W0bSJE%jR&mVme11FQO;eJ23Y6m=|;}|FEwE3QPumPr&@ofs$krY z)P!94#?(MTspLf^9vwXft%bcH=&y8W+iV0>r3J@aEcPw zxtl}-*Yf^XZD`^h(v{StoBL`tE}UT>SnBGgk>l*goyciCBUr22D+egfZ9#_bdXfy} z>N+c1SAcZAJJU9)pGi4dU+<)3702{(TApnkid^|BTU{pdwXQ$vKX~BsxiZidq+WFB z-TyXLg59hFof|XnV!_FKrRP%Q>&=XwC1omq5qUpy3v0Dyb#e(Xel6=JPwCCgqWz00 i)jK!KzfAJ<^ih?C{5It4ZQiwVcE`JqOv+5X|Nj>XMh-pz literal 0 HcmV?d00001 diff --git a/tests/samples/document.docx b/tests/samples/document.docx new file mode 100644 index 0000000000000000000000000000000000000000..3581e4ed9b2d7e2de77d6ad9cd810a805c830d95 GIT binary patch literal 4949 zcmaJ_c{o)4-yVDRbworM3?@s2Y}vCVTZ9m1Y+Gut`$F~ z)o2=wqVqbN5ZfEvgC4!#^9-|MV$|A7UDdoK2O1>Z{3nzpcyA3!G+3yhhV?RTik41? z<=eC9CPcOytd__~c@;n5bGl~IspP<4U6klGRk!yz=t5qu)X1n?mvtj)?UBy0oa}w} zDo~h^lWpP6=hgD!Ktlv{;MP+_iPwRfOe&qfjSC?FEl=0GVJdT_PpuNMX(@lBu09o9 zV6A=chpP-cu&1fheZY1;=0kql7}rj&(H{(ybc_jdlK=o*L_ zZ!4uz}2mcWXiC19ddR5j^E zo}Nn=>@@qb+-zjNm& zli8>@o~@F^j0%VrogWv{JgWhwAD6+0)tptRu}&2fhq@&fdl@5+DEfo91FM3pbMbcN zMX!4pBsEuTw7yUXaLVK3{2HSJN8txaQ07$W(rpG&?=YRwEww;8m6l+3hqP1T=FkKV z^Av6DBinZVTASq>H(w>gzFqO=p|9QK zXeY|wWVrv4A)k&K+!b%^SBqDx0@=}Sa;rPnwtkA7#I2J|_lbceXl<2%fpsrDvofE5 z&bLe%o6<~1?vupTd~*MGVZ8U#&HjMNC@l8z`NaeZeJ`HosjI~Vnw zYsC^bV5@o+nw}MK-BI`f6tXNS61M`PSQ7OtQ0S&P0L3p(?p{21)DN9d51wGB#vHqC z@_Lx2#kJNN7cpFX@M1Qn^88^Q*d`n_`|xQV*>NKQs&h2x()}6TelO(ouGm}Oeg+iy z7}gvvye-!1-gS!YZot{exln@~PHmAN@|ac1Khs7|%9r1d$V3k1UlsU^jeqkna{Ytn zkSf#bZML-asD(PScDF%&ig`fk@CM+??zk;(xAJbMS6kFhD~YGhIr8}k6jO+pux$r; zUh45K+?APTCyWiI6f`3@^S1Gll0tH;^##XQ-9oP@%?_iNGpFPsPj3vUS8*$8c6mH2MA~n44aVv^TuCy0TCR6XGd0#M=}x!RLj75% zWoLi_O-P+m|=uA=&Ssmn99DS`Z!`wfO$Dpm{sJfsxb+Oj`Hb zad3BeyOz$thFk6Y#*+6ax8G*zqzr!r4>%! zFqF<83r@3T#K#+lul0kbv@1YgjL}maGT-xQ#pTQ0=Gu(rrgsY)nyQ1^8aKX&$^Brs zRYZl7X?XIb8x`J?9G2{FnDZ#|J00V&db^mjAHmPT&4)(pP16M{7OCCa!jpTU4OyK{ z`UaV(k+6L8SaFWZj926eAQGp;{6{q_ERcsnPTba#xU{L8?^2oHf8ng>K}I^i8kRq+ zC{TNyy5sOR`V$JB%ckz1_OM5f5+tOm)O1bhW@=frBNYR^md5wX5VTcuac{20io_F( zbvgKD!v21xRnT-(U;|R5H!+CgkhR`wfeT}$Z%!)v^b zloI8VO`qd6Ibr)vL70Gk;G_WMQ^OFk@hP8+5k+GZa_94aYloas0aPhG-7EzWRpF#Y zRafiG_GmA`wXcF&J7c|dHIx0W9KHoF`GPcY=Hz+L96gQ1ZE=L&4}o)oZ!nR00Y5_L za$(m1C*6G)1B%BPr49 zuBsF&w;Y})pI@26uSKAy0VP_?oV&f~`f2%^>t#p-a=f$e$4!$3c{cU%`%zfwj4`=& z<-XQIH9hPD2yhOfIb=$*B^l#ya6?hEQL(HR@kxXpLfOs9V9YC4Q?u`awyu-7Qo$;wVZE$4)YQoI<}D|0>U0qopA?VgP8bA~fH5h2phw6!_jud%9NjC~p=yCR z6BgY{|NJ8UoI@pDov>>2!a#+tX_MtM&jvh#pPk!l$agxV?ti8Q9w5-sZ4DN`9(B5@ z9+T-x&r{>#6o;q@VO=t*ST**1clnK!rXCvTTu-W#oVS1=JLF+hm<^K#Uvv3#0n9O2qde~`zQnc%-FRzvKE2a!m`9b*;P>3iWQ?YYmDU-}O7L&ty&=NskW z;H`*u_V1_qGTIGl*JuSI0Ms7T`HoUfCAGLD2j7AWJvx)sweGgD$>$c*cmioRPZZ+0 zJ$Z6gLlSv$Ya2~ZFfEl<*N^aV=AxSl{b9B z>=zg8PskBR{t_7g!17ORIKe%A&FmfAe@UncR1b0RGimJ+XnKz3Tk=c!NlUH#MykL& zWD!tx$XJd`2h;vhCcuQ(vIeLaz_XY0YUp5Eu~vg!AKkxTTnLOv66KN&N7^aROy3q{ zR;P$3Wj_qgU#x!On*Nb3zqp16v`#bi7Wbt?XDpA3(>R82*Uf=Z>H)%0VZIb5Y78m~ za2B$>>#T&!>$`Bw#aVjWdY(1aI)N`oljH1L--j}FKxM_D5^=^^)}8MAY_%04FuAR)A6Dxdi zbk_7ag4w`>qLCOIfj#>&!K#V?i#uM44X3l$5l{H4HfuHZ4Jg$jS`04{coy~4uqJtY zSy#rZv{Zz4sleBOTShvyQ;xe7Uyo5^RDZUhOsCTaWJH>yiT|HM_1Bp6f%*EndOH7V zLNFB8vrUpNY)>T|jZI5NU)EkV5NW8<$W{sFH+=!nWdkY1AML;#$VK`Sn-=C&JnNj- zqgkK1OUXD~H_s2wdr?1!i(Ds6EX05>l=O=|A~_?a!9MZLUV4Nx5|>JAcx1q}o6qvgJP-FI)^wkr9P{yfz+!z3ZM8XN{@gcYi08B zpv$^STDas;{ptn2?aT*E0ihMaJ51Tg@^^c@1$=$ySQV%lVJpB!z##oAh!Xurz*qaj z%;}m@E^lQl0I>KvWo~SMbY6E7bH3O%{u5K*dR|Zc&H@ReSFAE&zk7yZKfEa5u-*bG zJ9{v7ws}&Qwlsa48F@wvH%DfsgJygx7w>!+0xdXJitck_hJ2$(7TmZUxa^NtXdBKH zphzefyMIpRV_&h*+6{ZkNgv)=fUd3}w?tiDn9?De$Q%VnOt0LwuMC9QN{TBmZ2cOP zYWy|NJAAHImF>LVlAKlhMXym$GnXfBT-xSReRO*E z_%b=che8&jzg($h%c79rTvQg?-Rv3&CzxD!Rj=q3A;~!EBe^F;=L0LB2tL2e)~f6G zDCLez=3X-LNq2eBrN^+YDG4Quq4g2|r;S0pPL3J)ZTyc*&4kt)i|`&w-fz{;gcB;@ zpvy+bZ6aWA#F`=wAh+U2-*6qv?cVIp?Khp`D^2;p}TU~UQb;?WFLngg(iUdJl zdJ0DnQ>{ms(STI150d-6Y{-Uj81N?;8 zMA;^f`KCzJu?L1=$4^5!L$K)KZ9(uCX-7530vW>>!S$8-X`A=VNwC}r&%tsE!3?gE zJkZP;wRh%zI%{(KFF)sJf$BZ%G=X83*807qX|Cmiw%MWMyiAp~IQ{gy?=G0hDkDWI zn)h>V`PEXPL=D+<_r#_lJNlWVZaH_R3~6+}ybr%pht**=6FNi~(){R%vqYJma zP-ktANo>uE*yCC#s5_Yl4i{BT(~`*Um$T_zh>T%B7Y_~M5N`Oq2Eh<6vrpwUt6$4h zY?$lfZtApS6~OTI7ph;ib~@qua0>(G6cMA~mwKjuTZ^f1*t(ht6|GmK=9?P#^F!7xJ*k_yso;3b{!2p0@5>g`c zzt#XJ;U~@RZ}?SW%lQxdZ|i#!ebNE`M&Bh)(f`j6o{V(TJ^hXpM*erCe|xEu@RP#! zHylj$&y4wx%smM{Ir)EsHHkX;*H`?Z15V;kjOJs>q85>QY;kSfwa z=|zf27mydd?>?`5-+Rf*IV*q6J}dK^*?UiPH1P@O0RX@SfUCB?>B<1`@;L6BEFJ(r zhP#`&S|Z$pul~GNB;D4;hmc2XoiZqHP9p@Py88Axp-!B8#91M&f4O~F4LHUCxK#lxuPFO?7B=|;?*Cko!pS{0o8DY z@mRr%?mw@Uyvm^E4IeZw@Gf;ZvJ+1we`IQN>Wh*j@B6ARuG$Z&^xig>yo+f_E<)YB!|-@3niWYMlVq?KjGwQ^tuSY|?yxqOzD{BLBga8Qkp{aVBZIO2 zviLf4o$TSz%yqO7NC=-7J5*^uHu=Tk$E%vKHvR|Ap%L-Ou^MctWy!O|s%SX{B?|t6 zOX&j4EEq*{hGq55S1|_2_Q#ETgQQ$cH`&8Z=r4Yqd^?w8qJSbG4IqEtp4An=nVNN3 zk)xMBkBb-|#Kr>yai8sRDm|@af09C0{85sq(M~qD_87T~V2eO8ZSn|WTNk#zET^xQ zzwqoVME3jC8wSQ;?#7)N0ij!x=-Qu)By`6rXv z$k&c-A~;3)#t6(!@F-nWwQc(5VYf06KImVVTLYl z>7f)+`eLfQM0lB5awFd3WsKj5(@_$fCiPm`4yAx=nA*rUh#$G!t02(BvRaxRUbFtYrJUyV=x8 zC-RZr@Ia&%SPkN2yP@k*zX7S?G9%k&gId?OEksR?Z2@Qc1^I-@U@{DTm}-@^R?->GOU_Q{pyL_$p$_QMYZ*du|!b3}Q<%N&Mr1f}CN z$A-ImIl01x|FRxVejW~%j_yDC`pbrTJ)~_o`5iMn*pjG-i4x!A47pRvLH%I^f0B!< zeB~3XL0w2xB)hf5&1<(LOii28vImbkY=G+Y_=**`23Is*$$g4rtm;--r8H5WjegGh z#dvHGrf?gcp5<$3!RrqfS`*=q|H4JMBH&mk-A#JLm9RXucm2}w0DKY}GzlW9I!fWro8T7{W+)d%J)1z({QfO!3BBvCofOoYR5x1-HWKVULlNof^XahD3N?8 zmaXBuJA!S9t_$>@z89xvLvQ6UtMH#mR;?5LDH~E9Ch=9AWaJ3{a^NpE{>ejOO0Vlv z5$et@e^M1MR<)tH9)jgL+u}$o8H)p87ZjTn&hJF zsM_pzw|Pz7)#>9hAF0}g?W--hyYpWFH?ApEuiIAuZNaCvzojanuw8WTM$Ag^y5-PI zCu8&8QjM^0B|TaOOAGvQT1kg!P~nsHcjn(;^Mo#>vLcuyw`&sapQJk(5Tq}@r_dOE zD%HBw@2G7rmv`I$a8dRLv~ac98E!gqcyQdiVMXg#uu_4_ywrpxW1;tOtFACvmW*JB z$`mg)yO@KF39GtDohgS`*z{%V*CbQ!@h1vP(e0qycpFrm^*I<)JrQk3X`XogFMmXe zB~m3SJ9qY6np^doTkLN*rT9}9cc}*MJjXdCkpuvs`r9RL?%oJXx1a9lHRyGu6Qyq1 zp%?u0;dZ%RS;?fpvof1fsPZK7ll4BHBG}~EOwnvdDRE!aP2!v&x;Y0O!o=VWbh||0 zGkMzkw7xvoek>igyziSae4VdpkV%?@+^gq_Miz4xH0L6BYC}6DqU(S;Ksq)$G3f1m z&)ju#lZiN{wXPAjX%-&1c^u+xwHEQ+AQ&C(i{3>jgZaAQ-aJYt!+=w{!r?gNEo<+R z(S5)oSDo>vEL(#C1ClGQwP`V&Aa5=G-3r8!&xc%65DWdPfC8xwVE-0W&*YZ#Uj1J1d@FL~-G#(!86ACy6x|0Zj zS?7@fH%Jf>YO30uOkV?{z)o4YGZEj2Smu{~#cLah>s-b>!dguB%iSz(Jp%Je;mGfJ5YTo|y+J`o?B z@VF?H^aaF{aiL?nzQN}TvZec$;1&VQ@GvuL(rsZXAWDFaz1Y$vdWEW86HdZv4izk5 zGfQ`D@JTnKiQOFsl~xDD?(}+Reex=?D6=O1_S(C43Zv(s-1{LQL+>Db61dtd7ApDX zjOP{GL8ONtB6=$=yuJYT*?)Qe=A*zD`m2Yv1$SVIRFgc~){@6Zdpo@ciHWM?7E<7> z-A$)Iwf(wt{K7?C&ojga0BHZ#ww05kyTOBph+k_*g%0GewS%b}q~( zZxLu>{o!uS=;KkY-GfFgL2wr;Z)QDvJY(FrQa+nOmPjH;Kq{9(?~<(C#hm2ZO!SNJhI3i$qy+u|J&ETW4j1>*nwO)3Y(UUl2=*3#ott? zm@OB?sb39Hr{|JRINr5=)L1H+)`b^2ETYtJ?{%YzzY3%?fvc=Uyp?5&3bI7@L{Sl0 zN5eN?RF?6src&fDc5x_~-SP-~2Bsb@6#xrEsZ`<<_SXhzR4`d7_HW$wn!4<`{Bq|3 z5E{2Bca6(JMDcP0O+2%_Lz`?FC9G(aaGq$me39epxwt$Td|?ycs=lF}Yaf5HsN*3q{!K%hR;|gJi1#CidVD;%a#`deWL)dYxb?^cJ!0o^mi&WxehFrHYNN|s^6E0mW67bKwe#SeOn++M(2Ag4>g)jY zy916waQ`JbX%Z^SFAOgMLliHBWGF@fs}H_s&eV;ty2@e!faR8y+3`WVIrXWkDLc_5Wbob2gg_b1B3@M7QZ4R-=0J|9hAY@JdkD@)&@31Cqko5eR!Ls1pn zNU;7M0xvv~iRrhZ5!$B+D77lWi6>gv-)-b7*TCj9R&LyTd);Nk(ZJ@Z z9V@gu`=*9p6-v!S&sT-k$kmg#YGK^L_(Ym-iw7b>T z&*@ap&=y+J#}CXn?gx7Dkh@u;Cxfo6(YL94M5Z{~WF903JncpYN&9I)~9rTTJWgL4e|hIzd_s%M~4%Xi6X9-;nr9) z18-+bH`AYT!lwMnQOXCWFINtQ@oA9*SwE(k?)gs4KteWvzoPXp$K0cy7%89y$~_RA5!%(v#IwtB zp4K8&TY4X^T2vL>@f}x9yFcT3f0cM*)-e5*4$}E$2(?7pX?(NNj1rGa-?6^ z5_(Acm~8*v*<2{AMpZ5Q3@|Jxvvmx*EW-r;98I#};p!UKQKXpQI>9#hL0d-?h({0j z@7W%AEWchsxO4g=>7NIlkHmk$0DvGM9uE3{0`htI`55>cejC?v{saGaC_IlopN)Q_ zJ#n-0e)g?Z&hURnYoCh{!%Ae+syifJT2 z2_GX3ERr~$l-M#yVu%y6hEv(^Fj5K(S6oj&d$HZl@w#z=G)lWf@dVY_^T0(5I7+CJ zl!TFv9f9s7F%2*3T^^3m^q{Jx!*sSl>M=sI;=xfG7#Ke@FuprByG65JN{W{`@@|46-d92j$FgkV*$W5?*sfm5l{@2 z0Hr_~z^)bm7a+rda)AAbPr}Db1Fr3GO@5g9ryW%SBY=^>C}1>j3@`>53seEe0^%wAGH?N~1h^3R3a}Ko2)Gzn1~dUdU^&nXga8L<0agI5Kp6Ncuo8#>Q6L7yfi~b0 zAOR$S6wnTI0IPu20KNI8z*=A(a2b%lRzHBpDlklx9vlp0_kHu0*WO;(8u;;bemVKX zpFK@|Pr>4+4`4mVXHSkp+xad)zi~T2-?0Or|M(R^AF&IdFL?o=Klu+ppF(1!S&Eo` zWi(>?6&o?>KL_HpnbAN)e3g@Eh=*e-yCLBuomeWE3ddur9?ex?+rHAIM>*Jrwz?qP zoQNmmEg;uJ8O2h^S1#1K3B!hEr$Q`h_8pUcTG!}uLusdLBh@nlW0%+>^** z5#=&z12x4-&pB#VHf;3g+G>yRt{G60!cLSH4AY$0xF(r$XzB=@_0_c%sxK{pL67NG zaR#~{Jl^Tc$a|x*+)yy(OizWQ&P~H$u|5SKB&|g1fK(qHcRiXlIbHdh{pfh$%2paO zx%OM)@syLeDN5aF4f&6%nR^Gp=4drF#hvmUgPvx7JmEy`d2Pw|D0Y*{*pW`g4ln{c z)nx28OjBRepv1%LZPy@hsON5!p;yHX8!Lx^hj~g%ql`<{-dgWO;t4w$PT9ezld_xR zv82(N28 zkFCaf<@q*m96JO&9Ite<)wo-o?aqzI+N04mb}YQS)ppv$$!I)u9OjYFNq9!#W-t~r z$vjhI;WLlHDUPWl&+C(eroX+$E~UR4HXhBk27?Elert;N8qcfUd2u+_91gYP=+d4V z1Ko^*R{7>rif1HnWIg?D(DYPp`qVvg^TvuH;DP&hO>wz-f`Q|u%n!m{!vTfj@JCK1 z{6;xeA>YM#nD?ua@*&_~PAdaKG;cxTYTmcsG3p;J1tC0G=FQ21uKCRh|^y19&I!Vlb{g z-dTBvJq0m&cJHbjdtq#9vKgK%W;eFRaUsvYmy*Y==%_W-zj$q+Sn-6eea7R!Ecu|U z`sPMX>YbKgdn9GoJIU}0em;;=v|}6cL^(-+Z=`Us#l5 zHomZg@uen$H2GxMcUpzQx~1z1n`TtV*}&G_zG}CgWv97u@|(QYYhH#ZaW_ zXA73S30oX58H*p+ z4sFRh9@;*>h zAiT;Pb|MK_z1ZiFuG*9?kr(-=;>41-JU@qDGUXKyud?74%JCJs=EatDNKKOQ*+^l+ zSs>|jTE+M8+b^}XT_Rmw($$S=?$vJd=FO_zyU#XCwyQ5-3Xe-B<-FzEJtdQYo%t5)&DB>8<)YP-87+t7gV543{-T9ykh zK#Ypea*4(tW(VLvyR@Q_m8IiFp53!a@&CY@Ce`OJw;j{NAEeUsY9S+N)RM0qWG^e( z=Z(B}9D?;-vi~K>1l`SDe~xOG`Nx5ATUTDB+EI?j9{Zzoc79#bTeeHKyG34n=Dr?! zEucesGiNSUanq8u($bQ}Twq+t<<7^VnJ1|FsrCqYcIUOSac!G?d)aj9zE@=9mRlj^ zHftu{#vu;C=!XNCf}Y}`7vE;BLsGjDqn8`u@^}i+PT^2DVxx`*n8w<#upc*Uq)lAj znV)FyR?fu4XF6SeHED3P1jJa~HYP0%1$s#~w+X#!<(r8=_#LL16BTVOI5 zs3*4tb>zA%>PL}9I}=AK*AFnAJ`9!7c~mO%sFdeX3FJ}Xr<77}DHyTD%yk0=VQ^rG?Dy-fzaa1!W(-y^(mPvE7UD@s7WpY-h zv`ERaMhBniLUzRIa3W=kTf+%AJvZK-NVVoF=7w9`G_}ChHbITZ^PB$Zz*NUsk2=g! zaIKw2RnMl>&Mw5RcD#LPr+Ul4v~#fq?G&yL^wthniqFt9-arO&GtOZ}ZnaoH8Q5yk zf4S?*O_}OP>bLT2M9af=TECR%5#(HX3i^`-~Q+KWsr4?R-f^5E3! z$^+9ka%nBmI|Go2tDGydxBaGmFi?r?K+}jrB_etM{~?iikVH=%MWVAYiaST@pB`|! zu0&UGDsixcU-EAo8G6TfuHnV znno}6=Iaj`ZKqCMXvx1lf5zzU>8+PvO{1wgIMYX8j&}6X9f<9mkEWI4S25+IhpGC0 z`e^iVee{!{aa(E5N0Wg(OGwLAcm6ATwc0xl_GH{A^s!uPPanu#@7!s08V!}Zee}W| zAlci7Iadw1EIIf&TK}0Qh##f(UwCYkMUW1m#s*g7LKVf189};C?yG*W`u6HSRL80} zEL>5&q}r_J%H{>;cn+h2o8rnn$SNpZpH>mu)CH?};O>qATZMFYsq-$+z4zR64+cq$ z;!uV<fZmW-4%2>ytKN|Nd^igpK8U#Yt zd~Hzo5t`KDdmSic>kkvo!aoKQYRdTUFB`v2bLtZPCOJk1{{N@TLEY~62%j_@Obz@G DRG@RK literal 0 HcmV?d00001 diff --git a/tests/samples/presentation.pptx b/tests/samples/presentation.pptx new file mode 100644 index 0000000000000000000000000000000000000000..f05207e0949fb2bd85b1a07feac7431e5989c8b7 GIT binary patch literal 6713 zcmai22Q*x3*B-qaogg~F=n+KkjFKQql#JdxlMpRhqDJr0d+#+25g}26i0DL(ZX!yQ z;G4Mjeq#Rnt$)s1=d5$q+Rv>0y!(0gejg20G;}fm0DujE-hc4O%2~U14)Lah3IO0C zp2lvL4(_}>zwZ@sGWK13_|M^I zhC8FYyu9FI#nV|`U|$U$Ndf<{rr79HXv&j`38N5ih{O z9--a$$kc2}9+bagTrtUydMYn;_%Skgx)JvNml)cKexaxc!STFMc2t&HlSPr~_aviQ zctEaE$s1|2bmAZ(O9S^xMt$+bDbpXyxj7Iiy%r-{b#6f0V8a<`dbYWCmj;v;I?BaelL(Zmpi$7yc_pyg_@N-x@NIc_W_ZxPs@XF96L5yQ5{ z#T%_RS31+S@XSW$?v!&;aG~9Tja1rCOmCW>zo`v=%VF9U^x|dAL@lhuvN%7wI!sbp zo`55dA(`_Ab)y_U`6_tlO}KVUS8VgaCtMaPF}mPiWH-N0cQ0ldO6Q>C`r!|BrGM~a zN=#>#qyI>j&4PtSLCXfAV7)kCl7Cf6my&>2^hE@-*-j#&?lflKf%ZiX)eaj7*0BuV zTuojtYkBg%l%={+w>FL#{bXY77s+rrLvJyf*2=A2pv(mJ_`&G?R?6egR<#48HBXsI zDs7bv3GD7@^^ivm=AWAbN$z_L_*;$I%*$uJe|GB7fpVQnND@l)DT2au1f4{GQfcbq z@|($wm`&Wi^TC--Tr zaY|3CTFKQK!$B zWg0Erm@Gluji7x$R+6)$}0$Ld40PT7Kt=Z3T@ga(5 z1i{Pijp{cq$c$vfeMHC+-ybfnkZ_t*Xy004(26H38%X*c!1%Z+^qZ1vk5FdjuzIxs z5oy!A!=J24u!lQPB}ivpZDFoYx(2G^lwzumLKdB~;%yZ^?9ZWt)WvIbF+7`P&p{bj z=%n(YEMXKu1;uNk=z0P#+z1TjCq()9#|^3!LiIY#UQ$lmjNep^&o?n(my2#|4VLo# zc`K|#-fUVI+}!Avm$%8VJA9yJ1PaMAUYtz54_fD*5{fMNqD1cg1aD6N%e1G!R>bN> zY(838QnFy-^JkOIn$Xu119cnP6F;{qVT|pmp0VBU=Z+ud+x2 z;Tk`z>kRYEx=nQ;n5{st%=I_J?hdvVmg=VNP)j%WzsvBCPyAP97}JS+DPKv2-q|O6 z3{0DCp?u$2_E1z_Id)fYJKoya<_a5uLOmc*(V#$m zXkjXIOiX07Y$Gr)6ikA-jS8kC6~X6!EnzN0C=w=JAr~){aS$I`avrYqb$}!gfv8m?y4Bc+*cZl056(GKb&s-g>A8QIlTqj-+MxRfja>hhc4bM}Cwt z57q>UM|2PJ1X$6eyMv(9b9ROhbHzJTJ$VixWn*J@qpJahM zG~H40j}_DBwzRWnaqGPS3Q3NPqB2@qtJ_Mfx}vvci3TWz4#Iw^Yt<)vz~XG)XHY8Y ziM&$dUrb9iSZjRS0coF`FPb7%9;&Dhcy-O;=Kdx8FT=$eO_=(1V`Vc)KZ&@)1zkmd+3J z-*u;YtN0K;_a@FLJV@31T*%g|0sK-IIGFLqSREGn#6T)W>QKwbF6`-TN3ehhs;q!U(L0F*AqRg-VLNf?5Uryul7}_jisX{@8$hc z;7grJhw|m-*PQy(XUXwcY6}FMACk6t{LXWtiE(tHZA;umGgN zxVtdL!_b1&k21aETL)1J+GX(&p;NXxt=V0KjT{!ouVy)J5=2HFq&2YME#V zp55QHzk~vGjg&Y$>ibG~^t{OXq)E?DX#RNiX2%Q|uQYj&*pFIaVgXHCIR}_09A$kH zXi@M>JbcKCnCpjb+^h0+?SpmW4Z5T2YdwSj)-jv>UQx>2bBArXzXE-DM^>#+54KGV{% zm8}@$KuJ9vLh)e3cegB`_UGOpdkcOXlHg*K_A}n^RbTnDyD63*W_iW+KEuB-<+VPe zw=z%N-rK&o+i~{x;i_{#CjDWZ^%?2)dYV{~(=9^?h7GxksF^->ehj_VMgID9)kR)L ze?)W>S1I`CUY;?kiu>p`-O-Zc3yF zsvQJ{xu{LZiOdDgJB?QV@f$Qx{~HN79w#i3J{eu@ZV0oP7O8$}GjKWl_K}Sxk3bYv zzSoCX7d;*CNL5Th5Zh+G|HL}=!wZ~!;50Bu6MDE|+q*(fo??}^&iSD5VYNV10)|U_ zTb1k`UUGj+p2|ZZq=SM7xnM&ZLPUKC5nbyT|0I3cG5$z*TX))RijT+_u4l7<&w?}? zKd3i9IX|P9MGY(|{Q*@lmh~kI_WovT+=|GcV@e~?x0rswj`w}5^|*bZ^{Gnk&8B-c zI`h9+xd zw=|?`UdrRBt%Pzp*zjbGYZUx^!QXTB%1>JmM5%q zR;l$JCYV3^w@VZr+nY~al$mcd_D@S?j-GppQ^CYax;}p*68-*6D>`NL^LfYJlKeB0 z*pncg%`_>-I;7Iyl0}D>|E+h1zZ>OcwO>>E|Ij#iQ$}9Rn>c44ynz;M@e1=5eVq2z z^eb~bI`P55E#ZNJvx%@aezv8XA!)kO!{DmO&$O9mj?V%cCm+#?gc&i}3Npt2SRx7G zUv;tK>LspF9>s!o-|(qtqP%ejPqsZ+@kwzhbA&ynAPw6(S;#~QwQ>jguaNDKPUSYQ zyN1wXB@ESgO;5^*EcedvVt)5*sPETcI|EMt#1@CTreoCWJKK4w3Z@hq01%A}0FYnL z^fKYg99MKdI+5}dweOK7!At7EHgco+oB>SdDx>wldC!b@qxLk@$4m14IJ@XGu|$+R zMxLqP86sh=;0IdTt^l0FYB9PYpS`72eK`HV?Xbcb(HtF`TVgtq&1RzHcB4JkMM{Zm zih~alwMxjHmoPkzvfZt})~BlxvxCNS9mNml>akrN25xUWsb9c7d@*G;+BFiTY`2at zeH(1|WrL@;SfhDZxareh5}#iaBJ!polAy)yp|=t01%R-AOI(P~#@78WyN?cobNoD7pP}vcU#k>iJEAP<9jMLwiAtfN@Ew7npGn!KOi2#AI^*!#@ z0IknTWBckuPc(MXYnE(>m||sbn&jJHzJJ+!CqULQsG`umNAmavt-RuR#iubJo4PNe zZVWUwb3Gb?5#QNfwfgK3_cu*8`U?5&sv>Tlx8F)my0FV@zNXzvS}ixKBGoZuEhU96 zDl!rW$7X#U-;23FqR+~C{}Z*C58mb`%hs1TMdoOWD%}b=rgaqyV@uO`@ttt#I7Qzk zVvG;e?;ozxyJqA_*Vn8JEWIiEY(2P|TcB(L@@2`76v~&ai^(7m!uhrK8bk}sDTrVY z^yhsiT1O_T-au0$3!F3{jpmqPjyn@(Gf5u$rW6Q8m{+-X`kp}QO;&5RU^ zL9%|PM==6`t&K49nOt*85Ml6Kn`&;qOrM9BUJs?4NdU8CQEIGs=&_LqYH7U^x5V*j z4V`_JW|kqp{Oqm!pW-1o^@n^_xev)L@5Iy^I_;#M(66O_o13-T^BQ6~@cyDRB)Hjd zQDY4m!KGcMuga6u*=fmsg7CYr7%D=lFd4 z?8oBgAB$Z-KCVbjYFM28WLp7Bm{-WHx9E)SRqE1xQ?+>Vp4y3gEx02Mn@Y?PzbVxd z_?1tecBDyD;=Sx*JI1J)BNd_~&^v5D`2t#vCw|Oxy8aCmlN)!!e)bL` z&8FQ23(x?>96^F;tE33U8Ok|3K@k>+vG!9JOZP{=$8eaAm~#*x(cB&=W*j#iT3m`7 zmlItQjlXWB??TjMV%SK^deLs3O~R0-@w`@kaSQJH^(Ldy&G4D&AR(jhP_DqcyVCCj z7MLPFoPO)=U@cqWnxQ490>W`gV=EdqqO?ezb4=J`aM3gKwI~M`^9GB}b}9S)Aw#Re zf=O=!tJ@#bTHKJ7?t9X5KnHwS2_?SQcDj)nACCQibCti&Q*D-GyW94|N1S;G%S{iz zZ7ykxr|%tF1oS#U?6Hs0Jr_jX5B%O01^OvJ^-mFwG~7dPyYu-lk=!%Dulb%0pPQ7g zN?7E$0cVi;C!%!1^WaRmQ`|6VL&7ey*b{i5fcjJX4>cen5`R3=0E_HKt;RuJTRvrt z4H3!*7Tcq(uPGmYW$B_Yn88?DOurFtzWe+}1g}qqYF>ScKyp%IWf;GloeMBkr=|3Q zzc}tF9WCNniN+a(P9A*QhHz+$-J`LAM920d@u+6x4*!XY_Dqm?Ltv_8T;caM{!v0J zM(&Q?erF9;6jZWHN9NzHW5lvu{`?W^c+E&g4n#Iv{|W;D{86wFq5sEvMGi+c7_Y*$ z5QEu&!~eD$k)x4ax2tGx#E$>}cyP#I$X3-ASUkc}_~#_~r+5C-pL`Kc?|ZClwhPd8WTY<|FtQWMoWaIOI9$3J!$u!+#gW zCBn7E{NKxnJd0eRkt5uSKhQ2g|8?n*H`gnO5QM+_FI%mKDh8rN002V7wSiE@U5ejt F{{i=~I{*Lx literal 0 HcmV?d00001 diff --git a/tests/samples/spreadsheet.xls b/tests/samples/spreadsheet.xls new file mode 100644 index 0000000000000000000000000000000000000000..ee1bed0548b8957ce27539d2880be880b6b3ee81 GIT binary patch literal 5632 zcmeHKU2IfU5T3i;Eq7^Yx1~h^scV(e(yiIbgBZkaDWHIbiZv2LqT6x{#ihI1ZlfkD zOOXc?NhD3=1!GA1Vl@85L`^iYee*%1F(E+`lj0lEhs9_NN!#`N=H6|u%`WUVRfFl= z%=wvf=A1cm=FH6f{=1r~iyyDMDhptnl*w$?FXb+}fwr0J2Shw*lQx^pW^-gTm*FFg+@6d}&^Z|M2D}bc=o%%L9(z2)}XyBm4@uegvyJQG;QeMK17Y}P-G4Ab(;dJ|p zV&>m!&hq~A8n~6NIPTUz{Wtx18F1nIr?2PxUkR)LRsvT6tAN!&gaD}l)&f@p>wxvZ zHNZQ7cLLV}*8$f9?*eWBHUKF@(5czJ7w!9;ev?zrpK7`FPieq!;ugz&^2Mre=4c2j@W(_XDtr&nlQm-c0!(|2$NUi&97(z_2Ql2-J{@G;9u zy@AL61ineb1d8;YHRKn0hAuyIP|sSPp=+m4ppnqv+_*(5G7Je{;b*~=QwBnaIblRl zMx^07g9d_#iC|(PNSJ>jj!a7n7gIz6x;r^Cm_pqZ9~olZnMlN}GBHku_l3v8S;Qk# z&S5^|QNLLaM8g5bAA*NL%XV&ozd5%`$b1B6p*yHi`2$#P`rYe4bocdiJgf9HC^^_j z)#8AsFf!@gtQ@Ucj_TYx*@nPoG{H|aNu%__7n~4FJ;#33FTiq`W1Ucc5|XvY-x_Mf z%yxMIWhdfg7iKJq{YrQ`{CPNtWqRbaO7xnRO^|+z!nKY~u68rpr`s-%wq4G2w7A+A zA+{enG4vaikP$L+{JGkYr?9puuc*>R+I>s_MEm6HE}!qiCXClB?7n!IEZFIVZ7rsU z7NBh!4dzx0Y3z{GO`~YsB2 zy@#P&!Uku5H0@K09K{njI&W(fi-DfAX}?8)flR6lbIBGkc}T=dEoC=zuI|D9WPCV2 zkP7a4vEPaXA9}E@H`-^#VwOn{Xs|g^5yf$O!o@{fi2rvhc zTmyBXwrk2*8S%36qy;U$Bm z{Qpz=7uaAvLHVyS!HWYt4d0lQ=cNyI9CfKs3vE$fczOB;isgS>vt8=EOTBYK4nT(E z#j?-M literal 0 HcmV?d00001 diff --git a/tests/samples/spreadsheet.xlsx b/tests/samples/spreadsheet.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..4a83543892c06adaff4de09cc115d035175a49a3 GIT binary patch literal 5463 zcmaJ_1yodRx1JFPkZy-&Xrx0)m6mP^k!I)~K!Fi(Bm|KdK%_xHLP}D)q>&I98YGk) zkS>8c>VJPfx!<~T)|z*nv(B^EInUYee)hAqRI#p700017fQP28N#mdd^%u-p{uKa# z2D3hP;y3evI>GoKxq3KSy1F{@J$82D`+22EN5-K|fTVtz(sHYEBRfckXPFGI=Itt0 zP<@q}ld_$t^Ui63x*Z*ECQkatJ_p+ni{4T9m39urwR&AGYjIr9j9|V!=I^TrQQ@d360dyD-gX~MF?WIv+LRc_?Mj*Th@k@46m^{( zK<>KNnY?oYFNO}(hp8X>!N7yt^!VNO*JCpxdCL%HkJ(@FggvcKx3~kvUeJy5By{hH z+PJPe8F2i8<3-ol)c`%b!(FM31)tA+g^`9$FtyH`#nx(Ot$iv!Fpp7Dt-pA|1MQNY z0RaFw763r}o0oqw@={mbbxMGsVaotqD_IrGuKG+&;oFGa( zU{1ByJ;6e9!(5d$r}N&9y+5olry9p9SYPl3He^@&Yb}WbdHI%R{#|IALNOj+CE|5z zb6IkG=&?+)^V@w?)@e^ip{$dQjQK8}g9}bAP$5PqIf*6FhJ3sO!0!YKtRx7r5WD8U zN3l*~^DRKY(Ky|>%u9=p53e)a6ruS(ldA`rrb*kUu=AC z8|69pX!5y1QFO1DOGTWZT)AW}TLU2GhU$` zGInUM;Be;YxE{FN-t)FK!_LNgv|(Qd#DqKe{R&cyLL+|T=}sjTCB;L%rI4|bG>_B< zuw!q+t5d{h$x){E!`5&$GG}3O3W3A+u9gEsJ($KY32w)}Z|?JZBV@u_{zD)co@;2j z0LIY?)2H$pYB(0)%2KzF15`7o9MI)}*n9;xTdG2$cCPn7S?y~(`3i~IFNe)qOF!dOBDn7VczX*Aet zxsCmIrW}-RBN#BdX9yPnApe~yo_0`YDF3g2!JqqQCjM8pglwHsA~xSGe7N?RNkfna z-eXA3%A9L-nd9etPvnVLHu8Fq<$-ULe>Eb9TMM#1KgkEl42F#&1$!H4H`I+X5B!Ym zDa2fpQgX0O>kE-i5{Z3@S2FF%UHv&4`-EEDb$2*|Cr6pF*~HRodF7#8LmBW$vnxtt z%dB^mr4+_vpQ)BFLUxm$koY|-3fv<}^)Ku`WXypKFp*1KBZjV1)&V{f*6>IXjsaFJ zPLgITP_!Pm8vuai#<=;3p(_i@)1|z5rjbh|16vur)q9ITV)uyKXGcAAL`T87ekaui z{Of;qb1r# zl2|}dITOB2B4Y!2?3;!b*wZk^2!OIO8y$aDMv&AA1=ltf$Ey?s}$k-(I$;p9`3HIskmcP5^@zP6hI=QM4+1 zg~qwrB^}24C-)Qvh`)w*&G1VZ3~!AxXVnET+E^u`chMK(O=qo!%dWlHj0femXXJmU zQ(k%>A_tRBi(~)*;qP>UdHOg(VLuZIZD`^+B|!0Hu7t$b44rL1Nz|FM-erixQpiQn z!JubcrLb?on)C8>-47}rIaBi9sr2d&>*`8c*5W&E9!UvfyvWVGTWa0c80dn)+KG8_ z-Ec(7^6()Cm|E3Z$6tD zx3tLHZH`Mqj))<@v`NqEO0O=Fb@~dP`P3zS#&fgG+w=DAhlt zapg{R{MO*PI?p#;H()Bt*ATILt3eKfz7bZb*ULqqHJon1wh+Vu@gy`bfL zo?ITS;i2LxMRU}Q*+IF?2mN7KTc+4gj8#rKR^Z4AnukmpW>T z%#(n)b1lUlYHzuYW!&a2um3bpE9rJ%#Z+^>IMXKF_?1}KU4F$^eCS8eFax0GVNx6n zFvTjAh{A76;S!0hhTVjsMY8jqoSBM|#!1C$R6HOD4k1GuO(DwSU6FAS&6fkYVQZ~3 zYDGQfkNOA>ZA=BrwN*tg=AJR6tQqF+Pp9G^UYt4926zy`6oKV$Z-?(j&x zWV?5xX16%KsiM*G(TVA#V!*4afa-b4=*kSP=6+kKafH>hOw6hDqYKUox7n$WGd$pG zwG-{{DQ=lTOz)0bw4>aPZUD~+0xGlJqeT2;Dh88!vK_=bZ}+e8YPc4~GHe~O%|@(U zJ;=yOIp91GZIMV6FuX_$X9WJ8&)ymp>MWM}gbBxoSOK(}eIR)X+KJY}}L1M7`e3-;?}@-!hH^m!^|LFl(g zkUN+G#P(+T5pM2|Aeec;Ua?<6KOx8whxwWu&B{B3et|+=dsJ&1nKNYM)j;4vP4epZ zH?}yB**;}!F^^@b#SvDvacUxBOBU+mx2quvTegQ3i}wypTL1QpDSWgW{{PfVc8`;{NA$Vt*?D-SwpkgGu$LlQZZ z@9Yn}88)ygZ{<6RM;6iiIMjnw!M+#>XFm{g{9&X6mk|fPd*x*X=dKcNF?~kmrWQ)x zL`qV0Q>m6Oa30j{g_A$zpbYL7X=!`%;(@SR*>pU2iv&Q;Lr>V5Gf)y@e2NX$eB!cX z4dA-5XsHMmwm31hL7iVYx;Fsd(s=HX2*GAnun+^AO`=crS(ZVQVUskZ+V5T0SivuY5TJ}@l%Pq?!1 z%f$;_bCNE9V6T0v?_oGzxZpf-ZRH(RPkf&F1dr2qd;Qs6&(Y@WSt}oL_bz-RXrs2V z7@`yp{IBxq7`R6(+B3JKVqfLJ*5y~F_I`xSZygqX4mrB-e^5CAb^V6 z(lzntZA!esDG8G~?9RX&khf>J(IY~l5n+~J!F8J8Rl~fQHWuD@_~|rXtb{W~gIp>3 z*(@?FoXihP8b22`?XYB#v>eS*%6Niw-;Cbt|G2sIY*|Ha$*d zXDcKoIKc=DFJIjxBQ^Nb^2sq4#f#j2;;_M1Vj-UCSPQEASs$r*j@9lu~J1W7Vau2A{ChyV#I8wB?+)?{I@(FQ&#B5ibcC zK#=@*O_Ydgq;&aN)=2%3Q?-pTtCCog_;1B%K&y7QnEG8~{M(33Y;V#d{Jocsd!93f z$##s8bHC;|rulqC(P~J^*|p9$)l~Z$bp2igHgyhB7)p>*$x`*Wx4<)9MLdA@RlWB` z4NFpDJ*2!aNYJ0Ah{neqA1(Pe5BVJvY;>4UPmAG!6vKnHtCg0As~e2p%GCq=&(sIIBYntEhKCZ+MlO6tH_Zyprxd$Y}dcGByHeZ zexAKq$V0`0#nJG&*l~Pn-Riu#EU1;sqW)RP%jog4hJ0vVc2sHD9cg(&t}Ld68`Lya zas-si%A3vMdeLn$ANGgvSizDEK|d(yzfZL=vCk}@T+Hw~4nKEP4Euw_?HA>z(R)sGA(`M!cJ79zrLFblgw=O7-kk-k zrB(W6NYbk(OlM9}nA<^^LsuVIeEFzh_M7a8sv) zGO|DW+=`e?^HE=*&4}F>`Si|!Q>Ug&>TLQ77&IaC7^~WNLJaORE4u z$d+IDjfF{ed3rfc!buTyPzAk2g>gI^pS445p1ZcsDQGZo7hV=%oQ^gt%x&x?5|Cdv zQTPU8p%z2G^r{Poqg;=X)}-;m1-nEyJc5)^J2G&BJI#SqmalO6^sS@7`f-X4p{DK3 zl_qNy{wtJkJ;t@?s`T65jad;BaoIY_Sjf6zt1%Qydq9ebj?D75Pdv@0QJv9`bw&n5 zH6ThduJ&kcuS&E`8LJuoHl3_(W$V)O>G3V#>;N|>Kfb9HCA{jrOG;A7*EdFS6EpI$ z{zt%<6-!PBLX-VV4+s6G!x|c1^Dak$RNWblhOagp-w3!wRL?`8m>q_sF3@k6XDsLH z;t6%}G}H5OgThRHa@C-xs)^ZyjavqKHE(Xw;sj~c(LbiMAX|)(U8u5fCS#izKfkSk zHw^X@a9;SF_r59AWbD{(=vFsfVeA7+>*#&uE8&ii`}d<0y5)S;728bj7}2<0`*y=1 zdeRRjm%n(kq7OP5*s7H8+$+^M8BWuYTX@BHV#`dsI3DM#E~8kNK+`kiCz|wW{_nNJ z3S!DGb@fd5No>?u#CO4OAu81iJR&Z{Zp9x%>h^k#Lb_frI@nLQyMVF^2!RtJJ*{Wn zAM2i?qruXB3=v&H1E~DIO1ufLL|Fu z^dh>C8ei=LO>@@qV;kPswZO(sLByuG*o0iRIMd3HGT@_450+KN2lT+mj>SFSfF#3h z#x?0rJkM=iw?*&zv7l-Wk5{nI?}pH&Mi$*7GKdLY>^(!_uZ%E}8uA|otj9C!q)(gS z_9_G(70vY8PxrNHYN-ORQ2eU<|31RPr17s`Am(%Ze*>+{aW0R1en$ZS0)b#mB#g%V za{zQ1e0d7+2P})3AYs6lX9AZ4T=so`0>ofa?x({2*9BgNUT(4fKrJvciT^zK|7o}{ zN4YGjf1*?p{CtqWqtY+p%CL_|DjfwftL&VA7BO1zm)gO@XJN)5Bvf%y8Ev> n{HLT{Mql39f6$y5aru{huB8gX{&^D~<_N?{Pz%PX0|5U6ugX^@ literal 0 HcmV?d00001 diff --git a/tests/test_java.py b/tests/test_java.py new file mode 100644 index 0000000..a1fa2a6 --- /dev/null +++ b/tests/test_java.py @@ -0,0 +1,158 @@ +"""Tests that run the bundled jar with a real ``java``. + +They are skipped when ``java`` is not on PATH, unless +C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1 is set (CI sets it); then they fail. +""" + +from pathlib import Path +from plone import api +from plone.app.contenttypes.indexers import SearchableText_file +from plone.namedfile.file import NamedBlobFile + +import logging +import os +import pytest +import shutil + + +SAMPLES = Path(__file__).parent / "samples" +REQUIRE_JAVA = "C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA" + +DOC = "application/msword" +DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +XLS = "application/vnd.ms-excel" +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +PPT = "application/vnd.ms-powerpoint" +PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation" + + +@pytest.fixture +def java(): + if shutil.which("java") is None: + if os.environ.get(REQUIRE_JAVA) == "1": + pytest.fail(f"java is not on PATH and {REQUIRE_JAVA}=1") + pytest.skip("java is not on PATH") + + +@pytest.fixture +def no_locale(monkeypatch): + """Run java without a locale, like a slim container image.""" + monkeypatch.setenv("LC_ALL", "C") + monkeypatch.delenv("LANG", raising=False) + + +def add_file(portal, name: str, content_type: str): + with api.env.adopt_roles(["Manager"]): + return api.content.create( + container=portal, + type="File", + id=name.replace(".", "-"), + title="Sample", + file=NamedBlobFile( + data=(SAMPLES / name).read_bytes(), + filename=name, + contentType=content_type, + ), + ) + + +def text_of(portal, name: str, content_type: str) -> str: + return SearchableText_file(add_file(portal, name, content_type))() + + +def transformed(portal, name: str, content_type: str) -> str: + data = portal.portal_transforms.convertTo( + "text/plain", (SAMPLES / name).read_bytes(), mimetype=content_type + ) + return data.getData() + + +@pytest.mark.usefixtures("java", "no_locale") +class TestRealJava: + def test_docx_words_are_searchable(self, portal): + text = text_of(portal, "document.docx", DOCX) + for word in ("simple", "example", "document", "metadata", "comments"): + assert word in text + + def test_japanese_is_not_garbled(self, portal): + assert "これは日本語の文書の例です。" in text_of(portal, "document.docx", DOCX) + + def test_docx_text(self, portal): + assert transformed(portal, "document.docx", DOCX) == ( + "This is a simple example of a document with a full set of metadata " + "and comments.\nこれは日本語の文書の例です。\n\n" + ) + + @pytest.mark.parametrize( + ("name", "content_type", "expected"), + [ + ( + "document.doc", + DOC, + [ + "This is a simple example of a document", + "これは日本語の文書の例です。", + ], + ), + ("spreadsheet.xls", XLS, ["Fruit\tCount", "Apple\t3", "みかん\t5"]), + ("spreadsheet.xlsx", XLSX, ["Fruit\tCount", "Apple\t3", "みかん\t5"]), + ( + "presentation.pptx", + PPTX, + ["Presentation slide example", "スライドの例です。"], + ), + ], + ) + def test_other_formats(self, portal, name, content_type, expected): + text = text_of(portal, name, content_type) + for part in expected: + assert part in text + + def test_document_without_text(self, portal, caplog): + with caplog.at_level(logging.INFO, logger="c2.transform.msoffice"): + assert transformed(portal, "empty.docx", DOCX) == "" + assert caplog.records == [] + + def test_ppt_gives_no_text_with_the_bundled_jar(self, portal, caplog): + """Known issue: the bundled jar cannot read .ppt files (see the xfail).""" + obj = add_file(portal, "presentation.ppt", PPT) + assert obj.file.data == (SAMPLES / "presentation.ppt").read_bytes() + # Creating the File indexes it once already. + caplog.clear() + with caplog.at_level(logging.ERROR, logger="c2.transform.msoffice"): + text = SearchableText_file(obj)() + assert "Presentation slide example" not in text + [record] = caplog.records + assert record.getMessage().startswith( + 'msdoc2txt failed with exit code 1: Exception in thread "main" ' + "java.lang.IllegalArgumentException: Error creating Scratchpad Extractor" + ) + + @pytest.mark.xfail( + strict=True, + reason=( + "The bundled msdoc2txt.jar (POI 3.17) fails on every .ppt: " + "CurrentUserAtom. -> 'Block 3 not found' / 'Unable to read 512 " + "bytes from 2048 in stream of length -1'. MSDocExtracter seems to close " + "the POIFS before HSLF reads the Current User stream. To be fixed when " + "the jar is updated." + ), + ) + def test_ppt_text_is_searchable(self, portal): + text = text_of(portal, "presentation.ppt", PPT) + assert "Presentation slide example" in text + assert "スライドの例です。" in text + + +def test_saving_works_without_java(portal, monkeypatch, tmp_path, caplog): + monkeypatch.setenv("PATH", str(tmp_path)) + obj = add_file(portal, "document.docx", DOCX) + assert obj.file.data == (SAMPLES / "document.docx").read_bytes() + # Creating the File indexes it once already. + caplog.clear() + with caplog.at_level(logging.ERROR, logger="c2.transform.msoffice"): + text = SearchableText_file(obj)() + assert "simple example" not in text + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt could not run: java was not found" + ] From 7cdbbaccd7c8062709ef90f904c1d278f7a9c6f5 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:04:47 +0900 Subject: [PATCH 05/13] Add a GitHub Actions workflow that runs the tox matrix with Java - lint: tox -e ruff-check on Python 3.12. - test: the tox matrix (Plone 6.0 and 6.1 on Python 3.10-3.13, Plone 6.2 on Python 3.10-3.14) with Temurin 21 from actions/setup-java. - C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1 turns a missing java into test failures, and a last step fails the job if the pytest output (-rs) reports any skipped test. - Actions are pinned to commit SHAs of releases at least two weeks old; permissions are contents: read. Checked with actionlint 1.7.12. The skip check was tried on a log with skipped tests (matches) and on a clean log (no match). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/plone-package.yml | 82 +++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/plone-package.yml diff --git a/.github/workflows/plone-package.yml b/.github/workflows/plone-package.yml new file mode 100644 index 0000000..d8a0496 --- /dev/null +++ b/.github/workflows/plone-package.yml @@ -0,0 +1,82 @@ +name: Plone package + +# Every action is pinned to a full commit SHA of a release that has been out +# for at least two weeks (cooldown). The comment after each SHA is the tag. + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Cancel superseded runs of the same PR only. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install tox + run: python -m pip install tox + - name: Ruff + run: tox -e ruff-check + + test: + runs-on: ubuntu-latest + needs: lint + strategy: + fail-fast: false + matrix: + include: + - {python-version: '3.10', tox-env: 'py310-Plone60'} + - {python-version: '3.11', tox-env: 'py311-Plone60'} + - {python-version: '3.12', tox-env: 'py312-Plone60'} + - {python-version: '3.13', tox-env: 'py313-Plone60'} + - {python-version: '3.10', tox-env: 'py310-Plone61'} + - {python-version: '3.11', tox-env: 'py311-Plone61'} + - {python-version: '3.12', tox-env: 'py312-Plone61'} + - {python-version: '3.13', tox-env: 'py313-Plone61'} + - {python-version: '3.10', tox-env: 'py310-Plone62'} + - {python-version: '3.11', tox-env: 'py311-Plone62'} + - {python-version: '3.12', tox-env: 'py312-Plone62'} + - {python-version: '3.13', tox-env: 'py313-Plone62'} + - {python-version: '3.14', tox-env: 'py314-Plone62'} + env: + # Fail instead of skipping the tests that need java. + C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + # The runner has several JDKs already; this pins the one on PATH. + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: temurin + java-version: '21' + - name: Install tox + run: python -m pip install tox + - name: Test with tox + run: | + set -o pipefail + tox -e ${{ matrix.tox-env }} | tee tox.log + - name: Check that no test was skipped + run: | + if grep -E '^SKIPPED|[0-9]+ skipped' tox.log; then + echo "::error::Some tests were skipped" + exit 1 + fi From 1b47e9d37a44d4ae3ff5f2cdc5233eea9851a6d7 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:07:12 +0900 Subject: [PATCH 06/13] Rewrite the README, convert the changelog and add AGENTS.md - README.md: what the package does, supported Plone, Python and Java versions, installation, upgrading from 1.0b6 (reinstall, PEP 420 and horse-with-no-namespace), how the jar is called, the environment variables, memory use (one JVM per Zope thread), the .ppt known issue, uninstall, credits and contributors. It absorbs docs/INSTALL.txt. - CHANGELOG.md: the old docs/HISTORY.txt in Markdown, below the towncrier start marker. The 2.0.0 changes are news fragments. - AGENTS.md: rules (English, no specs in the repository, news fragments), layout, how to run tox, and how the test samples were made. Checked: towncrier build --draft renders the fragments; uv build makes a wheel with msdoc2txt.jar and without tests/ or .github/; twine check passes for the wheel and the sdist. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 56 ++++++++++++++++ CHANGELOG.md | 83 +++++++++++------------- README.md | 134 ++++++++++++++++++++++++++++----------- news/+encoding.bugfix | 1 + news/+env.feature | 1 + news/+exitcode.bugfix | 1 + news/+pep420.breaking | 1 + news/+plone6.breaking | 1 + news/+ppt.documentation | 1 + news/+reinstall.breaking | 1 + news/+str-input.bugfix | 1 + news/+tooling.internal | 1 + news/+uninstall.feature | 1 + 13 files changed, 201 insertions(+), 82 deletions(-) create mode 100644 AGENTS.md create mode 100644 news/+encoding.bugfix create mode 100644 news/+env.feature create mode 100644 news/+exitcode.bugfix create mode 100644 news/+pep420.breaking create mode 100644 news/+plone6.breaking create mode 100644 news/+ppt.documentation create mode 100644 news/+reinstall.breaking create mode 100644 news/+str-input.bugfix create mode 100644 news/+tooling.internal create mode 100644 news/+uninstall.feature diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d58eae0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,56 @@ +# Notes for contributors and coding agents + +## Rules + +- This is a public package. Write code, comments, documentation, commit + messages and pull requests in English. +- Do not add design documents, specs or plans to the repository. +- Record user-facing changes as towncrier fragments in `news/` + (`news/+.` or `news/.`, types as in + `pyproject.toml`), not in `CHANGELOG.md`. + +## Layout + +- `src/c2/transform/msoffice/`: the package. `c2` and `c2.transform` are + PEP 420 namespace packages, so they have no `__init__.py`. + - `transform.py`: the `msoffice_to_text` transform that runs the jar. + - `config.py`: MIME types; importing it adds them to `mimetypes`. + - `setuphandlers.py`: install and uninstall (`post_handler`s). + - `msdoc2txt.jar`: the bundled Java program (Apache POI 3.17). +- `tests/`: pytest tests (not shipped in the wheel). + - `test_transform.py`, `test_config.py`: unit tests; `java` is replaced. + - `test_setup.py`: install and uninstall in a Plone site. + - `test_java.py`: runs the real jar on the files in `tests/samples/`. + +## Development + +Run the tests and the linters with tox: + +```shell +tox -e py314-Plone62 # one Python and Plone version +tox -e ruff-check # lint and format check +tox -e ruff-fix # fix and format +tox # the whole matrix +``` + +Extra arguments go to pytest, for example +`tox -e py314-Plone62 -- tests/test_transform.py`. + +`tests/test_java.py` needs `java` on `PATH`. Without it the tests are +skipped; set `C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1` to make them fail +instead, as CI does. Write expected values in tests as literals. + +## Test samples + +All files in `tests/samples/` were made for this package; do not add +documents of unknown origin. The `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx` +files were converted with LibreOffice in headless mode from short text and +CSV files (`soffice --headless --convert-to docx document.txt`, the CSV +with `--infilter="CSV:44,34,76,1"`, and the presentation by converting a PDF +of the text with `--infilter=impress_pdf_import`); `` was then +removed from `docProps/core.xml` of the OOXML files, because the jar prints +it as text. The `.ppt` file was written with the Apache POI classes in the +bundled jar (`HSLFSlideShow` with one text box), because it is much smaller +than the one LibreOffice writes. The jar cannot read either of them (see the +known issues in `README.md`). `empty.docx` is a converted empty +text file. diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e125a..f13235d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,74 +1,65 @@ -Changelog -========= +# Changelog + -1.0b6 (2018-04-02) ------------------------ + -* change from POI 3.6 to POI 3.17 with Java 8 [Tetsuya Morimoto] +## 1.0b6 (2018-04-02) +- change from POI 3.6 to POI 3.17 with Java 8 [Tetsuya Morimoto] -1.0b5 (2018-02-28) ------------------------ +## 1.0b5 (2018-02-28) -* modify setup.py for target Plone [terapyon] -* tested Plone 5.1 +- modify setup.py for target Plone [terapyon] +- tested Plone 5.1 -1.0b4 (2013-03-16) ------------------------ +## 1.0b4 (2013-03-16) -* modify kill process for Popen [Takayuki Nagai] +- modify kill process for Popen [Takayuki Nagai] -1.0b3 (2010-10-06) ---------------------- +## 1.0b3 (2010-10-06) -* adding terminated suspicious zombie process [terapyon] +- adding terminated suspicious zombie process [terapyon] -1.0b2 (2010-02-19) ---------------------- +## 1.0b2 (2010-02-19) -* remove the copyright in the files [terapyon] +- remove the copyright in the files [terapyon] -1.0b1 (2010-02-01) ---------------------- +## 1.0b1 (2010-02-01) -* modify getting from metadata and note in the msoffice file [terapyon] -* no support for Windows [terapyon] +- modify getting from metadata and note in the msoffice file [terapyon] +- no support for Windows [terapyon] +## 0.3.2 (2010-01-19) -0.3.2 (2010-01-19) ---------------------- +- setting some mimetypes [terapyon] -* setting some mimetypes [terapyon] +## 0.3.1 (2009-12-22) -0.3.1 (2009-12-22) ---------------------- +- bug fix for Windows [terapyon] -* bug fix for Windows [terapyon] +## 0.3 (2009-12-22) -0.3 (2009-12-22) ---------------------- +- support Windows OS -* support Windows OS +## 0.2 (2009-12-21) -0.2 (2009-12-21) ---------------------- +- support Plone 4.x [terapyon] -* support Plone 4.x [terapyon] +## 0.1.2 (2009-12-17) -0.1.2 (2009-12-17) ---------------------- +- modify typo -* modify typo +## 0.1.1 (2009-12-16) -0.1.1 (2009-12-16) ---------------------- +- modify exception code +- adding java args for big size file -* modify exception code -* adding java args for big size file +## 0.1 (2009-12-16) - -0.1 (2009-12-16) ------------------ - -* Initial release +- Initial release diff --git a/README.md b/README.md index 007e0e4..c9334ce 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,119 @@ -Introduction -============ +# c2.transform.msoffice -Description -------------- +Make Microsoft Office files searchable in Plone. -- This product enables to search MS-Office files on Plone. -- Include Apache POI 3.17 package by jar file. +The package adds a `portal_transforms` transform, `msoffice_to_text`, that +extracts plain text from Word, Excel and PowerPoint files. Plone uses it to +index the text of uploaded File objects, so a search finds words inside the +documents. - - http://poi.apache.org/ +Text is extracted by a bundled Java program (`msdoc2txt.jar`, built on +[Apache POI](https://poi.apache.org/) 3.17). -- Code repository +## Supported versions - - https://bitbucket.org/cmscom/c2.transform.msoffice +- Plone 6.0, 6.1 and 6.2 +- Python 3.10 to 3.14 (as far as each Plone version supports them) +- Java 8 or later on `PATH` (CI uses Temurin 21) +- Linux and other POSIX systems. Windows is not supported. -- Java code +## Supported files - - https://bitbucket.org/cmscom/msdoc2txt +- Word: `.doc`, `.docx`, `.docm` +- Excel: `.xls`, `.xlsx`, `.xlsm`, `.xlsb` +- PowerPoint: `.pptx`, `.pptm` (see the known issues for `.ppt`) +## Installation -Support files ----------------- -- MS-Office files +1. Install Java (a headless JRE is enough) so that `java` is on the `PATH` of + the Zope process. +2. Add `c2.transform.msoffice` to your project's dependencies, for example + with pip: - - MS-Word 95-2000, XP, 2007 and xml-format - - MS-Excel 95-2000, XP, 2007 and xml-format - - MS-PowerPoint 95-2000, XP, 2007 and xml-format + ```shell + pip install c2.transform.msoffice + ``` -Requirement -------------- +3. Restart Zope and activate **c2.transform.msoffice** in the add-ons control + panel (or apply the `c2.transform.msoffice:default` profile). -- Plone +Existing File objects get their text indexed the next time they are +reindexed, for example with "Clear and Rebuild" in the catalog. - - Plone 3.0-3.3 and Plone 4.x and 5.0 / 5.1 (tested by 5.1 CentOS 7 64bit) +### Upgrading from 1.0b6 -- Java +2.0 has no upgrade step from 1.0b6. After upgrading the package, deactivate +the add-on and activate it again in the add-ons control panel. This replaces +the transform that 1.0b6 registered. - - Java 8 +1.0b6 used `pkg_resources` namespace packages; 2.0 uses PEP 420 namespace +packages. If your site also uses other `c2.*` packages that still declare a +`pkg_resources` namespace, install +[horse-with-no-namespace](https://pypi.org/project/horse-with-no-namespace/) +so that both kinds can be imported together. -- OS +## How it works - - Linux(like Unix) - - no support for Windows +For each document, the transform starts `java` with the bundled jar, writes +the document to its standard input and reads the text from its standard +output: -Credits ---------- +```shell +java -Xmx512m -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar msdoc2txt.jar < document.docx +``` -This code was inspired by Products.OpenXml +- The text is always written in UTF-8, also when the process has no locale + (as in slim container images). +- The exit code decides success. A document without text gives empty text. +- If `java` cannot be run, exits with an error or takes too long, the text is + empty and an error is logged. Saving the file is not affected. +- Neither the document nor its text is written to the log. +## Configuration -Contributors ---------------- -- Manabu Terada(Twitter: @Terapyon) (Main code) -- Mikio Hokari (Java code & Extend call) -- Takashi Nagai (Support Windows) -- Takayuki Nagai (Bug fix for 1.0b4 tnagai@cc.kumamoto-u.ac.jp) -- Tetsuya Morimoto (Refactoring Java code and Support POI 3.17) \ No newline at end of file +Set these environment variables for the Zope process: + +- `C2_TRANSFORM_MSOFFICE_TIMEOUT`: seconds to wait for `java` before it is + killed. Default `60`. +- `C2_TRANSFORM_MSOFFICE_JAVA_OPTS`: JVM options, split like a shell command + line. They replace the default `-Xmx512m`, for example + `-Xmx1g -XX:+UseSerialGC`. The encoding options are always added. + +## Memory + +Each transform runs its own JVM. Up to one JVM per Zope thread can run at the +same time, each with a heap of up to 512 MB (by default) plus the memory of +the JVM itself. Plan the memory of the server or container for that. + +## Known issues + +- The bundled jar cannot extract text from `.ppt` files (PowerPoint 97-2003). + The transform runs, logs an error and gives empty text. `.pptx` files work. + This will be fixed when the jar is updated. + +## Uninstall + +Deactivate the add-on in the add-ons control panel. This removes the +transform. The MIME types are kept, because other add-ons may use them. + +## Source code + +- This package: +- Java program: + +## Credits + +This code was inspired by Products.OpenXml. + +## Contributors + +- Manabu Terada (main code) +- Mikio Hokari (Java code and extended calls) +- Takashi Nagai (Windows support) +- Takayuki Nagai (bug fix for 1.0b4) +- Tetsuya Morimoto (refactoring of the Java code and POI 3.17 support) + +## License + +GNU General Public License, version 2 or later. See `LICENSE.txt` and +`LICENSE.GPL`. diff --git a/news/+encoding.bugfix b/news/+encoding.bugfix new file mode 100644 index 0000000..7638d81 --- /dev/null +++ b/news/+encoding.bugfix @@ -0,0 +1 @@ +Always pass `-Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8` to `java`, so non-ASCII text is not replaced by `?` when the process has no locale (for example in slim container images). [terapyon] diff --git a/news/+env.feature b/news/+env.feature new file mode 100644 index 0000000..9d3e2b6 --- /dev/null +++ b/news/+env.feature @@ -0,0 +1 @@ +Add the `C2_TRANSFORM_MSOFFICE_TIMEOUT` and `C2_TRANSFORM_MSOFFICE_JAVA_OPTS` environment variables for the timeout and the JVM options. [terapyon] diff --git a/news/+exitcode.bugfix b/news/+exitcode.bugfix new file mode 100644 index 0000000..6d99afa --- /dev/null +++ b/news/+exitcode.bugfix @@ -0,0 +1 @@ +Decide success by the exit code of `java`. Text is no longer thrown away when the jar writes a warning to stderr, and a document without text no longer logs an error. [terapyon] diff --git a/news/+pep420.breaking b/news/+pep420.breaking new file mode 100644 index 0000000..89752c6 --- /dev/null +++ b/news/+pep420.breaking @@ -0,0 +1 @@ +`c2` and `c2.transform` are now PEP 420 namespace packages. Sites that also use `c2.*` packages with `pkg_resources` namespaces need `horse-with-no-namespace`. [terapyon] diff --git a/news/+plone6.breaking b/news/+plone6.breaking new file mode 100644 index 0000000..836012b --- /dev/null +++ b/news/+plone6.breaking @@ -0,0 +1 @@ +Support Plone 6.0, 6.1 and 6.2 on Python 3.10 to 3.14 only. Python 2, Plone 3 to 5 and the Windows code are removed. [terapyon] diff --git a/news/+ppt.documentation b/news/+ppt.documentation new file mode 100644 index 0000000..c7a8861 --- /dev/null +++ b/news/+ppt.documentation @@ -0,0 +1 @@ +Document a known issue: the bundled jar cannot extract text from `.ppt` files (this was already the case in 1.0b6). [terapyon] diff --git a/news/+reinstall.breaking b/news/+reinstall.breaking new file mode 100644 index 0000000..c711bb8 --- /dev/null +++ b/news/+reinstall.breaking @@ -0,0 +1 @@ +The profile version is now `2000` and there is no upgrade step from 1.0b6: deactivate and activate the add-on again after upgrading. [terapyon] diff --git a/news/+str-input.bugfix b/news/+str-input.bugfix new file mode 100644 index 0000000..e1b6db3 --- /dev/null +++ b/news/+str-input.bugfix @@ -0,0 +1 @@ +Log text input and give empty text instead of failing an `assert`. [terapyon] diff --git a/news/+tooling.internal b/news/+tooling.internal new file mode 100644 index 0000000..0c88c78 --- /dev/null +++ b/news/+tooling.internal @@ -0,0 +1 @@ +Move to `pyproject.toml` and a `src/` layout, with tox, pytest, ruff, towncrier, zest.releaser and a GitHub Actions workflow that runs the tests with Java. Replace the test sample with self-made documents. [terapyon] diff --git a/news/+uninstall.feature b/news/+uninstall.feature new file mode 100644 index 0000000..a6b42c4 --- /dev/null +++ b/news/+uninstall.feature @@ -0,0 +1 @@ +Add an uninstall profile that removes the transform, and register `application/vnd.ms-powerpoint` as an input explicitly. [terapyon] From a4b9ad464348cde9df9df40674969c7e3e1b1232 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:19:46 +0900 Subject: [PATCH 07/13] Load the test layer in the INonInstallable test test_uninstall_profile_is_hidden used no fixture, so the ZCML of the package was loaded only if another test had set up the layer before it. On Plone 6.1 it ran without the layer and failed with ComponentLookupError. It now requests the portal fixture. RED: py310-py313 with Plone61 failed with ComponentLookupError for INonInstallable 'c2.transform.msoffice'. Co-Authored-By: Claude Opus 5.5 --- tests/test_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_setup.py b/tests/test_setup.py index 6860db2..0642bff 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -63,7 +63,7 @@ def test_install_twice(transforms, setup_tool, caplog): ) in caplog.record_tuples -def test_uninstall_profile_is_hidden(): +def test_uninstall_profile_is_hidden(portal): hidden = getUtility(INonInstallable, name=PACKAGE_NAME) assert hidden.getNonInstallableProfiles() == ["c2.transform.msoffice:uninstall"] From 2c5b39a4d499d62062e72e4b5b163d5e717f9a21 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:38:27 +0900 Subject: [PATCH 08/13] Log unparsable C2_TRANSFORM_MSOFFICE_JAVA_OPTS separately shlex.split raises ValueError for options such as an unclosed quote. It was logged as "msdoc2txt could not run" with a traceback; it is now logged as "JVM options in C2_TRANSFORM_MSOFFICE_JAVA_OPTS could not be parsed" and java is not started. The text is empty, as for other failures. RED: test_java_opts_that_cannot_be_parsed got ['msdoc2txt could not run']. Co-Authored-By: Claude Opus 5.5 --- src/c2/transform/msoffice/transform.py | 8 +++++++- tests/test_transform.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/c2/transform/msoffice/transform.py b/src/c2/transform/msoffice/transform.py index c81ba7d..18087e0 100644 --- a/src/c2/transform/msoffice/transform.py +++ b/src/c2/transform/msoffice/transform.py @@ -55,9 +55,15 @@ def msdoc2txt(data: bytes) -> str | None: document without text. Neither the document nor the text is logged. """ seconds = timeout() + try: + command = java_command() + except ValueError: + # shlex could not split the options, for example an unclosed quote. + logger.error("JVM options in %s could not be parsed", JAVA_OPTS_ENV) + return None try: result = subprocess.run( # noqa: S603 - java_command(), input=data, capture_output=True, timeout=seconds + command, input=data, capture_output=True, timeout=seconds ) except subprocess.TimeoutExpired: # subprocess.run kills the process and waits for it. diff --git a/tests/test_transform.py b/tests/test_transform.py index 05ae54b..cb116ec 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -218,3 +218,14 @@ def test_blank_java_opts_use_default(monkeypatch): run = fake_run(monkeypatch, stdout=b"x") convert(DOCUMENT) assert run.calls[0][0][1] == "-Xmx512m" + + +def test_java_opts_that_cannot_be_parsed(monkeypatch, caplog): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", '-Xmx1g "-Dunclosed') + run = fake_run(monkeypatch, stdout=b"x") + assert convert(DOCUMENT) == "" + assert run.calls == [] + assert [r.getMessage() for r in caplog.records] == [ + "JVM options in C2_TRANSFORM_MSOFFICE_JAVA_OPTS could not be parsed" + ] + assert caplog.records[0].exc_info is None From ca5f021dec6935fd12a50c359e6080ccd265ba7e Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:38:46 +0900 Subject: [PATCH 09/13] Add plone.api to the test extra tests/test_java.py imports plone.api. It was only installed because another test dependency pulls it in. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4d32c39..7114952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ [project.optional-dependencies] test = [ "horse-with-no-namespace", + "plone.api", "plone.app.contenttypes", "plone.app.testing", "pytest", From 2297495ec524adbae8338d33d88e054161043dfd Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 19:39:20 +0900 Subject: [PATCH 10/13] List the accepted MIME types and the tested formats in the README "Supported files" listed every extension, although only .docx, .doc, .xlsx, .xls and .pptx are covered by the tests and .ppt gives no text. The README now lists the MIME types the transform is registered for, and the tested formats separately. AGENTS.md now says that the last editor name in the .ppt sample's Current User stream was cleared, and to check new samples for names. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 14 +++++++++----- README.md | 27 ++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d58eae0..2bf8ced 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,8 +49,12 @@ CSV files (`soffice --headless --convert-to docx document.txt`, the CSV with `--infilter="CSV:44,34,76,1"`, and the presentation by converting a PDF of the text with `--infilter=impress_pdf_import`); `` was then removed from `docProps/core.xml` of the OOXML files, because the jar prints -it as text. The `.ppt` file was written with the Apache POI classes in the -bundled jar (`HSLFSlideShow` with one text box), because it is much smaller -than the one LibreOffice writes. The jar cannot read either of them (see the -known issues in `README.md`). `empty.docx` is a converted empty -text file. +it as text. `empty.docx` is a converted empty text file. + +The `.ppt` file was written with the Apache POI classes in the bundled jar +(`HSLFSlideShow` with one text box), because it is much smaller than the one +LibreOffice writes; the jar cannot read either of them (see the known issues +in `README.md`). POI's empty template names a person as the last editor in +the Current User stream, so the name was cleared with +`getSlideShowImpl().getCurrentUserAtom().setLastEditUsername("")`. Check new +samples for names in their metadata before adding them. diff --git a/README.md b/README.md index c9334ce..e6ccd0d 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,28 @@ Text is extracted by a bundled Java program (`msdoc2txt.jar`, built on - Java 8 or later on `PATH` (CI uses Temurin 21) - Linux and other POSIX systems. Windows is not supported. -## Supported files - -- Word: `.doc`, `.docx`, `.docm` -- Excel: `.xls`, `.xlsx`, `.xlsm`, `.xlsb` -- PowerPoint: `.pptx`, `.pptm` (see the known issues for `.ppt`) +## Accepted MIME types + +The transform is registered for these MIME types: + +- Word: `application/msword`, + `application/vnd.openxmlformats-officedocument.wordprocessingml.document`, + `application/vnd.ms-word.document.macroEnabled.12` +- Excel: `application/vnd.ms-excel`, `application/msexcel`, + `application/x-msexcel`, + `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`, + `application/vnd.ms-excel.sheet.binary.macroEnabled.12`, + `application/vnd.ms-excel.sheet.macroEnabled.12` +- PowerPoint: `application/vnd.ms-powerpoint`, `application/powerpoint`, + `application/mspowerpoint`, `application/x-mspowerpoint`, + `application/vnd.openxmlformats-officedocument.presentationml.presentation`, + `application/vnd.ms-powerpoint.presentation.macroEnabled.12` + +### Tested formats + +The tests extract text from `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx` +files. `.ppt` files give no text (see the known issues). The other formats +are passed to the jar but are not covered by the tests. ## Installation From bb3b712481e9ed889a4a54d5e9a252a379f1492e Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 22:27:17 +0900 Subject: [PATCH 11/13] Do not log the stderr of java A failed run of the jar logged the first 500 characters of its stderr. A Java exception can quote parts of the document, such as the name of a part, so the log now has only the exit code. STDERR_LOG_LIMIT is gone. test_stderr_is_not_logged puts a part name into stderr and checks that it is not in the log, also at DEBUG level. RED: test_non_zero_exit_code, test_stderr_is_not_logged and test_ppt_gives_no_text_with_the_bundled_jar failed with the stack trace in the message. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- news/+exitcode.bugfix | 2 +- src/c2/transform/msoffice/transform.py | 12 +++--------- tests/test_java.py | 8 +++----- tests/test_transform.py | 22 +++++++++++++--------- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index e6ccd0d..1eeb989 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,9 @@ java -Xmx512m -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar msdoc2txt.jar < - The exit code decides success. A document without text gives empty text. - If `java` cannot be run, exits with an error or takes too long, the text is empty and an error is logged. Saving the file is not affected. -- Neither the document nor its text is written to the log. +- Neither the document, its text nor the error output of `java` is written + to the log, because a Java exception can quote parts of the document. A + failed run logs only its exit code. ## Configuration diff --git a/news/+exitcode.bugfix b/news/+exitcode.bugfix index 6d99afa..cc40f2a 100644 --- a/news/+exitcode.bugfix +++ b/news/+exitcode.bugfix @@ -1 +1 @@ -Decide success by the exit code of `java`. Text is no longer thrown away when the jar writes a warning to stderr, and a document without text no longer logs an error. [terapyon] +Decide success by the exit code of `java`. Text is no longer thrown away when the jar writes a warning to stderr, and a document without text no longer logs an error. A failed run logs its exit code but not the error output of `java`, which can quote parts of the document. [terapyon] diff --git a/src/c2/transform/msoffice/transform.py b/src/c2/transform/msoffice/transform.py index 18087e0..c9d2b79 100644 --- a/src/c2/transform/msoffice/transform.py +++ b/src/c2/transform/msoffice/transform.py @@ -22,8 +22,6 @@ # file.encoding and JDK 19+ from stdout.encoding, so both are always set. # They come after the options from the environment so they always win. ENCODING_OPTS = ("-Dfile.encoding=UTF-8", "-Dstdout.encoding=UTF-8") -# How much of the jar's stderr (a Java stack trace) goes into the log. -STDERR_LOG_LIMIT = 500 def timeout() -> int: @@ -52,7 +50,8 @@ def msdoc2txt(data: bytes) -> str | None: The document goes to the jar on stdin and the text comes back on stdout. The exit code decides success: an empty stdout with exit code 0 is a - document without text. Neither the document nor the text is logged. + document without text. Neither the document, the text nor the jar's + stderr is logged: a Java exception can quote parts of the document. """ seconds = timeout() try: @@ -76,12 +75,7 @@ def msdoc2txt(data: bytes) -> str | None: logger.exception("msdoc2txt could not run") return None if result.returncode != 0: - stderr = result.stderr.decode("utf-8", errors="replace") - logger.error( - "msdoc2txt failed with exit code %d: %s", - result.returncode, - stderr[:STDERR_LOG_LIMIT], - ) + logger.error("msdoc2txt failed with exit code %d", result.returncode) return None if not result.stdout: logger.debug("msdoc2txt returned no text") diff --git a/tests/test_java.py b/tests/test_java.py index a1fa2a6..6bd1a76 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -122,11 +122,9 @@ def test_ppt_gives_no_text_with_the_bundled_jar(self, portal, caplog): with caplog.at_level(logging.ERROR, logger="c2.transform.msoffice"): text = SearchableText_file(obj)() assert "Presentation slide example" not in text - [record] = caplog.records - assert record.getMessage().startswith( - 'msdoc2txt failed with exit code 1: Exception in thread "main" ' - "java.lang.IllegalArgumentException: Error creating Scratchpad Extractor" - ) + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt failed with exit code 1" + ] @pytest.mark.xfail( strict=True, diff --git a/tests/test_transform.py b/tests/test_transform.py index cb116ec..a877664 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -120,17 +120,21 @@ def test_stdout_that_is_not_utf8_does_not_raise(monkeypatch): def test_non_zero_exit_code(monkeypatch, caplog): - stderr = b"java.lang.IllegalArgumentException: UNKNOWN is not supported" + ( - b"x" * 2000 - ) + stderr = b"java.lang.IllegalArgumentException: UNKNOWN is not supported" fake_run(monkeypatch, returncode=1, stdout=b"partial", stderr=stderr) assert convert(DOCUMENT) == "" - assert [r.levelname for r in caplog.records] == ["ERROR"] - message = caplog.records[0].getMessage() - assert message == ( - "msdoc2txt failed with exit code 1: " - "java.lang.IllegalArgumentException: UNKNOWN is not supported" + "x" * 440 - ) + assert [(r.levelname, r.getMessage()) for r in caplog.records] == [ + ("ERROR", "msdoc2txt failed with exit code 1") + ] + + +def test_stderr_is_not_logged(monkeypatch, caplog): + """A Java exception can quote the document, for example a part name.""" + stderr = b"org.apache.poi.POIXMLException: /word/secret-part-name.xml" + fake_run(monkeypatch, returncode=1, stderr=stderr) + with caplog.at_level(logging.DEBUG, logger="c2.transform.msoffice"): + assert convert(DOCUMENT) == "" + assert "secret-part-name" not in caplog.text assert "secret document body" not in caplog.text From c8f7560bfc030dc5e19d231cc54888cf5c486a50 Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 22:34:45 +0900 Subject: [PATCH 12/13] Remove the transform also when unmapping it fails TransformEngine.unregisterTransform unmaps the transform before it deletes it. When a MIME type mapping was missing, _unmapTransform raised KeyError, uninstall only logged a warning and the transform stayed, so installing again skipped it as already registered. On KeyError the transform is now deleted with _delObject and suppress_events=True: Transform.manage_beforeDelete would unmap again, and OFS logs that KeyError as an error and re-raises it in debug mode for users without the Manager role. The tests delete the DOCX -> text/plain mapping of a real site instead of replacing unregisterTransform: uninstall removes the transform and logs one warning, and installing again maps it. RED: test_missing_mapping_still_removes_the_transform (transform still in objectIds) and test_install_after_a_missing_mapping_adds_the_transform (KeyError: install was skipped). Mutations caught: no deletion (2 failed), suppress_events=False (1 failed: "_delObject() threw" logged). Co-Authored-By: Claude Opus 5.5 --- src/c2/transform/msoffice/setuphandlers.py | 9 +++++-- tests/test_setup.py | 31 ++++++++++++++-------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/c2/transform/msoffice/setuphandlers.py b/src/c2/transform/msoffice/setuphandlers.py index 9f7ed5a..8bef927 100644 --- a/src/c2/transform/msoffice/setuphandlers.py +++ b/src/c2/transform/msoffice/setuphandlers.py @@ -56,9 +56,14 @@ def uninstall(context): transforms.unregisterTransform(config.TRANSFORM_NAME) except KeyError as error: # unregisterTransform unmaps first and raises KeyError when a MIME - # type was never mapped to this transform. + # type was never mapped to this transform; it has not deleted the + # transform yet. Delete it, so that installing again adds it anew. + # Without events, because Transform.manage_beforeDelete unmaps again + # and OFS re-raises that KeyError in debug mode for non-Managers. + transforms._delObject(config.TRANSFORM_NAME, suppress_events=True) logger.warning( - "Could not unregister %s from portal_transforms: %r", + "Could not unmap %s in portal_transforms: %r; " + "the transform was removed anyway", config.TRANSFORM_NAME, error, ) diff --git a/tests/test_setup.py b/tests/test_setup.py index 0642bff..f6394c7 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -86,21 +86,30 @@ def test_uninstall_twice(self, transforms, setup_tool): setup_tool.runAllImportStepsFromProfile(UNINSTALL_PROFILE) assert TRANSFORM_NAME not in transforms.objectIds() - def test_unmapping_error_is_logged_and_uninstall_continues( - self, installer, transforms, setup_tool, monkeypatch, caplog + def test_missing_mapping_still_removes_the_transform( + self, installer, transforms, caplog ): installer.install_product(PACKAGE_NAME) - - def broken_unregister(self, name): - raise KeyError("text/plain") - - monkeypatch.setattr( - type(aq_base(transforms)), "unregisterTransform", broken_unregister - ) + # A site where the DOCX -> text/plain mapping got lost: + # unregisterTransform raises KeyError before it deletes the transform. + del transforms._mtmap[DOCX]["text/plain"] with caplog.at_level(logging.WARNING, logger=PACKAGE_NAME): installer.uninstall_product(PACKAGE_NAME) assert installer.is_product_installed(PACKAGE_NAME) is False + assert TRANSFORM_NAME not in transforms.objectIds() assert [r.getMessage() for r in caplog.records] == [ - "Could not unregister msoffice_to_text from portal_transforms: " - "KeyError('text/plain')" + "Could not unmap msoffice_to_text in portal_transforms: " + "KeyError('text/plain'); the transform was removed anyway" + ] + + def test_install_after_a_missing_mapping_adds_the_transform( + self, installer, transforms + ): + installer.install_product(PACKAGE_NAME) + del transforms._mtmap[DOCX]["text/plain"] + installer.uninstall_product(PACKAGE_NAME) + installer.install_product(PACKAGE_NAME) + assert TRANSFORM_NAME in transforms.objectIds() + assert [t.name() for t in transforms._mtmap[DOCX]["text/plain"]] == [ + "msoffice_to_text" ] From b722d1095b5945dfa51b9aa29c2d13dacef2f22b Mon Sep 17 00:00:00 2001 From: Manabu TERADA Date: Mon, 19 Oct 2026 22:40:44 +0900 Subject: [PATCH 13/13] Fall back to the default timeout when int() rejects the value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C2_TRANSFORM_MSOFFICE_TIMEOUT was checked with str.isdigit(), which accepts "²", and int("²") then raised ValueError out of the transform. The value is now parsed with int() directly; a ValueError or a value that is not positive logs the warning and uses the default. RED: test_invalid_timeout_falls_back_to_default["²"] failed with "ValueError: invalid literal for int() with base 10: '²'". Mutation caught: "seconds >= 0" (4 failed). Co-Authored-By: Claude Opus 5.5 --- src/c2/transform/msoffice/transform.py | 9 +++++++-- tests/test_transform.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/c2/transform/msoffice/transform.py b/src/c2/transform/msoffice/transform.py index c9d2b79..0c5ecd0 100644 --- a/src/c2/transform/msoffice/transform.py +++ b/src/c2/transform/msoffice/transform.py @@ -29,8 +29,13 @@ def timeout() -> int: value = os.environ.get(TIMEOUT_ENV, "").strip() if not value: return DEFAULT_TIMEOUT - if value.isdigit() and int(value) > 0: - return int(value) + try: + seconds = int(value) + except ValueError: + # Not only "abc": str.isdigit() accepts "²", which int() rejects. + seconds = 0 + if seconds > 0: + return seconds logger.warning( "Ignoring %s=%r: expected a positive integer; using %d", TIMEOUT_ENV, diff --git a/tests/test_transform.py b/tests/test_transform.py index a877664..885fafe 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -190,7 +190,7 @@ def test_timeout_from_environment(monkeypatch, value, expected): assert run.calls[0][1]["timeout"] == expected -@pytest.mark.parametrize("value", ["abc", "0", "-3", "1.5"]) +@pytest.mark.parametrize("value", ["abc", "0", "-3", "1.5", "²"]) def test_invalid_timeout_falls_back_to_default(monkeypatch, caplog, value): monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", value) run = fake_run(monkeypatch, stdout=b"x")