diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b3561a7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,35 @@ + +# EditorConfig Configuration file, for more details see: +# https://EditorConfig.org +# EditorConfig is a convention description, that could be interpreted +# by multiple editors to enforce common coding conventions for specific +# file types + +# top-most EditorConfig file: +# Will ignore other EditorConfig files in Home directory or upper tree level. +root = true + + +[*] # For All Files +# Unix-style newlines with a newline ending every file +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +# Set default charset +charset = utf-8 +# Indent style default +indent_style = space + +[*.{py,cfg,ini}] +# 4 space indentation +indent_size = 4 + +[*.{html,dtml,pt,zpt,xml,zcml,js,json,ts,less,scss,css,sass,yml,yaml}] +# 2 space indentation +indent_size = 2 + +[{Makefile,.gitmodules}] +# Tab indentation (no size specified, but view as 4 spaces) +indent_style = tab +indent_size = unset +tab_width = unset diff --git a/.github/workflows/plone-package.yml b/.github/workflows/plone-package.yml new file mode 100644 index 0000000..d8a0496 --- /dev/null +++ b/.github/workflows/plone-package.yml @@ -0,0 +1,82 @@ +name: Plone package + +# Every action is pinned to a full commit SHA of a release that has been out +# for at least two weeks (cooldown). The comment after each SHA is the tag. + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Cancel superseded runs of the same PR only. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install tox + run: python -m pip install tox + - name: Ruff + run: tox -e ruff-check + + test: + runs-on: ubuntu-latest + needs: lint + strategy: + fail-fast: false + matrix: + include: + - {python-version: '3.10', tox-env: 'py310-Plone60'} + - {python-version: '3.11', tox-env: 'py311-Plone60'} + - {python-version: '3.12', tox-env: 'py312-Plone60'} + - {python-version: '3.13', tox-env: 'py313-Plone60'} + - {python-version: '3.10', tox-env: 'py310-Plone61'} + - {python-version: '3.11', tox-env: 'py311-Plone61'} + - {python-version: '3.12', tox-env: 'py312-Plone61'} + - {python-version: '3.13', tox-env: 'py313-Plone61'} + - {python-version: '3.10', tox-env: 'py310-Plone62'} + - {python-version: '3.11', tox-env: 'py311-Plone62'} + - {python-version: '3.12', tox-env: 'py312-Plone62'} + - {python-version: '3.13', tox-env: 'py313-Plone62'} + - {python-version: '3.14', tox-env: 'py314-Plone62'} + env: + # Fail instead of skipping the tests that need java. + C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + # The runner has several JDKs already; this pins the one on PATH. + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: temurin + java-version: '21' + - name: Install tox + run: python -m pip install tox + - name: Test with tox + run: | + set -o pipefail + tox -e ${{ matrix.tox-env }} | tee tox.log + - name: Check that no test was skipped + run: | + if grep -E '^SKIPPED|[0-9]+ skipped' tox.log; then + echo "::error::Some tests were skipped" + exit 1 + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..461e060 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# python related +*.egg-info +*.pyc +*.pyo + +# tools related +__pycache__/ +.coverage +.coverage.* +.mypy_cache +.pytest_cache +.ruff_cache +/build/ +coverage.xml +dist/ +docs/_build +node_modules/ + +# venv / buildout related +.eggs/ +.installed.cfg +.mr.developer.cfg +.venv/ +bin/ +develop-eggs/ +eggs/ +etc/ +include/ +lib/ +lib64 +parts/ +pyvenv.cfg +var/ + +# mxdev +.installed.txt +.lock +/.mxdev_cache/ +/.make-sentinels/ +/*-mxdev.txt +/instance/ +/reports/ +/sources/ +/venv/ +# mxdev generated +constraints.txt +constraints-mxdev.txt +requirements.txt +requirements-mxdev.txt + +.tox/ diff --git a/.hgignore b/.hgignore deleted file mode 100644 index ddf4120..0000000 --- a/.hgignore +++ /dev/null @@ -1,18 +0,0 @@ -syntax: glob -*~ -*.pyc -*.pyo -dist/* -build/* -c2.*.*.egg-info/* -.semantic.cache -.DS_Store -TAGS -.DS_Store -.svn -.hg -.hgsvn -.as2proj -.as3proj -Thumbs.db -.~lock* diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2bf8ced --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,60 @@ +# Notes for contributors and coding agents + +## Rules + +- This is a public package. Write code, comments, documentation, commit + messages and pull requests in English. +- Do not add design documents, specs or plans to the repository. +- Record user-facing changes as towncrier fragments in `news/` + (`news/+.` or `news/.`, types as in + `pyproject.toml`), not in `CHANGELOG.md`. + +## Layout + +- `src/c2/transform/msoffice/`: the package. `c2` and `c2.transform` are + PEP 420 namespace packages, so they have no `__init__.py`. + - `transform.py`: the `msoffice_to_text` transform that runs the jar. + - `config.py`: MIME types; importing it adds them to `mimetypes`. + - `setuphandlers.py`: install and uninstall (`post_handler`s). + - `msdoc2txt.jar`: the bundled Java program (Apache POI 3.17). +- `tests/`: pytest tests (not shipped in the wheel). + - `test_transform.py`, `test_config.py`: unit tests; `java` is replaced. + - `test_setup.py`: install and uninstall in a Plone site. + - `test_java.py`: runs the real jar on the files in `tests/samples/`. + +## Development + +Run the tests and the linters with tox: + +```shell +tox -e py314-Plone62 # one Python and Plone version +tox -e ruff-check # lint and format check +tox -e ruff-fix # fix and format +tox # the whole matrix +``` + +Extra arguments go to pytest, for example +`tox -e py314-Plone62 -- tests/test_transform.py`. + +`tests/test_java.py` needs `java` on `PATH`. Without it the tests are +skipped; set `C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1` to make them fail +instead, as CI does. Write expected values in tests as literals. + +## Test samples + +All files in `tests/samples/` were made for this package; do not add +documents of unknown origin. The `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx` +files were converted with LibreOffice in headless mode from short text and +CSV files (`soffice --headless --convert-to docx document.txt`, the CSV +with `--infilter="CSV:44,34,76,1"`, and the presentation by converting a PDF +of the text with `--infilter=impress_pdf_import`); `` was then +removed from `docProps/core.xml` of the OOXML files, because the jar prints +it as text. `empty.docx` is a converted empty text file. + +The `.ppt` file was written with the Apache POI classes in the bundled jar +(`HSLFSlideShow` with one text box), because it is much smaller than the one +LibreOffice writes; the jar cannot read either of them (see the known issues +in `README.md`). POI's empty template names a person as the last editor in +the Current User stream, so the name was cleared with +`getSlideShowImpl().getCurrentUserAtom().setLastEditUsername("")`. Check new +samples for names in their metadata before adding them. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f13235d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + + + + + +## 1.0b6 (2018-04-02) + +- change from POI 3.6 to POI 3.17 with Java 8 [Tetsuya Morimoto] + +## 1.0b5 (2018-02-28) + +- modify setup.py for target Plone [terapyon] +- tested Plone 5.1 + +## 1.0b4 (2013-03-16) + +- modify kill process for Popen [Takayuki Nagai] + +## 1.0b3 (2010-10-06) + +- adding terminated suspicious zombie process [terapyon] + +## 1.0b2 (2010-02-19) + +- remove the copyright in the files [terapyon] + +## 1.0b1 (2010-02-01) + +- modify getting from metadata and note in the msoffice file [terapyon] +- no support for Windows [terapyon] + +## 0.3.2 (2010-01-19) + +- setting some mimetypes [terapyon] + +## 0.3.1 (2009-12-22) + +- bug fix for Windows [terapyon] + +## 0.3 (2009-12-22) + +- support Windows OS + +## 0.2 (2009-12-21) + +- support Plone 4.x [terapyon] + +## 0.1.2 (2009-12-17) + +- modify typo + +## 0.1.1 (2009-12-16) + +- modify exception code +- adding java args for big size file + +## 0.1 (2009-12-16) + +- Initial release diff --git a/docs/LICENSE.GPL b/LICENSE.GPL similarity index 100% rename from docs/LICENSE.GPL rename to LICENSE.GPL diff --git a/docs/LICENSE.txt b/LICENSE.txt similarity index 100% rename from docs/LICENSE.txt rename to LICENSE.txt diff --git a/MANIFEST.in b/MANIFEST.in index 0355823..09b5ed8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,8 @@ -recursive-include c2 * -recursive-include docs * -global-exclude *pyc *pyo +graft src/c2 +include *.md +include pyproject.toml +exclude tox.ini +exclude *.txt +global-exclude *.pyc +include LICENSE.GPL +include LICENSE.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..1eeb989 --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# c2.transform.msoffice + +Make Microsoft Office files searchable in Plone. + +The package adds a `portal_transforms` transform, `msoffice_to_text`, that +extracts plain text from Word, Excel and PowerPoint files. Plone uses it to +index the text of uploaded File objects, so a search finds words inside the +documents. + +Text is extracted by a bundled Java program (`msdoc2txt.jar`, built on +[Apache POI](https://poi.apache.org/) 3.17). + +## Supported versions + +- Plone 6.0, 6.1 and 6.2 +- Python 3.10 to 3.14 (as far as each Plone version supports them) +- Java 8 or later on `PATH` (CI uses Temurin 21) +- Linux and other POSIX systems. Windows is not supported. + +## Accepted MIME types + +The transform is registered for these MIME types: + +- Word: `application/msword`, + `application/vnd.openxmlformats-officedocument.wordprocessingml.document`, + `application/vnd.ms-word.document.macroEnabled.12` +- Excel: `application/vnd.ms-excel`, `application/msexcel`, + `application/x-msexcel`, + `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`, + `application/vnd.ms-excel.sheet.binary.macroEnabled.12`, + `application/vnd.ms-excel.sheet.macroEnabled.12` +- PowerPoint: `application/vnd.ms-powerpoint`, `application/powerpoint`, + `application/mspowerpoint`, `application/x-mspowerpoint`, + `application/vnd.openxmlformats-officedocument.presentationml.presentation`, + `application/vnd.ms-powerpoint.presentation.macroEnabled.12` + +### Tested formats + +The tests extract text from `.docx`, `.doc`, `.xlsx`, `.xls` and `.pptx` +files. `.ppt` files give no text (see the known issues). The other formats +are passed to the jar but are not covered by the tests. + +## Installation + +1. Install Java (a headless JRE is enough) so that `java` is on the `PATH` of + the Zope process. +2. Add `c2.transform.msoffice` to your project's dependencies, for example + with pip: + + ```shell + pip install c2.transform.msoffice + ``` + +3. Restart Zope and activate **c2.transform.msoffice** in the add-ons control + panel (or apply the `c2.transform.msoffice:default` profile). + +Existing File objects get their text indexed the next time they are +reindexed, for example with "Clear and Rebuild" in the catalog. + +### Upgrading from 1.0b6 + +2.0 has no upgrade step from 1.0b6. After upgrading the package, deactivate +the add-on and activate it again in the add-ons control panel. This replaces +the transform that 1.0b6 registered. + +1.0b6 used `pkg_resources` namespace packages; 2.0 uses PEP 420 namespace +packages. If your site also uses other `c2.*` packages that still declare a +`pkg_resources` namespace, install +[horse-with-no-namespace](https://pypi.org/project/horse-with-no-namespace/) +so that both kinds can be imported together. + +## How it works + +For each document, the transform starts `java` with the bundled jar, writes +the document to its standard input and reads the text from its standard +output: + +```shell +java -Xmx512m -Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8 -jar msdoc2txt.jar < document.docx +``` + +- The text is always written in UTF-8, also when the process has no locale + (as in slim container images). +- The exit code decides success. A document without text gives empty text. +- If `java` cannot be run, exits with an error or takes too long, the text is + empty and an error is logged. Saving the file is not affected. +- Neither the document, its text nor the error output of `java` is written + to the log, because a Java exception can quote parts of the document. A + failed run logs only its exit code. + +## Configuration + +Set these environment variables for the Zope process: + +- `C2_TRANSFORM_MSOFFICE_TIMEOUT`: seconds to wait for `java` before it is + killed. Default `60`. +- `C2_TRANSFORM_MSOFFICE_JAVA_OPTS`: JVM options, split like a shell command + line. They replace the default `-Xmx512m`, for example + `-Xmx1g -XX:+UseSerialGC`. The encoding options are always added. + +## Memory + +Each transform runs its own JVM. Up to one JVM per Zope thread can run at the +same time, each with a heap of up to 512 MB (by default) plus the memory of +the JVM itself. Plan the memory of the server or container for that. + +## Known issues + +- The bundled jar cannot extract text from `.ppt` files (PowerPoint 97-2003). + The transform runs, logs an error and gives empty text. `.pptx` files work. + This will be fixed when the jar is updated. + +## Uninstall + +Deactivate the add-on in the add-ons control panel. This removes the +transform. The MIME types are kept, because other add-ons may use them. + +## Source code + +- This package: +- Java program: + +## Credits + +This code was inspired by Products.OpenXml. + +## Contributors + +- Manabu Terada (main code) +- Mikio Hokari (Java code and extended calls) +- Takashi Nagai (Windows support) +- Takayuki Nagai (bug fix for 1.0b4) +- Tetsuya Morimoto (refactoring of the Java code and POI 3.17 support) + +## License + +GNU General Public License, version 2 or later. See `LICENSE.txt` and +`LICENSE.GPL`. diff --git a/README.txt b/README.txt deleted file mode 100644 index 007e0e4..0000000 --- a/README.txt +++ /dev/null @@ -1,57 +0,0 @@ -Introduction -============ - -Description -------------- - -- This product enables to search MS-Office files on Plone. -- Include Apache POI 3.17 package by jar file. - - - http://poi.apache.org/ - -- Code repository - - - https://bitbucket.org/cmscom/c2.transform.msoffice - -- Java code - - - https://bitbucket.org/cmscom/msdoc2txt - - -Support files ----------------- -- MS-Office files - - - MS-Word 95-2000, XP, 2007 and xml-format - - MS-Excel 95-2000, XP, 2007 and xml-format - - MS-PowerPoint 95-2000, XP, 2007 and xml-format - -Requirement -------------- - -- Plone - - - Plone 3.0-3.3 and Plone 4.x and 5.0 / 5.1 (tested by 5.1 CentOS 7 64bit) - -- Java - - - Java 8 - -- OS - - - Linux(like Unix) - - no support for Windows - -Credits ---------- - -This code was inspired by Products.OpenXml - - -Contributors ---------------- -- Manabu Terada(Twitter: @Terapyon) (Main code) -- Mikio Hokari (Java code & Extend call) -- Takashi Nagai (Support Windows) -- Takayuki Nagai (Bug fix for 1.0b4 tnagai@cc.kumamoto-u.ac.jp) -- Tetsuya Morimoto (Refactoring Java code and Support POI 3.17) \ No newline at end of file diff --git a/c2/__init__.py b/c2/__init__.py deleted file mode 100644 index f48ad10..0000000 --- a/c2/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/c2/transform/__init__.py b/c2/transform/__init__.py deleted file mode 100644 index f48ad10..0000000 --- a/c2/transform/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) diff --git a/c2/transform/msoffice/__init__.py b/c2/transform/msoffice/__init__.py deleted file mode 100644 index 332b398..0000000 --- a/c2/transform/msoffice/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ - -import logging -import config - -logger = logging.getLogger(config.PROJECTNAME) - -# def initialize(context): -# """Initializer called when used as a Zope 2 product.""" diff --git a/c2/transform/msoffice/config.py b/c2/transform/msoffice/config.py deleted file mode 100644 index b5cb74d..0000000 --- a/c2/transform/msoffice/config.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -import mimetypes - -PROJECTNAME = 'c2.transform.msoffice' -PROJECT_GLOBALS = globals() -SITE_CHARSET = 'utf-8' -TRANSFORM_NAME = 'msoffice_to_text' - -office_mimetypes = ( - {'name': 'Office Word 2007 XML macro enabled document', - 'mimetypes': ('application/vnd.ms-word.document.macroEnabled.12',), - 'extensions': ('docm',) - }, - {'name': 'Office Word 2007 XML document', - 'mimetypes': ('application/vnd.openxmlformats-officedocument.wordprocessingml.document',), - 'extensions': ('docx',) - }, - {'name': 'Office Powerpoint 2007 macro-enabled XML presentation', - 'mimetypes': ('application/vnd.ms-powerpoint.presentation.macroEnabled.12',), - 'extensions': ('pptm',) - }, - {'name': 'Office Powerpoint 2007 XML presentation', - 'mimetypes': ('application/vnd.openxmlformats-officedocument.presentationml.presentation',), - 'extensions': ('pptx',) - }, - {'name': 'Office Excel 2007 binary workbook (BIFF12)', - 'mimetypes': ('application/vnd.ms-excel.sheet.binary.macroEnabled.12',), - 'extensions': ('xlsb',) - }, - {'name': 'Office Excel 2007 XML macro-enabled workbook', - 'mimetypes': ('application/vnd.ms-excel.sheet.macroEnabled.12',), - 'extensions': ('xlsm',) - }, - {'name': 'Office Excel 2007 XML workbook', - 'mimetypes': ('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',), - 'extensions': ('xlsx',) - }, - ) - -for mt in office_mimetypes: - mt['globs'] = tuple(['*.' + ext for ext in mt['extensions']]) - mt['icon_path'] = 'application.png' - # Adding to standard mimetypes - mimetypes.add_type(mt['mimetypes'][0], '.' + mt['extensions'][0]) - -del mimetypes - diff --git a/c2/transform/msoffice/configure.zcml b/c2/transform/msoffice/configure.zcml deleted file mode 100644 index b0f7b09..0000000 --- a/c2/transform/msoffice/configure.zcml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - diff --git a/c2/transform/msoffice/profiles/default/import_steps.xml b/c2/transform/msoffice/profiles/default/import_steps.xml deleted file mode 100644 index 06ba2ea..0000000 --- a/c2/transform/msoffice/profiles/default/import_steps.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - C2TransformMsOffice setup for MimeTypesRegistry - - \ No newline at end of file diff --git a/c2/transform/msoffice/profiles/default/metadata.xml b/c2/transform/msoffice/profiles/default/metadata.xml deleted file mode 100644 index d5d3247..0000000 --- a/c2/transform/msoffice/profiles/default/metadata.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - 1.0b6 - diff --git a/c2/transform/msoffice/setuphandlers.py b/c2/transform/msoffice/setuphandlers.py deleted file mode 100644 index 843f3b9..0000000 --- a/c2/transform/msoffice/setuphandlers.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -from Products.CMFCore.utils import getToolByName -import config -from c2.transform.msoffice import logger - -def setupMSOfficeFile(context): - """Add our transform in PT for MS Office""" - - site = context.getSite() - - # Adding our file types to MTR - mtr = getToolByName(site, 'mimetypes_registry') - for mt_dict in config.office_mimetypes: - main_mt = mt_dict['mimetypes'][0] - mt_name = mt_dict['name'] - if bool(mtr.lookup(main_mt)): - # Already installed - logger.info("%s (%s) Mime type already installed, skipped", main_mt, mt_name) - continue - mtr.manage_addMimeType( - mt_name, - mt_dict['mimetypes'], - mt_dict['extensions'], - mt_dict['icon_path'], - binary=True, - globs=mt_dict['globs']) - logger.info("%s (%s) Mime type installed", main_mt, mt_name) - - # Registering our transform in PT - transforms_tool = getToolByName(site, 'portal_transforms') - if config.TRANSFORM_NAME not in transforms_tool.objectIds(): - # Not already installed - transforms_tool.manage_addTransform(config.TRANSFORM_NAME, 'c2.transform.msoffice.transform') - return - - -def removeMSOfficeFile(context): - """Removing various resources from plone site""" - - # At the moment, there's no uninstall support in GenericSetup. So - # this is run by the old style quickinstaller uninstall handler. - - site = context - # Removing our transform from PT - transforms_tool = getToolByName(site, 'portal_transforms') - transforms_tool.unregisterTransform(config.TRANSFORM_NAME) - return - diff --git a/c2/transform/msoffice/tests/__init__.py b/c2/transform/msoffice/tests/__init__.py deleted file mode 100644 index 6f804bc..0000000 --- a/c2/transform/msoffice/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 diff --git a/c2/transform/msoffice/tests/test_main.py b/c2/transform/msoffice/tests/test_main.py deleted file mode 100644 index 99486de..0000000 --- a/c2/transform/msoffice/tests/test_main.py +++ /dev/null @@ -1,144 +0,0 @@ -import unittest - -import os -import StringIO -from Products.CMFCore.utils import getToolByName -from Products.PloneTestCase import PloneTestCase - -from zope.testing import doctestunit -from zope.component import testing -from Testing import ZopeTestCase as ztc - -from Products.Five import zcml -from Products.Five import fiveconfigure -from Products.PloneTestCase import PloneTestCase as ptc -from Products.PloneTestCase.layer import PloneSite -from Products.MimetypesRegistry.interfaces import IMimetype - -from c2.transform.msoffice.config import TRANSFORM_NAME -from c2.transform.msoffice.config import office_mimetypes - -ptc.setupPloneSite() - -import c2.transform.msoffice - -class TestCase(ptc.PloneTestCase): - class layer(PloneSite): - @classmethod - def setUp(cls): - fiveconfigure.debug_mode = True - zcml.load_config('configure.zcml', - c2.transform.msoffice) - fiveconfigure.debug_mode = False - - @classmethod - def tearDown(cls): - pass - -class PTTestCase(TestCase): - """Testing portal_transforms settings""" - - def afterSetUp(self): - qi = self.portal.portal_quickinstaller - qi.installProduct('c2.transform.msoffice') - qi.installProduct('collective.indexing') - - portal = self.getPortal() - self.pt = getToolByName(portal, 'portal_transforms') - - - def testInstallation(self): - """Checking installation of our transform""" - - self.failUnless(TRANSFORM_NAME in self.pt.objectIds(spec='Transform'), - "%s transform expected" % TRANSFORM_NAME) - return - - - def testATfileSearchableText(self): - """Do we index the text of an openxml office file""" - - self.loginAsPortalOwner() - class fakefile(StringIO.StringIO): - pass - this_dir = os.path.dirname(os.path.abspath(__file__)) - test_filename = os.path.join(this_dir, 'wordprocessing1.docx') - fakefile = fakefile(file(test_filename, 'rb').read()) - fakefile.filename = 'wordprocessing1.docx' - file_id = self.portal.invokeFactory('File', fakefile.filename, file=fakefile) - file_item = getattr(self.portal, file_id) - # We sample some words from the file and its metadata - words = ("a", "simple", "example", "of", "document", "with", - "full", "set", "metadata", "comments") - st = file_item.SearchableText() - for word in words: - self.failUnless(word in st, "Expected '%s' in indexable text" % word) - return - - -class MTRTestCase(TestCase): - - def afterSetUp(self): - portal = self.getPortal() - self.mtr = getToolByName(portal, 'mimetypes_registry') - - - def testInstallation(self): - """Checking installation of our Mime types""" - providedBy = getattr(IMimetype, 'providedBy', - getattr(IMimetype, 'isImplementedBy', None)) - mtr = self.mtr - for mt_dict in office_mimetypes: - mt_string = mt_dict['mimetypes'][0] - mimetype = mtr.lookup(mt_string) - self.failUnless( - len(mimetype) > 0, - "Didn't find MimeType obj for %s" % mt_string) - if len(mimetype) > 0: - self.failUnless( - providedBy(mtr.lookup(mt_string)[0]), - "Didn't find MimeType obj for %s" % mt_string) - return - - - def testExtensions(self): - """Finding mimetypes by extension""" - - mtr = self.mtr - for mt_dict in office_mimetypes: - ext = mt_dict['extensions'][0] - expected = mt_dict['mimetypes'][0] - got = mtr.lookupExtension(ext).normalized() - self.failUnlessEqual(expected, got) - return - - - -def test_suite(): - return unittest.TestSuite([ - unittest.makeSuite(PTTestCase), - unittest.makeSuite(MTRTestCase), - - # Unit tests - #doctestunit.DocFileSuite( - # 'README.txt', package='c2.transform.msoffice', - # setUp=testing.setUp, tearDown=testing.tearDown), - - #doctestunit.DocTestSuite( - # module='c2.transform.msoffice.mymodule', - # setUp=testing.setUp, tearDown=testing.tearDown), - - - # Integration tests that use PloneTestCase - #ztc.ZopeDocFileSuite( - # 'README.txt', package='c2.transform.msoffice', - # test_class=TestCase), - - #ztc.FunctionalDocFileSuite( - # 'browser.txt', package='c2.transform.msoffice', - # test_class=TestCase), - - ]) - -if __name__ == '__main__': - unittest.main(defaultTest='test_suite') diff --git a/c2/transform/msoffice/tests/wordprocessing1.docx b/c2/transform/msoffice/tests/wordprocessing1.docx deleted file mode 100644 index 82c5a47..0000000 Binary files a/c2/transform/msoffice/tests/wordprocessing1.docx and /dev/null differ diff --git a/c2/transform/msoffice/transform.py b/c2/transform/msoffice/transform.py deleted file mode 100644 index 7de3546..0000000 --- a/c2/transform/msoffice/transform.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -import os -import sys -import signal -import mimetypes -import subprocess -if sys.platform == 'win32': - isWIN = True - import time - try: - import ctypes - except ImportError: - ctypes = None -else: - isWIN = False - import select - import time - import threading - -from zope.interface import implements -try: - from Products.PortalTransforms.interfaces import ITransform - HAS_PLONE3 = False -except ImportError: - from Products.PortalTransforms.interfaces import itransform - HAS_PLONE3 = True -from config import SITE_CHARSET, TRANSFORM_NAME -from c2.transform.msoffice import logger - -scpath = os.path.join(os.path.dirname(__file__), 'msdoc2txt.jar') - -def win_kill(pid): - if ctypes: - h = ctypes.windll.kernel32.OpenProcess(1, False, pid) - ctypes.windll.kernel32.TerminateProcess(h, -1) - ctypes.windll.kernel32.CloseHandle(h) - else: - subprocess.call(["TASKKILL", "/PID", str(pid), "/F"]) - -def win_timeout(p, timeout): - s = time.time() - while time.time() - s < timeout and p.poll() == None: - time.sleep(0.1) - if p.poll() == None: - win_kill(p.pid) - return False - return True - -def msdoc2txt(data, timeout=60): #default 60 sec - assert isinstance(data, str) - p = subprocess.Popen(("java", '-jar', '-Xmx512m', scpath), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - if not isWIN: - # Based on the code from http://betabug.ch/blogs/ch-athens/1093 - kill_check = threading.Event() - def _kill_process_after_a_timeout(pid): - os.kill(pid, signal.SIGTERM) - kill_check.set() # tell the main routine that we had to kill - # use SIGKILL if hard to kill... - return - - pid = p.pid - watchdog = threading.Timer(timeout, _kill_process_after_a_timeout, args=(pid, )) - watchdog.start() - (out,err) = p.communicate(input=data) - watchdog.cancel() # if it's still waiting to run - success = not kill_check.isSet() - kill_check.clear() - - if success: - if err: - status = False - val = err - elif out: - status = True - val = out - else: - status = None - val = "timeout" #if timeout - - return status, val - else: - r = win_timeout(p, timeout) - if r: - if p.poll(): - return False, p.stderr.read() #if fail - else: - return True, p.stdout.read() #if not fail - else: - return None, "timeout" #if timeout - - -class msoffice_to_text: - if HAS_PLONE3: - __implements__ = itransform - else: - implements(ITransform) - __name__ = TRANSFORM_NAME - - inputs = ( - # MS-Word formats - 'application/msword', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/vnd.ms-word.document.macroEnabled.12', - # MS-Excle formats - 'application/vnd.ms-excel', - 'application/msexcel', - 'application/x-msexcel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.ms-excel.sheet.binary.macroEnabled.12', - 'application/vnd.ms-excel.sheet.macroEnabled.12', - # MS-PowerPoint formats - 'application/powerpoint', - 'application/mspowerpoint', - 'application/x-mspowerpoint', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - 'application/vnd.ms-powerpoint.presentation.macroEnabled.12', - ) - - output = 'text/plain' - - output_encoding = SITE_CHARSET - - def __init__(self,name=None): - if name: - self.__name__=name - return - - def name(self): - return self.__name__ - - def convert(self, orig, data, **kwargs): - #orig_file = kwargs.get('filename') or 'unknown.xxx' - mimetype = kwargs.get('mimetype') - filename = kwargs.get('filename') or 'unknown.xxx' - - if mimetype is None: - mimetype = mimetypes.guess_type(filename)[0] - try: - status, text = msdoc2txt(orig) - except Exception, e: - logger.error(e, exc_info=True) - data.setData('') - return data - if status: - data.setData(text) - else: - logger.error(text, exc_info=True) - data.setData('') - return data - -def register(): - return msoffice_to_text() diff --git a/c2/transform/msoffice/version.txt b/c2/transform/msoffice/version.txt deleted file mode 100644 index eda7b34..0000000 --- a/c2/transform/msoffice/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.0b6 diff --git a/constraints_plone60.txt b/constraints_plone60.txt new file mode 100644 index 0000000..693399a --- /dev/null +++ b/constraints_plone60.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.0-latest/constraints.txt +ruff>=0.4.0 diff --git a/constraints_plone61.txt b/constraints_plone61.txt new file mode 100644 index 0000000..d40bf26 --- /dev/null +++ b/constraints_plone61.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.1-latest/constraints.txt +ruff>=0.4.0 diff --git a/constraints_plone62.txt b/constraints_plone62.txt new file mode 100644 index 0000000..c6ba411 --- /dev/null +++ b/constraints_plone62.txt @@ -0,0 +1,2 @@ +-c https://dist.plone.org/release/6.2-latest/constraints.txt +ruff>=0.4.0 diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt deleted file mode 100644 index d0e125a..0000000 --- a/docs/HISTORY.txt +++ /dev/null @@ -1,74 +0,0 @@ -Changelog -========= - - -1.0b6 (2018-04-02) ------------------------ - -* change from POI 3.6 to POI 3.17 with Java 8 [Tetsuya Morimoto] - - -1.0b5 (2018-02-28) ------------------------ - -* modify setup.py for target Plone [terapyon] -* tested Plone 5.1 - -1.0b4 (2013-03-16) ------------------------ - -* modify kill process for Popen [Takayuki Nagai] - -1.0b3 (2010-10-06) ---------------------- - -* adding terminated suspicious zombie process [terapyon] - -1.0b2 (2010-02-19) ---------------------- - -* remove the copyright in the files [terapyon] - -1.0b1 (2010-02-01) ---------------------- - -* modify getting from metadata and note in the msoffice file [terapyon] -* no support for Windows [terapyon] - - -0.3.2 (2010-01-19) ---------------------- - -* setting some mimetypes [terapyon] - -0.3.1 (2009-12-22) ---------------------- - -* bug fix for Windows [terapyon] - -0.3 (2009-12-22) ---------------------- - -* support Windows OS - -0.2 (2009-12-21) ---------------------- - -* support Plone 4.x [terapyon] - -0.1.2 (2009-12-17) ---------------------- - -* modify typo - -0.1.1 (2009-12-16) ---------------------- - -* modify exception code -* adding java args for big size file - - -0.1 (2009-12-16) ------------------ - -* Initial release diff --git a/docs/INSTALL.txt b/docs/INSTALL.txt deleted file mode 100644 index 6f29fd1..0000000 --- a/docs/INSTALL.txt +++ /dev/null @@ -1,47 +0,0 @@ -c2.transform.msoffice Installation -===================================== - -* When you're reading this you have probably already run - ``easy_install c2.transform.msoffice``. Find out how to install setuptools - (and EasyInstall) here: - http://peak.telecommunity.com/DevCenter/EasyInstall - -* Get `pythonproducts`_ and install it via:: - - python setup.py install --home /path/to/instance - - into your Zope instance. - -* Create a file called ``c2.transform.msoffice-configure.zcml`` in the - ``/path/to/instance/etc/package-includes`` directory. The file - should only contain this:: - - - -.. _pythonproducts: http://plone.org/products/pythonproducts - -Alternatively, if you are using zc.buildout and the plone.recipe.zope2instance -recipe to manage your project, you can do this: - -* Add ``c2.transform.msoffice`` to the list of eggs to install, e.g.:: - - [buildout] - ... - eggs = - ... - c2.transform.msoffice - -* Tell the plone.recipe.zope2instance recipe to install a ZCML slug:: - - [instance] - recipe = plone.recipe.zope2instance - ... - zcml = - c2.transform.msoffice - -* Re-run buildout, e.g. with:: - - $ ./bin/buildout - -You can skip the ZCML slug if you are going to explicitly include the package -from another package's configure.zcml file. diff --git a/news/+encoding.bugfix b/news/+encoding.bugfix new file mode 100644 index 0000000..7638d81 --- /dev/null +++ b/news/+encoding.bugfix @@ -0,0 +1 @@ +Always pass `-Dfile.encoding=UTF-8 -Dstdout.encoding=UTF-8` to `java`, so non-ASCII text is not replaced by `?` when the process has no locale (for example in slim container images). [terapyon] diff --git a/news/+env.feature b/news/+env.feature new file mode 100644 index 0000000..9d3e2b6 --- /dev/null +++ b/news/+env.feature @@ -0,0 +1 @@ +Add the `C2_TRANSFORM_MSOFFICE_TIMEOUT` and `C2_TRANSFORM_MSOFFICE_JAVA_OPTS` environment variables for the timeout and the JVM options. [terapyon] diff --git a/news/+exitcode.bugfix b/news/+exitcode.bugfix new file mode 100644 index 0000000..cc40f2a --- /dev/null +++ b/news/+exitcode.bugfix @@ -0,0 +1 @@ +Decide success by the exit code of `java`. Text is no longer thrown away when the jar writes a warning to stderr, and a document without text no longer logs an error. A failed run logs its exit code but not the error output of `java`, which can quote parts of the document. [terapyon] diff --git a/news/+pep420.breaking b/news/+pep420.breaking new file mode 100644 index 0000000..89752c6 --- /dev/null +++ b/news/+pep420.breaking @@ -0,0 +1 @@ +`c2` and `c2.transform` are now PEP 420 namespace packages. Sites that also use `c2.*` packages with `pkg_resources` namespaces need `horse-with-no-namespace`. [terapyon] diff --git a/news/+plone6.breaking b/news/+plone6.breaking new file mode 100644 index 0000000..836012b --- /dev/null +++ b/news/+plone6.breaking @@ -0,0 +1 @@ +Support Plone 6.0, 6.1 and 6.2 on Python 3.10 to 3.14 only. Python 2, Plone 3 to 5 and the Windows code are removed. [terapyon] diff --git a/news/+ppt.documentation b/news/+ppt.documentation new file mode 100644 index 0000000..c7a8861 --- /dev/null +++ b/news/+ppt.documentation @@ -0,0 +1 @@ +Document a known issue: the bundled jar cannot extract text from `.ppt` files (this was already the case in 1.0b6). [terapyon] diff --git a/news/+reinstall.breaking b/news/+reinstall.breaking new file mode 100644 index 0000000..c711bb8 --- /dev/null +++ b/news/+reinstall.breaking @@ -0,0 +1 @@ +The profile version is now `2000` and there is no upgrade step from 1.0b6: deactivate and activate the add-on again after upgrading. [terapyon] diff --git a/news/+str-input.bugfix b/news/+str-input.bugfix new file mode 100644 index 0000000..e1b6db3 --- /dev/null +++ b/news/+str-input.bugfix @@ -0,0 +1 @@ +Log text input and give empty text instead of failing an `assert`. [terapyon] diff --git a/news/+tooling.internal b/news/+tooling.internal new file mode 100644 index 0000000..0c88c78 --- /dev/null +++ b/news/+tooling.internal @@ -0,0 +1 @@ +Move to `pyproject.toml` and a `src/` layout, with tox, pytest, ruff, towncrier, zest.releaser and a GitHub Actions workflow that runs the tests with Java. Replace the test sample with self-made documents. [terapyon] diff --git a/news/+uninstall.feature b/news/+uninstall.feature new file mode 100644 index 0000000..a6b42c4 --- /dev/null +++ b/news/+uninstall.feature @@ -0,0 +1 @@ +Add an uninstall profile that removes the transform, and register `application/vnd.ms-powerpoint` as an input explicitly. [terapyon] diff --git a/news/.changelog_template.jinja b/news/.changelog_template.jinja new file mode 100644 index 0000000..0cf429a --- /dev/null +++ b/news/.changelog_template.jinja @@ -0,0 +1,15 @@ +{% if sections[""] %} +{% for category, val in definitions.items() if category in sections[""] %} + +### {{ definitions[category]['name'] }} + +{% for text, values in sections[""][category].items() %} +- {{ text }} {{ values|join(', ') }} +{% endfor %} + +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} diff --git a/news/.gitkeep b/news/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7114952 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,179 @@ +[build-system] +requires = ["setuptools>=69.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "c2.transform.msoffice" +version = "2.0.0.dev0" +description = "Make Microsoft Office files searchable in Plone" +readme = "README.md" +license = {text = "GPL version 2 or later"} +requires-python = ">=3.10,<3.15" +authors = [ + { name = "Manabu TERADA", email = "terada@cmscom.jp" }, +] +keywords = [ + "Plone", + "transform", + "MS Office", + "search", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Plone", + "Framework :: Plone :: 6.0", + "Framework :: Plone :: 6.1", + "Framework :: Plone :: 6.2", + "Framework :: Plone :: Addon", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: POSIX", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "plone.base", + "Products.CMFPlone", + "Products.GenericSetup", + "Products.MimetypesRegistry", + "Products.PortalTransforms", +] + +[project.optional-dependencies] +test = [ + "horse-with-no-namespace", + "plone.api", + "plone.app.contenttypes", + "plone.app.testing", + "pytest", + "pytest-cov", + "pytest-plone>=0.5.0", +] +release = [ + "zest.releaser[recommended]", + "zestreleaser.towncrier", + "zest.pocompile", +] + +[project.urls] +Homepage = "https://github.com/cmscom/c2.transform.msoffice" +PyPI = "https://pypi.org/project/c2.transform.msoffice" +Source = "https://github.com/cmscom/c2.transform.msoffice" +Tracker = "https://github.com/cmscom/c2.transform.msoffice/issues" + +[project.entry-points."z3c.autoinclude.plugin"] +target = "plone" + +[tool.setuptools] +include-package-data = true +zip-safe = false + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = true + +[tool.towncrier] +directory = "news/" +filename = "CHANGELOG.md" +start_string = "\n" +title_format = "## {version} ({project_date})" +template = "news/.changelog_template.jinja" +issue_format = "[#{issue}](https://github.com/cmscom/c2.transform.msoffice/issues/{issue})" +underlines = ["", "", ""] + +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking changes:" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "New features:" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug fixes:" +showcontent = true + +[[tool.towncrier.type]] +directory = "internal" +name = "Internal:" +showcontent = true + +[[tool.towncrier.type]] +directory = "documentation" +name = "Documentation:" +showcontent = true + +[[tool.towncrier.type]] +directory = "tests" +name = "Tests" +showcontent = true + +[tool.ruff] +target-version = "py310" +line-length = 88 +fix = true +lint.select = [ + # flake8-2020 + "YTT", + # flake8-bandit + "S", + # flake8-bugbear + "B", + # flake8-builtins + "A", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", + # flake8-simplify + "SIM", + # mccabe + "C90", + # pycodestyle + "E", "W", + # pyflakes + "F", + # pygrep-hooks + "PGH", + # pyupgrade + "UP", + # ruff + "RUF", +] +lint.ignore = [ + # DoNotAssignLambda + "E731", +] + +[tool.ruff.lint.isort] +case-sensitive = false +no-sections = true +force-single-line = true +from-first = true +lines-after-imports = 2 +lines-between-types = 1 +order-by-type = false + +[tool.ruff.format] +preview = true + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["E501", "RUF001", "S101"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.run] +source_pkgs = ["c2.transform.msoffice", "tests"] +branch = true +parallel = true + +[tool.zest-releaser] +python-file-with-version = "src/c2/transform/msoffice/__init__.py" diff --git a/requirements_plone60.txt b/requirements_plone60.txt new file mode 100644 index 0000000..f5fbf6d --- /dev/null +++ b/requirements_plone60.txt @@ -0,0 +1,2 @@ +-c constraints_plone60.txt +setuptools diff --git a/requirements_plone61.txt b/requirements_plone61.txt new file mode 100644 index 0000000..443b65f --- /dev/null +++ b/requirements_plone61.txt @@ -0,0 +1,2 @@ +-c constraints_plone61.txt +setuptools diff --git a/requirements_plone62.txt b/requirements_plone62.txt new file mode 100644 index 0000000..471765c --- /dev/null +++ b/requirements_plone62.txt @@ -0,0 +1,2 @@ +-c constraints_plone62.txt +setuptools diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 21cced6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[egg_info] -tag_svn_revision = false diff --git a/setup.py b/setup.py deleted file mode 100644 index e7ffa45..0000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -from setuptools import setup, find_packages -import os - -version = '1.0b6' - -setup(name='c2.transform.msoffice', - version=version, - description="This product enables to search MS-Office files on Plone.", - long_description=open("README.txt").read() + "\n" + - open(os.path.join("docs", "HISTORY.txt")).read() + "\n" + - open(os.path.join("docs", "INSTALL.txt")).read(), - # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - "Framework :: Plone", - "Programming Language :: Python", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - keywords='Plone transform MS Office search', - author='Manabu Terada', - author_email='terada@cmscom.jp', - url='http://www.cmscom.jp', - license='GPL', - packages=find_packages(exclude=['ez_setup']), - namespace_packages=['c2', 'c2.transform'], - include_package_data=True, - zip_safe=False, - install_requires=[ - 'setuptools', - # -*- Extra requirements: -*- - ], - entry_points=""" - # -*- Entry points: -*- - [z3c.autoinclude.plugin] - target = plone - """, - ) diff --git a/src/c2/transform/msoffice/__init__.py b/src/c2/transform/msoffice/__init__.py new file mode 100644 index 0000000..dd91d5d --- /dev/null +++ b/src/c2/transform/msoffice/__init__.py @@ -0,0 +1,13 @@ +"""Make Microsoft Office files searchable in Plone.""" + +# Importing config registers the Office types with the mimetypes module. +from c2.transform.msoffice import config # noqa: F401 + +import logging + + +__version__ = "2.0.0.dev0" + +PACKAGE_NAME = "c2.transform.msoffice" + +logger = logging.getLogger(PACKAGE_NAME) diff --git a/src/c2/transform/msoffice/config.py b/src/c2/transform/msoffice/config.py new file mode 100644 index 0000000..d6d659b --- /dev/null +++ b/src/c2/transform/msoffice/config.py @@ -0,0 +1,60 @@ +import mimetypes + + +PROJECTNAME = "c2.transform.msoffice" +SITE_CHARSET = "utf-8" +TRANSFORM_NAME = "msoffice_to_text" + +office_mimetypes = ( + { + "name": "Office Word 2007 XML macro enabled document", + "mimetypes": ("application/vnd.ms-word.document.macroEnabled.12",), + "extensions": ("docm",), + }, + { + "name": "Office Word 2007 XML document", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + "extensions": ("docx",), + }, + { + "name": "Office Powerpoint 2007 macro-enabled XML presentation", + "mimetypes": ("application/vnd.ms-powerpoint.presentation.macroEnabled.12",), + "extensions": ("pptm",), + }, + { + "name": "Office Powerpoint 2007 XML presentation", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ), + "extensions": ("pptx",), + }, + { + "name": "Office Excel 2007 binary workbook (BIFF12)", + "mimetypes": ("application/vnd.ms-excel.sheet.binary.macroEnabled.12",), + "extensions": ("xlsb",), + }, + { + "name": "Office Excel 2007 XML macro-enabled workbook", + "mimetypes": ("application/vnd.ms-excel.sheet.macroEnabled.12",), + "extensions": ("xlsm",), + }, + { + "name": "Office Excel 2007 XML workbook", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + "extensions": ("xlsx",), + }, +) + +for mt in office_mimetypes: + mt["globs"] = tuple("*." + ext for ext in mt["extensions"]) + mt["icon_path"] = "application.png" + # plone.namedfile guesses a missing content type with the standard library. + # Python < 3.14 does not know these types and slim images have no + # /etc/mime.types, so they are added on import, in every process. + mimetypes.add_type(mt["mimetypes"][0], "." + mt["extensions"][0]) + +del mimetypes diff --git a/src/c2/transform/msoffice/configure.zcml b/src/c2/transform/msoffice/configure.zcml new file mode 100644 index 0000000..acdc045 --- /dev/null +++ b/src/c2/transform/msoffice/configure.zcml @@ -0,0 +1,31 @@ + + + + + + + + + + diff --git a/c2/transform/msoffice/javasrc/README.md b/src/c2/transform/msoffice/javasrc/README.md similarity index 100% rename from c2/transform/msoffice/javasrc/README.md rename to src/c2/transform/msoffice/javasrc/README.md diff --git a/c2/transform/msoffice/msdoc2txt.jar b/src/c2/transform/msoffice/msdoc2txt.jar similarity index 100% rename from c2/transform/msoffice/msdoc2txt.jar rename to src/c2/transform/msoffice/msdoc2txt.jar diff --git a/src/c2/transform/msoffice/profiles/default/metadata.xml b/src/c2/transform/msoffice/profiles/default/metadata.xml new file mode 100644 index 0000000..106aca5 --- /dev/null +++ b/src/c2/transform/msoffice/profiles/default/metadata.xml @@ -0,0 +1,4 @@ + + + 2000 + diff --git a/src/c2/transform/msoffice/profiles/uninstall/metadata.xml b/src/c2/transform/msoffice/profiles/uninstall/metadata.xml new file mode 100644 index 0000000..106aca5 --- /dev/null +++ b/src/c2/transform/msoffice/profiles/uninstall/metadata.xml @@ -0,0 +1,4 @@ + + + 2000 + diff --git a/src/c2/transform/msoffice/setuphandlers.py b/src/c2/transform/msoffice/setuphandlers.py new file mode 100644 index 0000000..8bef927 --- /dev/null +++ b/src/c2/transform/msoffice/setuphandlers.py @@ -0,0 +1,69 @@ +from c2.transform.msoffice import config +from c2.transform.msoffice import logger +from plone.base.interfaces import INonInstallable +from Products.CMFCore.utils import getToolByName +from zope.interface import implementer + + +TRANSFORM_MODULE = "c2.transform.msoffice.transform" + + +@implementer(INonInstallable) +class HiddenProfiles: + def getNonInstallableProfiles(self): + """Hide the uninstall profile from site creation and the add-ons list.""" + return ["c2.transform.msoffice:uninstall"] + + +def post_install(context): + """Register the Office MIME types and the transform. + + ``context`` is the setup tool. Running it again changes nothing. + """ + mtr = getToolByName(context, "mimetypes_registry") + for mt_dict in config.office_mimetypes: + main_mt = mt_dict["mimetypes"][0] + mt_name = mt_dict["name"] + if mtr.lookup(main_mt): + logger.info("%s (%s) MIME type already registered", main_mt, mt_name) + continue + mtr.manage_addMimeType( + mt_name, + mt_dict["mimetypes"], + mt_dict["extensions"], + mt_dict["icon_path"], + binary=True, + globs=mt_dict["globs"], + ) + logger.info("%s (%s) MIME type registered", main_mt, mt_name) + + transforms = getToolByName(context, "portal_transforms") + if config.TRANSFORM_NAME in transforms.objectIds(): + logger.info( + "%s is already registered in portal_transforms; skipped", + config.TRANSFORM_NAME, + ) + return + transforms.manage_addTransform(config.TRANSFORM_NAME, TRANSFORM_MODULE) + + +def uninstall(context): + """Remove the transform. The MIME types stay; others may use them.""" + transforms = getToolByName(context, "portal_transforms") + if config.TRANSFORM_NAME not in transforms.objectIds(): + return + try: + transforms.unregisterTransform(config.TRANSFORM_NAME) + except KeyError as error: + # unregisterTransform unmaps first and raises KeyError when a MIME + # type was never mapped to this transform; it has not deleted the + # transform yet. Delete it, so that installing again adds it anew. + # Without events, because Transform.manage_beforeDelete unmaps again + # and OFS re-raises that KeyError in debug mode for non-Managers. + transforms._delObject(config.TRANSFORM_NAME, suppress_events=True) + logger.warning( + "Could not unmap %s in portal_transforms: %r; " + "the transform was removed anyway", + config.TRANSFORM_NAME, + error, + ) diff --git a/src/c2/transform/msoffice/testing.py b/src/c2/transform/msoffice/testing.py new file mode 100644 index 0000000..4f2d037 --- /dev/null +++ b/src/c2/transform/msoffice/testing.py @@ -0,0 +1,31 @@ +from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE +from plone.app.testing import applyProfile +from plone.app.testing import FunctionalTesting +from plone.app.testing import IntegrationTesting +from plone.app.testing import PloneSandboxLayer + +import c2.transform.msoffice + + +class Layer(PloneSandboxLayer): + defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,) + + def setUpZope(self, app, configurationContext): + # z3c.autoinclude is disabled in the Plone fixture base layer. + self.loadZCML(package=c2.transform.msoffice) + + def setUpPloneSite(self, portal): + applyProfile(portal, "c2.transform.msoffice:default") + + +FIXTURE = Layer() + +INTEGRATION_TESTING = IntegrationTesting( + bases=(FIXTURE,), + name="C2.Transform.MsofficeLayer:IntegrationTesting", +) + +FUNCTIONAL_TESTING = FunctionalTesting( + bases=(FIXTURE,), + name="C2.Transform.MsofficeLayer:FunctionalTesting", +) diff --git a/src/c2/transform/msoffice/transform.py b/src/c2/transform/msoffice/transform.py new file mode 100644 index 0000000..0c5ecd0 --- /dev/null +++ b/src/c2/transform/msoffice/transform.py @@ -0,0 +1,144 @@ +"""Extract plain text from Microsoft Office files with the bundled jar.""" + +from c2.transform.msoffice import logger +from c2.transform.msoffice.config import SITE_CHARSET +from c2.transform.msoffice.config import TRANSFORM_NAME +from pathlib import Path +from Products.PortalTransforms.interfaces import ITransform +from zope.interface import implementer + +import os +import shlex +import subprocess + + +JAR_PATH = Path(__file__).with_name("msdoc2txt.jar") + +TIMEOUT_ENV = "C2_TRANSFORM_MSOFFICE_TIMEOUT" +JAVA_OPTS_ENV = "C2_TRANSFORM_MSOFFICE_JAVA_OPTS" +DEFAULT_TIMEOUT = 60 +DEFAULT_JAVA_OPTS = ("-Xmx512m",) +# The jar writes text in the JVM's default encoding. JDK 8-17 take it from +# file.encoding and JDK 19+ from stdout.encoding, so both are always set. +# They come after the options from the environment so they always win. +ENCODING_OPTS = ("-Dfile.encoding=UTF-8", "-Dstdout.encoding=UTF-8") + + +def timeout() -> int: + """Seconds to wait for the jar, from the environment or the default.""" + value = os.environ.get(TIMEOUT_ENV, "").strip() + if not value: + return DEFAULT_TIMEOUT + try: + seconds = int(value) + except ValueError: + # Not only "abc": str.isdigit() accepts "²", which int() rejects. + seconds = 0 + if seconds > 0: + return seconds + logger.warning( + "Ignoring %s=%r: expected a positive integer; using %d", + TIMEOUT_ENV, + value, + DEFAULT_TIMEOUT, + ) + return DEFAULT_TIMEOUT + + +def java_command() -> list[str]: + java_opts = shlex.split(os.environ.get(JAVA_OPTS_ENV, "")) or DEFAULT_JAVA_OPTS + return ["java", *java_opts, *ENCODING_OPTS, "-jar", str(JAR_PATH)] + + +def msdoc2txt(data: bytes) -> str | None: + """Return the text of an Office document, or None if extraction failed. + + The document goes to the jar on stdin and the text comes back on stdout. + The exit code decides success: an empty stdout with exit code 0 is a + document without text. Neither the document, the text nor the jar's + stderr is logged: a Java exception can quote parts of the document. + """ + seconds = timeout() + try: + command = java_command() + except ValueError: + # shlex could not split the options, for example an unclosed quote. + logger.error("JVM options in %s could not be parsed", JAVA_OPTS_ENV) + return None + try: + result = subprocess.run( # noqa: S603 + command, input=data, capture_output=True, timeout=seconds + ) + except subprocess.TimeoutExpired: + # subprocess.run kills the process and waits for it. + logger.error("msdoc2txt timed out after %d seconds", seconds) + return None + except FileNotFoundError: + logger.error("msdoc2txt could not run: java was not found") + return None + except Exception: + logger.exception("msdoc2txt could not run") + return None + if result.returncode != 0: + logger.error("msdoc2txt failed with exit code %d", result.returncode) + return None + if not result.stdout: + logger.debug("msdoc2txt returned no text") + return result.stdout.decode("utf-8", errors="replace") + + +@implementer(ITransform) +class msoffice_to_text: + __name__ = TRANSFORM_NAME + + inputs = ( + # MS-Word formats + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-word.document.macroEnabled.12", + # MS-Excel formats + "application/vnd.ms-excel", + "application/msexcel", + "application/x-msexcel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-excel.sheet.macroEnabled.12", + # MS-PowerPoint formats + "application/vnd.ms-powerpoint", + "application/powerpoint", + "application/mspowerpoint", + "application/x-mspowerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + ) + + output = "text/plain" + + output_encoding = SITE_CHARSET + + def __init__(self, name=None): + if name: + self.__name__ = name + + def name(self): + return self.__name__ + + def convert(self, orig, data, **kwargs): + """Set the text of ``orig`` (bytes) on ``data``. + + On any failure the text is empty, so saving the file still works. + """ + if not isinstance(orig, bytes): + logger.error( + "msoffice_to_text expects bytes, got %s; no text is extracted", + type(orig).__name__, + ) + data.setData("") + return data + text = msdoc2txt(orig) + data.setData(text or "") + return data + + +def register(): + return msoffice_to_text() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b7b5d61 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +from c2.transform.msoffice.testing import FUNCTIONAL_TESTING +from c2.transform.msoffice.testing import INTEGRATION_TESTING +from pytest_plone import fixtures_factory + + +pytest_plugins = ["pytest_plone"] + + +globals().update( + fixtures_factory(( + (FUNCTIONAL_TESTING, "functional"), + (INTEGRATION_TESTING, "integration"), + )) +) diff --git a/tests/samples/document.doc b/tests/samples/document.doc new file mode 100644 index 0000000..0c34e86 Binary files /dev/null and b/tests/samples/document.doc differ diff --git a/tests/samples/document.docx b/tests/samples/document.docx new file mode 100644 index 0000000..3581e4e Binary files /dev/null and b/tests/samples/document.docx differ diff --git a/tests/samples/empty.docx b/tests/samples/empty.docx new file mode 100644 index 0000000..ffd24d6 Binary files /dev/null and b/tests/samples/empty.docx differ diff --git a/tests/samples/presentation.ppt b/tests/samples/presentation.ppt new file mode 100644 index 0000000..3faedfb Binary files /dev/null and b/tests/samples/presentation.ppt differ diff --git a/tests/samples/presentation.pptx b/tests/samples/presentation.pptx new file mode 100644 index 0000000..f05207e Binary files /dev/null and b/tests/samples/presentation.pptx differ diff --git a/tests/samples/spreadsheet.xls b/tests/samples/spreadsheet.xls new file mode 100644 index 0000000..ee1bed0 Binary files /dev/null and b/tests/samples/spreadsheet.xls differ diff --git a/tests/samples/spreadsheet.xlsx b/tests/samples/spreadsheet.xlsx new file mode 100644 index 0000000..4a83543 Binary files /dev/null and b/tests/samples/spreadsheet.xlsx differ diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..8233741 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,47 @@ +from c2.transform.msoffice import config + +import importlib +import mimetypes +import pytest + + +@pytest.fixture +def stdlib_mimetypes_only(monkeypatch): + """Forget the host's mime.types (slim images do not have one). + + mimetypes.init(files=[]) keeps the current database, so it is replaced. + """ + monkeypatch.setattr(mimetypes, "_db", mimetypes.MimeTypes()) + + +@pytest.mark.usefixtures("stdlib_mimetypes_only") +def test_import_adds_office_types_to_stdlib_mimetypes(): + importlib.reload(config) + assert { + ext: mimetypes.guess_type(f"file.{ext}")[0] + for ext in ("docm", "docx", "pptm", "pptx", "xlsb", "xlsm", "xlsx") + } == { + "docm": "application/vnd.ms-word.document.macroEnabled.12", + "docx": ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ), + "pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "pptx": ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation" + ), + "xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + } + + +def test_office_mimetypes_have_globs_and_icon(): + assert config.office_mimetypes[1] == { + "name": "Office Word 2007 XML document", + "mimetypes": ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + "extensions": ("docx",), + "globs": ("*.docx",), + "icon_path": "application.png", + } diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..962975d --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,12 @@ +def test_version(): + from c2.transform.msoffice import __version__ + + assert __version__ == "2.0.0.dev0" + + +def test_c2_is_a_pep420_namespace(): + import c2 + import c2.transform + + assert getattr(c2, "__file__", None) is None + assert getattr(c2.transform, "__file__", None) is None diff --git a/tests/test_java.py b/tests/test_java.py new file mode 100644 index 0000000..6bd1a76 --- /dev/null +++ b/tests/test_java.py @@ -0,0 +1,156 @@ +"""Tests that run the bundled jar with a real ``java``. + +They are skipped when ``java`` is not on PATH, unless +C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1 is set (CI sets it); then they fail. +""" + +from pathlib import Path +from plone import api +from plone.app.contenttypes.indexers import SearchableText_file +from plone.namedfile.file import NamedBlobFile + +import logging +import os +import pytest +import shutil + + +SAMPLES = Path(__file__).parent / "samples" +REQUIRE_JAVA = "C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA" + +DOC = "application/msword" +DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +XLS = "application/vnd.ms-excel" +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +PPT = "application/vnd.ms-powerpoint" +PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation" + + +@pytest.fixture +def java(): + if shutil.which("java") is None: + if os.environ.get(REQUIRE_JAVA) == "1": + pytest.fail(f"java is not on PATH and {REQUIRE_JAVA}=1") + pytest.skip("java is not on PATH") + + +@pytest.fixture +def no_locale(monkeypatch): + """Run java without a locale, like a slim container image.""" + monkeypatch.setenv("LC_ALL", "C") + monkeypatch.delenv("LANG", raising=False) + + +def add_file(portal, name: str, content_type: str): + with api.env.adopt_roles(["Manager"]): + return api.content.create( + container=portal, + type="File", + id=name.replace(".", "-"), + title="Sample", + file=NamedBlobFile( + data=(SAMPLES / name).read_bytes(), + filename=name, + contentType=content_type, + ), + ) + + +def text_of(portal, name: str, content_type: str) -> str: + return SearchableText_file(add_file(portal, name, content_type))() + + +def transformed(portal, name: str, content_type: str) -> str: + data = portal.portal_transforms.convertTo( + "text/plain", (SAMPLES / name).read_bytes(), mimetype=content_type + ) + return data.getData() + + +@pytest.mark.usefixtures("java", "no_locale") +class TestRealJava: + def test_docx_words_are_searchable(self, portal): + text = text_of(portal, "document.docx", DOCX) + for word in ("simple", "example", "document", "metadata", "comments"): + assert word in text + + def test_japanese_is_not_garbled(self, portal): + assert "これは日本語の文書の例です。" in text_of(portal, "document.docx", DOCX) + + def test_docx_text(self, portal): + assert transformed(portal, "document.docx", DOCX) == ( + "This is a simple example of a document with a full set of metadata " + "and comments.\nこれは日本語の文書の例です。\n\n" + ) + + @pytest.mark.parametrize( + ("name", "content_type", "expected"), + [ + ( + "document.doc", + DOC, + [ + "This is a simple example of a document", + "これは日本語の文書の例です。", + ], + ), + ("spreadsheet.xls", XLS, ["Fruit\tCount", "Apple\t3", "みかん\t5"]), + ("spreadsheet.xlsx", XLSX, ["Fruit\tCount", "Apple\t3", "みかん\t5"]), + ( + "presentation.pptx", + PPTX, + ["Presentation slide example", "スライドの例です。"], + ), + ], + ) + def test_other_formats(self, portal, name, content_type, expected): + text = text_of(portal, name, content_type) + for part in expected: + assert part in text + + def test_document_without_text(self, portal, caplog): + with caplog.at_level(logging.INFO, logger="c2.transform.msoffice"): + assert transformed(portal, "empty.docx", DOCX) == "" + assert caplog.records == [] + + def test_ppt_gives_no_text_with_the_bundled_jar(self, portal, caplog): + """Known issue: the bundled jar cannot read .ppt files (see the xfail).""" + obj = add_file(portal, "presentation.ppt", PPT) + assert obj.file.data == (SAMPLES / "presentation.ppt").read_bytes() + # Creating the File indexes it once already. + caplog.clear() + with caplog.at_level(logging.ERROR, logger="c2.transform.msoffice"): + text = SearchableText_file(obj)() + assert "Presentation slide example" not in text + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt failed with exit code 1" + ] + + @pytest.mark.xfail( + strict=True, + reason=( + "The bundled msdoc2txt.jar (POI 3.17) fails on every .ppt: " + "CurrentUserAtom. -> 'Block 3 not found' / 'Unable to read 512 " + "bytes from 2048 in stream of length -1'. MSDocExtracter seems to close " + "the POIFS before HSLF reads the Current User stream. To be fixed when " + "the jar is updated." + ), + ) + def test_ppt_text_is_searchable(self, portal): + text = text_of(portal, "presentation.ppt", PPT) + assert "Presentation slide example" in text + assert "スライドの例です。" in text + + +def test_saving_works_without_java(portal, monkeypatch, tmp_path, caplog): + monkeypatch.setenv("PATH", str(tmp_path)) + obj = add_file(portal, "document.docx", DOCX) + assert obj.file.data == (SAMPLES / "document.docx").read_bytes() + # Creating the File indexes it once already. + caplog.clear() + with caplog.at_level(logging.ERROR, logger="c2.transform.msoffice"): + text = SearchableText_file(obj)() + assert "simple example" not in text + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt could not run: java was not found" + ] diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 0000000..f6394c7 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,115 @@ +from Acquisition import aq_base +from c2.transform.msoffice import PACKAGE_NAME +from plone.base.interfaces import INonInstallable +from zope.component import getUtility + +import logging +import pytest + + +TRANSFORM_NAME = "msoffice_to_text" +DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +DEFAULT_PROFILE = "profile-c2.transform.msoffice:default" +UNINSTALL_PROFILE = "profile-c2.transform.msoffice:uninstall" + + +@pytest.fixture +def transforms(portal): + return portal.portal_transforms + + +@pytest.fixture +def setup_tool(portal): + return portal.portal_setup + + +def test_addon_installed(installer): + assert installer.is_product_installed(PACKAGE_NAME) is True + + +def test_latest_version(profile_last_version): + assert profile_last_version(f"{PACKAGE_NAME}:default") == "2000" + + +def test_transform_installed(transforms): + assert TRANSFORM_NAME in transforms.objectIds() + transform = transforms[TRANSFORM_NAME] + assert transform.module == "c2.transform.msoffice.transform" + assert "application/vnd.ms-powerpoint" in transform.inputs + + +def test_install_restores_a_missing_mimetype(portal, setup_tool): + mtr = portal.mimetypes_registry + mtr.manage_delObjects([DOCX]) + assert mtr.lookup(DOCX) == () + setup_tool.runAllImportStepsFromProfile(DEFAULT_PROFILE) + [mimetype] = mtr.lookup(DOCX) + assert mimetype.name() == "Office Word 2007 XML document" + assert mimetype.extensions == ("docx",) + assert mimetype.globs == ("*.docx",) + assert mimetype.binary + + +def test_install_twice(transforms, setup_tool, caplog): + transform = transforms[TRANSFORM_NAME] + with caplog.at_level(logging.INFO, logger=PACKAGE_NAME): + setup_tool.runAllImportStepsFromProfile(DEFAULT_PROFILE) + assert transforms.objectIds().count(TRANSFORM_NAME) == 1 + assert aq_base(transforms[TRANSFORM_NAME]) is aq_base(transform) + assert ( + PACKAGE_NAME, + logging.INFO, + "msoffice_to_text is already registered in portal_transforms; skipped", + ) in caplog.record_tuples + + +def test_uninstall_profile_is_hidden(portal): + hidden = getUtility(INonInstallable, name=PACKAGE_NAME) + assert hidden.getNonInstallableProfiles() == ["c2.transform.msoffice:uninstall"] + + +class TestUninstall: + @pytest.fixture(autouse=True) + def uninstalled(self, installer): + installer.uninstall_product(PACKAGE_NAME) + + def test_addon_uninstalled(self, installer): + assert installer.is_product_installed(PACKAGE_NAME) is False + + def test_transform_removed(self, transforms): + assert TRANSFORM_NAME not in transforms.objectIds() + + def test_mimetypes_are_kept(self, portal): + assert len(portal.mimetypes_registry.lookup(DOCX)) == 1 + + def test_uninstall_twice(self, transforms, setup_tool): + setup_tool.runAllImportStepsFromProfile(UNINSTALL_PROFILE) + assert TRANSFORM_NAME not in transforms.objectIds() + + def test_missing_mapping_still_removes_the_transform( + self, installer, transforms, caplog + ): + installer.install_product(PACKAGE_NAME) + # A site where the DOCX -> text/plain mapping got lost: + # unregisterTransform raises KeyError before it deletes the transform. + del transforms._mtmap[DOCX]["text/plain"] + with caplog.at_level(logging.WARNING, logger=PACKAGE_NAME): + installer.uninstall_product(PACKAGE_NAME) + assert installer.is_product_installed(PACKAGE_NAME) is False + assert TRANSFORM_NAME not in transforms.objectIds() + assert [r.getMessage() for r in caplog.records] == [ + "Could not unmap msoffice_to_text in portal_transforms: " + "KeyError('text/plain'); the transform was removed anyway" + ] + + def test_install_after_a_missing_mapping_adds_the_transform( + self, installer, transforms + ): + installer.install_product(PACKAGE_NAME) + del transforms._mtmap[DOCX]["text/plain"] + installer.uninstall_product(PACKAGE_NAME) + installer.install_product(PACKAGE_NAME) + assert TRANSFORM_NAME in transforms.objectIds() + assert [t.name() for t in transforms._mtmap[DOCX]["text/plain"]] == [ + "msoffice_to_text" + ] diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..885fafe --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,235 @@ +"""Unit tests for the transform. ``java`` is replaced; Plone is not started.""" + +from c2.transform.msoffice import transform +from pathlib import Path +from Products.PortalTransforms.data import datastream +from Products.PortalTransforms.interfaces import ITransform + +import logging +import pytest +import subprocess + + +JAR = str(Path(transform.__file__).with_name("msdoc2txt.jar")) +DOCUMENT = b"PK\x03\x04 secret document body" + + +class FakeRun: + """Stands in for subprocess.run and records the calls.""" + + def __init__(self, returncode=0, stdout=b"", stderr=b"", error=None): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + self.error = error + self.calls = [] + + def __call__(self, args, **kwargs): + self.calls.append((args, kwargs)) + if self.error is not None: + raise self.error + return subprocess.CompletedProcess( + args, self.returncode, stdout=self.stdout, stderr=self.stderr + ) + + +@pytest.fixture(autouse=True) +def clean_environment(monkeypatch): + monkeypatch.delenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", raising=False) + monkeypatch.delenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", raising=False) + + +def fake_run(monkeypatch, **kwargs) -> FakeRun: + run = FakeRun(**kwargs) + monkeypatch.setattr(transform.subprocess, "run", run) + return run + + +def convert(orig) -> str: + data = datastream("test") + result = transform.msoffice_to_text().convert(orig, data) + assert result is data + return data.getData() + + +def test_implements_itransform(): + assert ITransform.providedBy(transform.msoffice_to_text()) + + +def test_name_and_mimetypes(): + t = transform.register() + assert t.name() == "msoffice_to_text" + assert t.output == "text/plain" + assert t.output_encoding == "utf-8" + assert t.inputs == ( + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.ms-excel", + "application/msexcel", + "application/x-msexcel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-powerpoint", + "application/powerpoint", + "application/mspowerpoint", + "application/x-mspowerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + ) + + +def test_success(monkeypatch, caplog): + run = fake_run(monkeypatch, stdout="Hello 日本語\n".encode()) + with caplog.at_level(logging.DEBUG, logger="c2.transform.msoffice"): + assert convert(DOCUMENT) == "Hello 日本語\n" + assert caplog.records == [] + args, kwargs = run.calls[0] + assert args == [ + "java", + "-Xmx512m", + "-Dfile.encoding=UTF-8", + "-Dstdout.encoding=UTF-8", + "-jar", + JAR, + ] + assert kwargs == {"input": DOCUMENT, "capture_output": True, "timeout": 60} + + +def test_jar_is_in_the_package(): + assert Path(JAR).is_file() + + +def test_stderr_with_exit_code_zero_is_success(monkeypatch): + fake_run(monkeypatch, stdout=b"text", stderr=b"WARNING: POI warning") + assert convert(DOCUMENT) == "text" + + +def test_empty_stdout_with_exit_code_zero_is_success(monkeypatch, caplog): + """A document without text gives empty output; it is not an error.""" + fake_run(monkeypatch, stdout=b"") + with caplog.at_level(logging.DEBUG, logger="c2.transform.msoffice"): + assert convert(DOCUMENT) == "" + assert [r.levelname for r in caplog.records] == ["DEBUG"] + + +def test_stdout_that_is_not_utf8_does_not_raise(monkeypatch): + fake_run(monkeypatch, stdout=b"ok \xff\xfe end") + assert convert(DOCUMENT) == "ok �� end" + + +def test_non_zero_exit_code(monkeypatch, caplog): + stderr = b"java.lang.IllegalArgumentException: UNKNOWN is not supported" + fake_run(monkeypatch, returncode=1, stdout=b"partial", stderr=stderr) + assert convert(DOCUMENT) == "" + assert [(r.levelname, r.getMessage()) for r in caplog.records] == [ + ("ERROR", "msdoc2txt failed with exit code 1") + ] + + +def test_stderr_is_not_logged(monkeypatch, caplog): + """A Java exception can quote the document, for example a part name.""" + stderr = b"org.apache.poi.POIXMLException: /word/secret-part-name.xml" + fake_run(monkeypatch, returncode=1, stderr=stderr) + with caplog.at_level(logging.DEBUG, logger="c2.transform.msoffice"): + assert convert(DOCUMENT) == "" + assert "secret-part-name" not in caplog.text + assert "secret document body" not in caplog.text + + +def test_timeout(monkeypatch, caplog): + fake_run( + monkeypatch, + error=subprocess.TimeoutExpired(["java"], 60, output=b"partial"), + ) + assert convert(DOCUMENT) == "" + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt timed out after 60 seconds" + ] + + +def test_java_not_found(monkeypatch, caplog): + fake_run(monkeypatch, error=FileNotFoundError(2, "No such file", "java")) + assert convert(DOCUMENT) == "" + assert [r.getMessage() for r in caplog.records] == [ + "msdoc2txt could not run: java was not found" + ] + + +def test_other_exception(monkeypatch, caplog): + fake_run(monkeypatch, error=OSError("Too many open files")) + assert convert(DOCUMENT) == "" + assert [r.getMessage() for r in caplog.records] == ["msdoc2txt could not run"] + assert caplog.records[0].exc_info is not None + + +def test_str_input_is_not_sent_to_java(monkeypatch, caplog): + run = fake_run(monkeypatch, stdout=b"never") + assert convert("not bytes") == "" + assert run.calls == [] + assert [r.getMessage() for r in caplog.records] == [ + "msoffice_to_text expects bytes, got str; no text is extracted" + ] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, 60), + ("", 60), + ("120", 120), + (" 5 ", 5), + ], +) +def test_timeout_from_environment(monkeypatch, value, expected): + if value is not None: + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", value) + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][1]["timeout"] == expected + + +@pytest.mark.parametrize("value", ["abc", "0", "-3", "1.5", "²"]) +def test_invalid_timeout_falls_back_to_default(monkeypatch, caplog, value): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_TIMEOUT", value) + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][1]["timeout"] == 60 + assert [r.getMessage() for r in caplog.records] == [ + f"Ignoring C2_TRANSFORM_MSOFFICE_TIMEOUT={value!r}: " + "expected a positive integer; using 60" + ] + + +def test_java_opts_from_environment(monkeypatch): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", "-Xmx1g -XX:+UseSerialGC") + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][0] == [ + "java", + "-Xmx1g", + "-XX:+UseSerialGC", + "-Dfile.encoding=UTF-8", + "-Dstdout.encoding=UTF-8", + "-jar", + JAR, + ] + + +def test_blank_java_opts_use_default(monkeypatch): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", " ") + run = fake_run(monkeypatch, stdout=b"x") + convert(DOCUMENT) + assert run.calls[0][0][1] == "-Xmx512m" + + +def test_java_opts_that_cannot_be_parsed(monkeypatch, caplog): + monkeypatch.setenv("C2_TRANSFORM_MSOFFICE_JAVA_OPTS", '-Xmx1g "-Dunclosed') + run = fake_run(monkeypatch, stdout=b"x") + assert convert(DOCUMENT) == "" + assert run.calls == [] + assert [r.getMessage() for r in caplog.records] == [ + "JVM options in C2_TRANSFORM_MSOFFICE_JAVA_OPTS could not be parsed" + ] + assert caplog.records[0].exc_info is None diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..05d74b4 --- /dev/null +++ b/tox.ini @@ -0,0 +1,82 @@ +[tox] +envlist = + ruff-check, + py{310,311,312,313}-Plone60, + py{310,311,312,313}-Plone61, + py{310,311,312,313,314}-Plone62, + +skip_missing_interpreters = True + + +[testenv] +skip_install = true + +commands_pre = + {envpython} -m pip install -c {toxinidir}/{env:constraints_file} -e "{toxinidir}[test]" + +commands = + coverage run {envbindir}/pytest -v -rs {posargs:{toxinidir}/tests} + +setenv = + COVERAGE_FILE=.coverage.{envname} + Plone60: constraints_file=constraints_plone60.txt + Plone61: constraints_file=constraints_plone61.txt + Plone62: constraints_file=constraints_plone62.txt + +# Set C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA=1 to turn the "java not found" skip +# into a failure (CI does this). JAVA_HOME is passed for setup-java. +passenv = + C2_TRANSFORM_MSOFFICE_REQUIRE_JAVA + JAVA_HOME + +deps = + Plone60: -rrequirements_plone60.txt + Plone61: -rrequirements_plone61.txt + Plone62: -rrequirements_plone62.txt + coverage[toml] + + +[testenv:ruff-check] +basepython = python3.12 +skip_install = True +commands_pre = +deps = + -cconstraints_plone62.txt + ruff + +commands = + ruff check --no-fix src tests + ruff format --check --diff src tests + + +[testenv:ruff-fix] +basepython = python3.12 +skip_install = True +commands_pre = +deps = + -cconstraints_plone62.txt + ruff + +commands = + ruff check --fix src tests + ruff format src tests + + +[testenv:coverage-report] +skip_install = true +basepython = python3.12 +commands_pre = + +deps = + coverage[toml] + -cconstraints_plone62.txt + +setenv = + COVERAGE_FILE=.coverage + +commands = + coverage erase + coverage combine + coverage html + coverage xml + coverage report