diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02bbe150e5..8032ee5f2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,12 @@ exclude: | (?x) # NOT INSTALLABLE ADDONS + ^base_import_async/| + ^queue_job_batch/| + ^queue_job_cron/| + ^queue_job_cron_jobrunner/| + ^queue_job_subscribe/| + ^test_queue_job_batch/| # END NOT INSTALLABLE ADDONS # Files and folders generated by bots, to avoid loops ^setup/|/static/description/index\.html$| diff --git a/queue_job/README.rst b/queue_job/README.rst index f9a974d192..70e9f6704e 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -21,13 +21,13 @@ Job Queue :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github - :target: https://github.com/OCA/queue/tree/19.0/queue_job + :target: https://github.com/OCA/queue/tree/20.0/queue_job :alt: OCA/queue .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/queue-19-0/queue-19-0-queue_job + :target: https://translation.odoo-community.org/projects/queue-20-0/queue-20-0-queue_job :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png - :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=19.0 + :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=20.0 :alt: Try me on Runboat |badge1| |badge2| |badge3| |badge4| |badge5| @@ -685,7 +685,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -747,6 +747,6 @@ Current `maintainers `__: |maintainer-guewen| |maintainer-sbidoul| -This module is part of the `OCA/queue `_ project on GitHub. +This module is part of the `OCA/queue `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/queue_job/controllers/main.py b/queue_job/controllers/main.py index c3e49ca04e..4ace83782b 100644 --- a/queue_job/controllers/main.py +++ b/queue_job/controllers/main.py @@ -8,7 +8,6 @@ import traceback from contextlib import contextmanager from io import StringIO -import psycopg2 from psycopg2 import OperationalError, errorcodes from werkzeug.exceptions import BadRequest, Forbidden @@ -161,16 +160,10 @@ def retry_postpone(job, message, seconds=None): job.store() try: - try: - cls._try_perform_job(env, job) - except OperationalError as err: - # Automatically retry the typical transaction serialization - # errors - if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: - raise - - _logger.debug("%s OperationalError, postponed", job) - raise RetryableJobError(err.pgerror, seconds=PG_RETRY) from err + # Job.perform() turns the retryable PostgreSQL concurrency errors + # into RetryableJobError itself, so that max_retries applies to + # them; there is nothing to convert here. + cls._try_perform_job(env, job) except RetryableJobError as err: # delay the job later, requeue diff --git a/queue_job/job.py b/queue_job/job.py index 04081976c5..0b2955efb3 100644 --- a/queue_job/job.py +++ b/queue_job/job.py @@ -12,6 +12,8 @@ from datetime import datetime, timedelta from random import randint +from psycopg2 import OperationalError, errorcodes + import odoo from odoo.models import BaseModel @@ -57,6 +59,13 @@ def _rebind_to_cr(value, cr): DEFAULT_PRIORITY = 10 # used by the PriorityQueue to sort the jobs DEFAULT_MAX_RETRIES = 5 +PG_RETRY = 5 # seconds to wait before retrying a serialization failure +# odoo.service.model stopped exporting these in 20.0 +PG_CONCURRENCY_ERRORS_TO_RETRY = ( + errorcodes.LOCK_NOT_AVAILABLE, + errorcodes.SERIALIZATION_FAILURE, + errorcodes.DEADLOCK_DETECTED, +) RETRY_INTERVAL = 10 * 60 # seconds _logger = logging.getLogger(__name__) @@ -511,12 +520,22 @@ def perform(self): """ self.retry += 1 try: - if self.job_config.allow_commit: - env_context_manager = self.in_temporary_env() - else: - env_context_manager = nullcontext() - with env_context_manager: - self.result = self.func(*tuple(self.args), **self.kwargs) + try: + if self.job_config.allow_commit: + env_context_manager = self.in_temporary_env() + else: + env_context_manager = nullcontext() + with env_context_manager: + self.result = self.func(*tuple(self.args), **self.kwargs) + except OperationalError as err: + # The typical transaction serialization errors are retryable, + # and converting them here rather than in the runners is what + # subjects them to max_retries below: a job raising them from + # outside perform() is postponed without ever consulting the + # limit, so it retries for ever. + if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: + raise + raise RetryableJobError(err.pgerror, seconds=PG_RETRY) from err except RetryableJobError as err: if err.ignore_retry: self.retry -= 1 diff --git a/queue_job/static/description/index.html b/queue_job/static/description/index.html index 58b23819d8..6267ae58a6 100644 --- a/queue_job/static/description/index.html +++ b/queue_job/static/description/index.html @@ -3,7 +3,7 @@ -README.rst +Job Queue