Skip to content

[python] Avoid concurrent import failures during split deserialization #9250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
XiaoHongbo-Hope wants to merge 3 commits into
base: master
Choose a base branch
from
+336 −40
Open

[python] Avoid concurrent import failures during split deserialization #9250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b521f1c
[python] Avoid concurrent import failures during split deserialization
XiaoHongbo-Hope Aug 16, 2026
2dd3805
[python] Cache lazy SQLContext export
XiaoHongbo-Hope Aug 16, 2026
e548f4c
[python] Lazify globalindex and tag inits to fix exposed import cycle…
XiaoHongbo-Hope Aug 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 35 additions & 9 deletions paimon-python/pypaimon/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,23 @@
# specific language governing permissions and limitations
# under the License.

import importlib
import sys
import threading

if sys.version_info[:2] < (3, 8):
try:
from pypaimon.manifest import fastavro_py36_compat # noqa: F401
except ImportError:
pass

from pypaimon.catalog.catalog_factory import CatalogFactory
from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem
from pypaimon.schema.schema import Schema
from pypaimon.tag.tag import Tag
from pypaimon.tag.tag_manager import TagManager
if sys.version_info[:2] < (3, 7):
# Module-level __getattr__ is unavailable before Python 3.7.
from pypaimon.catalog.catalog_factory import CatalogFactory
from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem
from pypaimon.schema.schema import Schema
from pypaimon.tag.tag import Tag
from pypaimon.tag.tag_manager import TagManager

__all__ = [
"PaimonVirtualFileSystem",
Expand All @@ -38,9 +42,31 @@
"SQLContext",
]

_LAZY_EXPORTS = {
"CatalogFactory": ("pypaimon.catalog.catalog_factory", "CatalogFactory"),
"PaimonVirtualFileSystem": (
"pypaimon.filesystem.pvfs", "PaimonVirtualFileSystem",
),
"Schema": ("pypaimon.schema.schema", "Schema"),
"Tag": ("pypaimon.tag.tag", "Tag"),
"TagManager": ("pypaimon.tag.tag_manager", "TagManager"),
"SQLContext": ("pypaimon_rust.datafusion", "SQLContext"),
}

# Unsynchronized lazy imports from two threads can acquire module locks in
# opposite orders (pypaimon.tag initializes both tag and tag_manager) and
# fail with _DeadlockError; the eager imports previously serialized this on
# the root module lock.
_LAZY_IMPORT_LOCK = threading.RLock()


def __getattr__(name):
if name == "SQLContext":
from pypaimon_rust.datafusion import SQLContext
return SQLContext
raise AttributeError("module 'pypaimon' has no attribute {}".format(name))
target = _LAZY_EXPORTS.get(name)
if target is None:
raise AttributeError(
"module 'pypaimon' has no attribute {}".format(name))
with _LAZY_IMPORT_LOCK:
if name not in globals():
module = importlib.import_module(target[0])
globals()[name] = getattr(module, target[1])
return globals()[name]
113 changes: 84 additions & 29 deletions paimon-python/pypaimon/globalindex/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,41 @@
# specific language governing permissions and limitations
# under the License.

from pypaimon.globalindex.global_index_result import GlobalIndexResult
from pypaimon.globalindex.global_index_reader import GlobalIndexReader, FieldRef
from pypaimon.globalindex.vector_search import VectorSearch
from pypaimon.globalindex.full_text_search import FullTextSearch
from pypaimon.globalindex.vector_search_result import (
ScoredGlobalIndexResult,
DictBasedScoredIndexResult,
ScoreGetter,
)
from pypaimon.globalindex.global_index_meta import GlobalIndexMeta, GlobalIndexIOMeta
from pypaimon.globalindex.global_index_evaluator import GlobalIndexEvaluator
from pypaimon.globalindex.data_evolution_global_index_scanner import (
DataEvolutionGlobalIndexScanner,
)
from pypaimon.globalindex.key_serializer import KeySerializer
from pypaimon.globalindex.memory_slice_input import MemorySliceInput
from pypaimon.globalindex.offset_global_index_reader import OffsetGlobalIndexReader
from pypaimon.globalindex.sorted_file_global_index_reader import SortedFileGlobalIndexReader
from pypaimon.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
from pypaimon.globalindex.sorted_index_file_meta import SortedIndexFileMeta
from pypaimon.globalindex.create_global_index import (
GlobalIndexBuilder,
create_global_index,
)
from pypaimon.globalindex.drop_global_index import (
GlobalIndexDropper,
drop_global_index,
)
from pypaimon.utils.range import Range
import importlib
import sys
import threading

if sys.version_info[:2] < (3, 7):
# Module-level __getattr__ is unavailable before Python 3.7.
from pypaimon.globalindex.global_index_result import GlobalIndexResult
from pypaimon.globalindex.global_index_reader import GlobalIndexReader, FieldRef
from pypaimon.globalindex.vector_search import VectorSearch
from pypaimon.globalindex.full_text_search import FullTextSearch
from pypaimon.globalindex.vector_search_result import (
ScoredGlobalIndexResult,
DictBasedScoredIndexResult,
ScoreGetter,
)
from pypaimon.globalindex.global_index_meta import GlobalIndexMeta, GlobalIndexIOMeta
from pypaimon.globalindex.global_index_evaluator import GlobalIndexEvaluator
from pypaimon.globalindex.data_evolution_global_index_scanner import (
DataEvolutionGlobalIndexScanner,
)
from pypaimon.globalindex.key_serializer import KeySerializer
from pypaimon.globalindex.memory_slice_input import MemorySliceInput
from pypaimon.globalindex.offset_global_index_reader import OffsetGlobalIndexReader
from pypaimon.globalindex.sorted_file_global_index_reader import SortedFileGlobalIndexReader
from pypaimon.globalindex.sorted_file_meta_selector import SortedFileMetaSelector
from pypaimon.globalindex.sorted_index_file_meta import SortedIndexFileMeta
from pypaimon.globalindex.create_global_index import (
GlobalIndexBuilder,
create_global_index,
)
from pypaimon.globalindex.drop_global_index import (
GlobalIndexDropper,
drop_global_index,
)
from pypaimon.utils.range import Range

__all__ = [
'GlobalIndexResult',
Expand All @@ -70,3 +76,52 @@
'drop_global_index',
'Range',
]

_MODULE_BY_EXPORT = {
'GlobalIndexResult': 'pypaimon.globalindex.global_index_result',
'GlobalIndexReader': 'pypaimon.globalindex.global_index_reader',
'FieldRef': 'pypaimon.globalindex.global_index_reader',
'VectorSearch': 'pypaimon.globalindex.vector_search',
'FullTextSearch': 'pypaimon.globalindex.full_text_search',
'ScoredGlobalIndexResult': 'pypaimon.globalindex.vector_search_result',
'DictBasedScoredIndexResult': 'pypaimon.globalindex.vector_search_result',
'ScoreGetter': 'pypaimon.globalindex.vector_search_result',
'GlobalIndexMeta': 'pypaimon.globalindex.global_index_meta',
'GlobalIndexIOMeta': 'pypaimon.globalindex.global_index_meta',
'GlobalIndexEvaluator': 'pypaimon.globalindex.global_index_evaluator',
'DataEvolutionGlobalIndexScanner':
'pypaimon.globalindex.data_evolution_global_index_scanner',
'KeySerializer': 'pypaimon.globalindex.key_serializer',
'MemorySliceInput': 'pypaimon.globalindex.memory_slice_input',
'OffsetGlobalIndexReader':
'pypaimon.globalindex.offset_global_index_reader',
'SortedFileGlobalIndexReader':
'pypaimon.globalindex.sorted_file_global_index_reader',
'SortedFileMetaSelector':
'pypaimon.globalindex.sorted_file_meta_selector',
'SortedIndexFileMeta': 'pypaimon.globalindex.sorted_index_file_meta',
'GlobalIndexBuilder': 'pypaimon.globalindex.create_global_index',
'create_global_index': 'pypaimon.globalindex.create_global_index',
'GlobalIndexDropper': 'pypaimon.globalindex.drop_global_index',
'drop_global_index': 'pypaimon.globalindex.drop_global_index',
'Range': 'pypaimon.utils.range',
}

# Eagerly importing the exports above builds a circular chain: submodules
# such as index_file_meta initialize this package on first import, while
# create_global_index and the scanner chain import those submodules back.
# Lazy resolution keeps this package init trivial; the lock serializes
# first-time imports racing from multiple threads.
_LAZY_IMPORT_LOCK = threading.RLock()


def __getattr__(name):
module_name = _MODULE_BY_EXPORT.get(name)
if module_name is None:
raise AttributeError(
"module 'pypaimon.globalindex' has no attribute {}".format(name))
with _LAZY_IMPORT_LOCK:
if name not in globals():
module = importlib.import_module(module_name)
globals()[name] = getattr(module, name)
return globals()[name]
32 changes: 30 additions & 2 deletions paimon-python/pypaimon/tag/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,35 @@
# specific language governing permissions and limitations
# under the License.

from pypaimon.tag.tag import Tag
from pypaimon.tag.tag_manager import TagManager
import importlib
import sys
import threading

if sys.version_info[:2] < (3, 7):
# Module-level __getattr__ is unavailable before Python 3.7.
from pypaimon.tag.tag import Tag
from pypaimon.tag.tag_manager import TagManager

__all__ = ["Tag", "TagManager"]

_MODULE_BY_EXPORT = {
"Tag": "pypaimon.tag.tag",
"TagManager": "pypaimon.tag.tag_manager",
}

# Eagerly importing both siblings here lets two threads acquire the module
# locks of tag and tag_manager in opposite orders and fail with
# _DeadlockError; lazy resolution keeps this package init trivial.
_LAZY_IMPORT_LOCK = threading.RLock()


def __getattr__(name):
module_name = _MODULE_BY_EXPORT.get(name)
if module_name is None:
raise AttributeError(
"module 'pypaimon.tag' has no attribute {}".format(name))
with _LAZY_IMPORT_LOCK:
if name not in globals():
module = importlib.import_module(module_name)
globals()[name] = getattr(module, name)
return globals()[name]
Loading
Loading