diff --git a/graalpython/com.oracle.graal.python.cext/src/exceptions.c b/graalpython/com.oracle.graal.python.cext/src/exceptions.c index 6bed9efbf6..30f33d8e08 100644 --- a/graalpython/com.oracle.graal.python.cext/src/exceptions.c +++ b/graalpython/com.oracle.graal.python.cext/src/exceptions.c @@ -47,10 +47,17 @@ PY_EXCEPTIONS #undef EXCEPTION +#ifdef MS_WINDOWS +PyObject* PyExc_WindowsError = NULL; +#endif + Py_LOCAL_SYMBOL void initialize_exceptions() { #define EXCEPTION(NAME) PyExc_##NAME = (PyObject*) GraalPyPrivate_Type(#NAME); PY_EXCEPTIONS #undef EXCEPTION +#ifdef MS_WINDOWS + PyExc_WindowsError = PyExc_OSError; +#endif } GraalPy_CAPI_HELPER_SYMBOL PyObject * diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py index 1d7f81df54..dcd47590b2 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py @@ -211,6 +211,20 @@ def raise_error(): class TestPyErr(CPyExtTestCase): + if sys.platform == 'win32': + test_PyExc_WindowsError = CPyExtFunction( + lambda args: OSError, + lambda: ((),), + code="""PyObject* get_PyExc_WindowsError(void) { + return Py_NewRef(PyExc_WindowsError); + } + """, + resultspec='O', + argspec='', + arguments=[], + callfunction='get_PyExc_WindowsError', + ) + test_PyErr_SetString = CPyExtFunctionVoid( _reference_setstring, diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py index 6eb0c52691..012b29d868 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py @@ -95,6 +95,72 @@ def compare_unicode_bytes_with_kind(x, y): class TestPyMemoryView(CPyExtTestCase): + def test_native_subclass_without_getbuffer(self): + for name, buffer_procs in ( + ('NativeWithoutBuffer', 'NULL'), + ('NativeWithEmptyBuffer', '&empty_buffer'), + ): + with self.subTest(buffer_procs=buffer_procs): + NativeType = CPyExtType( + name, + ''' + static PyBufferProcs empty_buffer = {NULL, NULL}; + ''', + cmembers='int value;', + tp_flags='Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE', + tp_as_buffer=buffer_procs, + ) + + class Iterable(NativeType): + def __iter__(self): + return iter((97, 98, 99)) + + for obj in (NativeType(), Iterable()): + assert (bytearray() == obj) is False + assert (bytearray() != obj) is True + with self.assertRaises(TypeError): + memoryview(obj) + + data = bytearray(b'x') + data.extend(Iterable()) + assert data == b'xabc' + + test_heap_type_buffer_procs = CPyExtFunction( + lambda args: b'abc', + lambda: ((type('HeapBuffer', (), {}),),), + code=''' + static int heap_getbuffer(PyObject *obj, Py_buffer *view, int flags) { + static char data[] = "abc"; + return PyBuffer_FillInfo(view, obj, data, 3, 0, flags); + } + static PyObject *heap_type_buffer_procs(PyObject *type) { + PyTypeObject *tp = (PyTypeObject *)type; + if (tp->tp_as_buffer == NULL || + tp->tp_as_buffer != &((PyHeapTypeObject *)tp)->as_buffer) { + PyErr_SetString(PyExc_AssertionError, "heap type must own its buffer procs"); + return NULL; + } + getbufferproc previous = tp->tp_as_buffer->bf_getbuffer; + tp->tp_as_buffer->bf_getbuffer = heap_getbuffer; + PyObject *obj = PyObject_CallNoArgs(type); + PyObject *result = NULL; + if (obj != NULL) { + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) == 0) { + result = PyBytes_FromStringAndSize(view.buf, view.len); + PyBuffer_Release(&view); + } + Py_DECREF(obj); + } + tp->tp_as_buffer->bf_getbuffer = previous; + return result; + } + ''', + argspec='O', + arguments=['PyObject* type'], + callfunction='heap_type_buffer_procs', + ) + test_memoryview_read = CPyExtFunction( lambda args: args[1], lambda: ( diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tp_slots.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tp_slots.py index fb2263deca..a4c25de0d0 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tp_slots.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tp_slots.py @@ -1820,3 +1820,77 @@ def test_cmp(op_lambda, rop_lambda, op_name, rop_name): assert create_mock(ne=1) == create_mock(eq=1) assert create_mock(ne=1, eq=0) != create_mock(ne=0, eq=1) assert not create_mock(ne=1, eq=0) == create_mock(ne=0, eq=1) + + +def test_heap_type_slot_groups(): + checker = CPyExtType("HeapSlotGroupChecker", """ + static PyObject *check_groups(PyObject *self, PyObject *type) { + PyTypeObject *tp = (PyTypeObject *)type; + PyHeapTypeObject *heap = (PyHeapTypeObject *)type; + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) || + tp->tp_as_async != &heap->as_async || + tp->tp_as_number != &heap->as_number || + tp->tp_as_sequence != &heap->as_sequence || + tp->tp_as_mapping != &heap->as_mapping || + tp->tp_as_buffer != &heap->as_buffer) { + PyErr_SetString(PyExc_AssertionError, "heap type must own all slot groups"); + return NULL; + } + Py_RETURN_NONE; + } + static PyObject *call_slots(PyObject *self, PyObject *obj) { + PyTypeObject *tp = Py_TYPE(obj); + PyObject *sum = NULL, *item = NULL, *awaitable = NULL, *result = NULL; + if (tp->tp_as_number->nb_add == NULL || + tp->tp_as_sequence->sq_length == NULL || + tp->tp_as_mapping->mp_subscript == NULL || + tp->tp_as_async->am_await == NULL) { + PyErr_SetString(PyExc_AssertionError, "populated slots must be preserved"); + return NULL; + } + sum = tp->tp_as_number->nb_add(obj, obj); + if (sum == NULL) goto done; + Py_ssize_t length = tp->tp_as_sequence->sq_length(obj); + if (length < 0) goto done; + item = tp->tp_as_mapping->mp_subscript(obj, Py_None); + if (item == NULL) goto done; + awaitable = tp->tp_as_async->am_await(obj); + if (awaitable == NULL) goto done; + result = Py_BuildValue("(OnOO)", sum, length, item, awaitable); + done: + Py_XDECREF(sum); + Py_XDECREF(item); + Py_XDECREF(awaitable); + return result; + } + """, tp_methods=''' + {"check", (PyCFunction)check_groups, METH_O, NULL}, + {"call_slots", (PyCFunction)call_slots, METH_O, NULL} + ''')() + + class Empty: + pass + + class Populated: + def __add__(self, other): + return 42 + + def __len__(self): + return 3 + + def __getitem__(self, key): + return 17 + + def __await__(self): + return iter(()) + + class Inherited(Populated): + pass + + for cls in (Empty, Populated, Inherited, *(type('HeapSubclass', (base,), {}) + for base in (int, list, dict, bytearray))): + checker.check(cls) + for cls in (Populated, Inherited): + total, length, item, awaitable = checker.call_slots(cls()) + assert (total, length, item) == (42, 3, 17) + assert list(awaitable) == [] diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_imports.py b/graalpython/com.oracle.graal.python.test/src/tests/test_imports.py index e9dcafd0f3..3cd52a344c 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_imports.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_imports.py @@ -120,6 +120,90 @@ def test_import_star_all(): assert 'unlistedProperty' not in g +def test_import_star_non_module(): + class Imported: + __all__ = ('public', '_private') + public = 42 + _private = 43 + + name = '_test_import_star_non_module' + sys.modules[name] = Imported() + try: + namespace = {} + exec('from _test_import_star_non_module import *', namespace) + assert namespace['public'] == 42 + assert namespace['_private'] == 43 + del Imported.__all__ + sys.modules[name].public = 44 + sys.modules[name]._private = 45 + namespace = {} + exec('from _test_import_star_non_module import *', namespace) + assert namespace['public'] == 44 + assert '_private' not in namespace + finally: + del sys.modules[name] + + +def test_import_star_materializes_mapping_keys(): + for fail in (False, True): + events = [] + + class Namespace: + def keys(self): + events.append('keys started') + yield 'public' + if fail: + raise RuntimeError('keys failed') + events.append('keys ended') + + class Imported: + @property + def __dict__(self): + return Namespace() + + def __getattr__(self, name): + if name == 'public': + events.append('attribute read') + return tuple(events) + raise AttributeError(name) + + name = '_test_import_star_mapping_keys' + sys.modules[name] = Imported() + try: + namespace = {} + if fail: + try: + exec('from _test_import_star_mapping_keys import *', namespace) + except RuntimeError as exc: + assert str(exc) == 'keys failed' + else: + assert False, 'key iteration errors must propagate' + assert events == ['keys started'] + assert 'public' not in namespace + else: + exec('from _test_import_star_mapping_keys import *', namespace) + assert namespace['public'] == ('keys started', 'keys ended', 'attribute read') + finally: + del sys.modules[name] + + +def test_import_star_missing_export(): + class Imported: + __all__ = ('missing',) + + name = '_test_import_star_missing_export' + sys.modules[name] = Imported() + try: + try: + exec('from _test_import_star_missing_export import *', {}) + except AttributeError: + pass + else: + assert False, 'missing exports must raise AttributeError' + finally: + del sys.modules[name] + + def test_import_as_local(): from sys import path from sys import path as mypath diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_nt.py b/graalpython/com.oracle.graal.python.test/src/tests/test_nt.py index 2842facd23..ec10103859 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_nt.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_nt.py @@ -51,6 +51,71 @@ @unittest.skipUnless(sys.platform == 'win32', 'requires Windows') class NtTests(unittest.TestCase): + def test_windows_path_helpers(self): + import nt + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'MiXeD_\u03be_\U0001f40d' + path.touch() + expected = '\\\\?\\' + str(path) + for value in (str(path), path, os.fsencode(path)): + with self.subTest(value=value): + result = nt._getfinalpathname(value) + self.assertEqual(result, os.fsencode(expected) if isinstance(value, bytes) else expected) + self.assertEqual(nt._findfirstfile(value), path.name) + self.assertEqual(nt._findfirstfile(str(path).lower()), path.name) + for helper in (nt._getfinalpathname, nt._findfirstfile): + with self.assertRaises(TypeError): + helper(path=path) + missing = str(Path(directory) / 'missing') + with self.assertRaises(FileNotFoundError) as caught: + helper(missing) + self.assertEqual(caught.exception.winerror, 2) + self.assertEqual(caught.exception.filename, missing) + with self.assertRaises(ValueError): + helper('bad\0path') + + def test_findfirstfile_dot_components(self): + import nt + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) + child = parent / 'MiXeD_directory' + child.mkdir() + for path, expected in ( + (str(child) + '\\.', child.name), + (str(child) + '\\..', parent.name), + (str(child).lower() + '\\.', child.name), + (str(child) + '\\..\\' + child.name + '\\.', child.name), + ): + with self.subTest(path=path): + self.assertEqual(nt._findfirstfile(path), expected) + + def test_windows_path_helpers_invalid_path(self): + import nt + with tempfile.TemporaryDirectory() as directory: + invalid = str(Path(directory) / 'bad|name') + for helper in (nt._getfinalpathname, nt._findfirstfile): + with self.subTest(helper=helper): + with self.assertRaises(OSError) as caught: + helper(invalid) + self.assertEqual(caught.exception.errno, errno.EINVAL) + self.assertEqual(caught.exception.winerror, 123) + self.assertEqual(caught.exception.filename, invalid) + self.assertEqual(os.path.realpath(invalid), invalid) + with self.assertRaises(OSError): + os.path.realpath(invalid, strict=True) + + @unittest.skipIf(JAVA_BACKEND, 'requires the Windows native POSIX backend') + def test_getfinalpathname_long_path(self): + import nt + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + for _ in range(6): + path /= 'segment_' + 'x' * 45 + path.mkdir() + self.assertGreater(len(str(path)), 260) + self.assertEqual(nt._getfinalpathname(path), '\\\\?\\' + str(path)) + self.assertEqual(nt._findfirstfile(path), path.name) + @unittest.skipIf(JAVA_BACKEND, 'requires the Windows native POSIX backend') def test_app_execution_alias_stat(self): local_app_data = os.environ.get('LOCALAPPDATA') @@ -215,6 +280,24 @@ def test_fchmod(self): @unittest.skipUnless(sys.platform == 'win32', 'requires Windows') class NtSymlinkTests(unittest.TestCase): + @unittest.skipIf(JAVA_BACKEND, 'requires the Windows native POSIX backend') + def test_realpath_symlink_missing_leaf(self): + import nt + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / 'target' + target.mkdir() + link = root / 'link' + self.create_link('target', link, target_is_directory=True) + self.assertEqual(nt._getfinalpathname(link), '\\\\?\\' + str(target)) + self.assertEqual(nt._findfirstfile(link), 'link') + self.assertEqual(os.path.realpath(link / 'missing', strict=os.path.ALLOW_MISSING), str(target / 'missing')) + self.assertEqual(os.path.realpath(os.fsencode(link / 'missing'), strict=os.path.ALLOW_MISSING), + os.fsencode(target / 'missing')) + with self.assertRaises(FileNotFoundError): + os.path.realpath(link / 'missing', strict=True) + os.rmdir(link) + def create_link(self, source, destination, **kwargs): try: os.symlink(source, destination, **kwargs) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py b/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py index 12b7826de8..5098829863 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py @@ -43,6 +43,24 @@ from tests.util import storage_to_native +def test_shutdown_write_sends_eof(): + for how in (socket.SHUT_WR, socket.SHUT_RDWR): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + with socket.create_connection(listener.getsockname()) as client: + peer, _ = listener.accept() + with peer: + peer.settimeout(5) + client.settimeout(5) + client.shutdown(how) + assert peer.recv(1) == b"" + if how == socket.SHUT_WR: + # Shutting down writes must leave the receive direction usable. + peer.sendall(b"r") + assert client.recv(1) == b"r" + + def test_inet_aton(): assert socket.inet_aton('127.255.1.2') == bytes([127, 255, 1, 2]) assert socket.inet_aton('127.0Xff.1.2') == bytes([127, 255, 1, 2]) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_type.py b/graalpython/com.oracle.graal.python.test/src/tests/test_type.py index ed9e5eebdb..c9f2411987 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_type.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_type.py @@ -38,6 +38,7 @@ # SOFTWARE. import sys +from textwrap import dedent from unittest import skipIf @@ -577,3 +578,41 @@ def f(self): assert Outer.__static_attributes__ == ("outer",) assert Outer.Inner.__static_attributes__ == ("inner",) + + +def test_type_alias_module_without_name(): + namespace = {} + exec("type Alias = int", namespace) + assert namespace["Alias"].__module__ is None + + +def test_type_alias_module_with_name(): + namespace = {"__name__": "alias_module"} + exec("type Alias = int", namespace) + assert namespace["Alias"].__module__ == "alias_module" + + +def test_type_alias_module_after_value_evaluation(): + for module_name in (None, "alias_module"): + for read_module_first in (False, True): + namespace = {} if module_name is None else {"__name__": module_name} + exec("type Alias = int", namespace) + alias = namespace["Alias"] + if read_module_first: + assert alias.__module__ == module_name + assert alias.__value__ is int + assert alias.__module__ == module_name + + +def test_type_alias_dataclass_without_module_name(): + namespace = {} + exec(dedent(""" + from dataclasses import dataclass + type Alias = int + @dataclass + class Example: + value: Alias + """), namespace) + cls = namespace["Example"] + assert cls(42).value == 42 + assert cls.__doc__ == "Example(value: Alias)" diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_os.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_os.txt index 929592dad6..a5afceacea 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_os.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_os.txt @@ -174,7 +174,7 @@ test.test_os.TestPEP519PurePython.test_return_bytes @ darwin-arm64,linux-aarch64 test.test_os.TestPEP519PurePython.test_return_string @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_os.TestScandir.test_attributes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_os.TestScandir.test_bad_path_type @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -test.test_os.TestScandir.test_broken_symlink @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github +test.test_os.TestScandir.test_broken_symlink @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_os.TestScandir.test_bytes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_os.TestScandir.test_bytes_like @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_os.TestScandir.test_close @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java index 2c62d1340b..38268126cd 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java @@ -54,10 +54,12 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.OpaquePathToBytesNode; import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.PathConversionNode; import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.PosixPath; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.lib.PyOSFSPathNode; +import com.oracle.graal.python.nodes.PConstructAndRaiseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryClinicBuiltinNode; @@ -65,6 +67,7 @@ import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToJavaStringNode; import com.oracle.graal.python.nodes.util.PosixSupportNodes; +import com.oracle.graal.python.runtime.PosixSupport.PosixException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; @@ -115,6 +118,55 @@ private TruffleString getTruffleStringPath(String fspath) { } } + @Builtin(name = "_getfinalpathname", minNumOfPositionalArgs = 1, numOfPositionalOnlyArgs = 1, parameterNames = {"path"}) + @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"}) + @GenerateNodeFactory + abstract static class GetFinalPathNameNode extends PythonUnaryClinicBuiltinNode { + @Specialization + Object getFinalPathName(VirtualFrame frame, PosixPath path, + @Bind Node inliningTarget, + @Cached OpaquePathToBytesNode opaquePathToBytesNode, + @Cached PosixSupportNodes.GetPathAsStringNode getPathAsStringNode, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + try { + Object result = getPosixSupport().getFinalPathName(path.value); + if (path.wasBufferLike) { + return opaquePathToBytesNode.execute(inliningTarget, getPosixSupport(), result, getLanguage()); + } + return getPathAsStringNode.execute(inliningTarget, getPosixSupport(), result); + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path.originalObject); + } + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return NtModuleBuiltinsClinicProviders.GetFinalPathNameNodeClinicProviderGen.INSTANCE; + } + } + + @Builtin(name = "_findfirstfile", minNumOfPositionalArgs = 1, numOfPositionalOnlyArgs = 1, parameterNames = {"path"}) + @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"}) + @GenerateNodeFactory + abstract static class FindFirstFileNode extends PythonUnaryClinicBuiltinNode { + @Specialization + Object findFirstFile(VirtualFrame frame, PosixPath path, + @Bind Node inliningTarget, + @Cached PosixSupportNodes.GetPathAsStringNode getPathAsStringNode, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + try { + return getPathAsStringNode.execute(inliningTarget, getPosixSupport(), getPosixSupport().findFirstFile(path.value)); + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path.originalObject); + } + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return NtModuleBuiltinsClinicProviders.FindFirstFileNodeClinicProviderGen.INSTANCE; + } + } + @Builtin(name = "_path_splitroot", minNumOfPositionalArgs = 1, parameterNames = {"path"}) @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"}) @GenerateNodeFactory diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java index 2b1dea273a..637655c41f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java @@ -50,6 +50,7 @@ import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbol.FUN_PY_TYPE_GENERIC_ALLOC; import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbol.FUN_SUBTYPE_TRAVERSE; import static com.oracle.graal.python.builtins.objects.cext.structs.CConstants.PYLONG_BITS_IN_DIGIT; +import static com.oracle.graal.python.builtins.objects.cext.structs.CFields.PyBufferProcs__bf_getbuffer; import static com.oracle.graal.python.builtins.objects.cext.structs.CFields.PyFloatObject__ob_fval; import static com.oracle.graal.python.builtins.objects.cext.structs.CFields.PyModuleDef__m_clear; import static com.oracle.graal.python.builtins.objects.cext.structs.CFields.PyModuleDef__m_doc; @@ -1366,8 +1367,8 @@ public abstract static class HasNativeBufferNode extends PNodeWithContext { @Specialization static boolean readTpAsBuffer(PythonAbstractNativeObject object) { long type = readPtrField(object.getPtr(), PyObject__ob_type); - long result = readPtrField(type, PyTypeObject__tp_as_buffer); - return result != NULLPTR; + long bufferProcs = readPtrField(type, PyTypeObject__tp_as_buffer); + return bufferProcs != NULLPTR && readPtrField(bufferProcs, PyBufferProcs__bf_getbuffer) != NULLPTR; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/ToNativeTypeNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/ToNativeTypeNode.java index 300fce9c11..5e2ffa9dc8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/ToNativeTypeNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/ToNativeTypeNode.java @@ -107,10 +107,14 @@ public abstract class ToNativeTypeNode { - private static long allocatePyAsyncMethods(TpSlots slots) { - long mem = CStructAccess.allocate(CStructs.PyAsyncMethods); - writeGroupSlots(CFields.PyTypeObject__tp_as_async, slots, mem); - return mem; + private static long initializeSlotGroup(TpSlots slots, boolean hasSlots, long typePointer, boolean heaptype, + CFields heapField, CFields groupField, CStructs groupStruct) { + if (!heaptype && !hasSlots) { + return NULLPTR; + } + long groupPointer = heaptype ? typePointer + heapField.offset() : CStructAccess.allocate(groupStruct); + writeGroupSlots(groupField, slots, groupPointer); + return groupPointer; } private static void writeGroupSlots(CFields groupField, TpSlots slots, long groupPointer) { @@ -121,24 +125,6 @@ private static void writeGroupSlots(CFields groupField, TpSlots slots, long grou } } - private static long allocatePyMappingMethods(TpSlots slots) { - long mem = CStructAccess.allocate(CStructs.PyMappingMethods); - writeGroupSlots(CFields.PyTypeObject__tp_as_mapping, slots, mem); - return mem; - } - - private static long allocatePyNumberMethods(TpSlots slots) { - long mem = CStructAccess.allocate(CStructs.PyNumberMethods); - writeGroupSlots(CFields.PyTypeObject__tp_as_number, slots, mem); - return mem; - } - - private static long allocatePySequenceMethods(TpSlots slots) { - long mem = CStructAccess.allocate(CStructs.PyNumberMethods); - writeGroupSlots(CFields.PyTypeObject__tp_as_sequence, slots, mem); - return mem; - } - private static long lookup(PythonManagedClass clazz, CFields member, HiddenAttr hiddenName) { return lookupNativeMemberInMRO(clazz, member, hiddenName); } @@ -196,11 +182,25 @@ static void initializeType(PythonManagedClass clazz, long mem, boolean heaptype, } else { weaklistoffset = lookupSize(clazz, PyTypeObject__tp_weaklistoffset, HiddenAttr.WEAKLISTOFFSET); } - long asAsync = slots.has_as_async() ? allocatePyAsyncMethods(slots) : NULLPTR; - long asNumber = slots.has_as_number() ? allocatePyNumberMethods(slots) : NULLPTR; - long asSequence = slots.has_as_sequence() ? allocatePySequenceMethods(slots) : NULLPTR; - long asMapping = slots.has_as_mapping() ? allocatePyMappingMethods(slots) : NULLPTR; + // Like CPython's type_new_alloc, every heap type owns all slot groups, + // even when empty. Extensions may populate the embedded structures. + long asAsync = initializeSlotGroup(slots, slots.has_as_async(), mem, heaptype, + CFields.PyHeapTypeObject__as_async, CFields.PyTypeObject__tp_as_async, CStructs.PyAsyncMethods); + long asNumber = initializeSlotGroup(slots, slots.has_as_number(), mem, heaptype, + CFields.PyHeapTypeObject__as_number, CFields.PyTypeObject__tp_as_number, CStructs.PyNumberMethods); + long asSequence = initializeSlotGroup(slots, slots.has_as_sequence(), mem, heaptype, + CFields.PyHeapTypeObject__as_sequence, CFields.PyTypeObject__tp_as_sequence, CStructs.PySequenceMethods); + long asMapping = initializeSlotGroup(slots, slots.has_as_mapping(), mem, heaptype, + CFields.PyHeapTypeObject__as_mapping, CFields.PyTypeObject__tp_as_mapping, CStructs.PyMappingMethods); long asBuffer = lookup(clazz, PyTypeObject__tp_as_buffer, HiddenAttr.AS_BUFFER); + if (heaptype) { + long heapBuffer = CStructAccess.getFieldPtr(mem, CFields.PyHeapTypeObject__as_buffer); + if (asBuffer != NULLPTR) { + writePtrField(heapBuffer, CFields.PyBufferProcs__bf_getbuffer, readPtrField(asBuffer, CFields.PyBufferProcs__bf_getbuffer)); + writePtrField(heapBuffer, CFields.PyBufferProcs__bf_releasebuffer, readPtrField(asBuffer, CFields.PyBufferProcs__bf_releasebuffer)); + } + asBuffer = heapBuffer; + } writeLongField(mem, CFields.PyTypeObject__tp_weaklistoffset, weaklistoffset); writePtrField(mem, CFields.PyTypeObject__tp_dealloc, lookup(clazz, PyTypeObject__tp_dealloc, HiddenAttr.DEALLOC)); writeLongField(mem, CFields.PyTypeObject__tp_vectorcall_offset, lookupSize(clazz, PyTypeObject__tp_vectorcall_offset, HiddenAttr.VECTORCALL_OFFSET)); @@ -292,11 +292,6 @@ static void initializeType(PythonManagedClass clazz, long mem, boolean heaptype, if (heaptype) { assert (flags & TypeFlags.HEAPTYPE) != 0; - writePtrField(mem, CFields.PyHeapTypeObject__as_async, asAsync); - writePtrField(mem, CFields.PyHeapTypeObject__as_number, asNumber); - writePtrField(mem, CFields.PyHeapTypeObject__as_mapping, asMapping); - writePtrField(mem, CFields.PyHeapTypeObject__as_sequence, asSequence); - writePtrField(mem, CFields.PyHeapTypeObject__as_buffer, asBuffer); writePtrField(mem, CFields.PyHeapTypeObject__ht_name, PythonToNativeInternalNode.executeNewRefUncached(clazz.getName())); writePtrField(mem, CFields.PyHeapTypeObject__ht_qualname, PythonToNativeInternalNode.executeNewRefUncached(clazz.getQualName())); writePtrField(mem, CFields.PyHeapTypeObject__ht_module, NULLPTR); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java index a7d0b0a2e6..fba257abe3 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java @@ -556,6 +556,11 @@ static long parse(VirtualFrame frame, Node inliningTarget, Object seconds, if (timeout < 0) { throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.TIMEOUT_VALUE_OUT_OF_RANGE); } + // CPython's socket_parse_timeout requires milliseconds to fit into an int on + // platforms without poll(), including Windows. Round up as for _PyTime_ROUND_TIMEOUT. + if (PosixConstants.IS_WIN32 && timeout > Integer.MAX_VALUE * 1_000_000L) { + throw raiseNode.raise(inliningTarget, OverflowError, ErrorMessages.TIMEOUT_DOESNT_FIT_INTO_C_TIMEVAL); + } return timeout; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/typing/TypeAliasTypeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/typing/TypeAliasTypeBuiltins.java index 214a07368f..bb13c9522d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/typing/TypeAliasTypeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/typing/TypeAliasTypeBuiltins.java @@ -47,7 +47,7 @@ import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___NAME__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___PARAMETERS__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___TYPE_PARAMS__; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NAME__; +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REDUCE__; import java.util.List; @@ -73,7 +73,7 @@ import com.oracle.graal.python.builtins.objects.type.slots.TpSlotBinaryOp.BinaryOpBuiltinNode; import com.oracle.graal.python.builtins.objects.types.GenericTypeNodes.UnionTypeOrNode; import com.oracle.graal.python.builtins.objects.typing.TypeAliasTypeBuiltinsClinicProviders.TypeAliasTypeNodeClinicProviderGen; -import com.oracle.graal.python.lib.PyObjectGetItem; +import com.oracle.graal.python.lib.PyObjectGetAttr; import com.oracle.graal.python.lib.PyTupleCheckNode; import com.oracle.graal.python.nodes.PGuards; import com.oracle.graal.python.nodes.PRaiseNode; @@ -195,7 +195,6 @@ static Object doEvaluate(VirtualFrame frame, PTypeAliasType self, @Cached CallNode callNode) { assert self.computeValue != null; self.value = callNode.execute(frame, self.computeValue); - self.computeValue = null; return self.value; } } @@ -211,9 +210,9 @@ static Object doModule(PTypeAliasType self) { @Specialization(guards = {"self.module == null", "self.computeValue != null"}) static Object doComputed(VirtualFrame frame, PTypeAliasType self, @Bind Node inliningTarget, - @Cached PyObjectGetItem getItemNode) { - if (self.computeValue instanceof PFunction fun && fun.getGlobals() != null) { - return getItemNode.execute(frame, inliningTarget, fun.getGlobals(), T___NAME__); + @Cached PyObjectGetAttr getAttrNode) { + if (self.computeValue instanceof PFunction fun) { + return getAttrNode.execute(frame, inliningTarget, fun, T___MODULE__); } return PNone.NONE; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java index 66fe39592b..43d0685324 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java @@ -762,6 +762,7 @@ public abstract class ErrorMessages { public static final TruffleString TIMED_OUT = tsLiteral("timed out"); public static final TruffleString TIMEOUT_VALUE_MUST_BE_POSITIVE = tsLiteral("timeout value must be positive"); public static final TruffleString TIMEOUT_VALUE_OUT_OF_RANGE = tsLiteral("Timeout value out of range"); + public static final TruffleString TIMEOUT_DOESNT_FIT_INTO_C_TIMEVAL = tsLiteral("timeout doesn't fit into C timeval"); public static final TruffleString TIMEOUT_VALUE_TOO_LARGE = tsLiteral("timeout value is too large"); public static final TruffleString TIMESTAMP_OUT_OF_RANGE = tsLiteral("timestamp out of range for platform time_t"); public static final TruffleString TOLERANCE_MUST_NON_NEGATIVE = tsLiteral("tolerances must be non-negative"); @@ -1710,4 +1711,5 @@ public abstract class ErrorMessages { public static final TruffleString S_MUST_BE_CALLABLE = tsLiteral("%s must be callable"); public static final TruffleString METHOD_CANNOT_BE_BOTH_CLASS_AND_STATIC = tsLiteral("method cannot be both class and static"); public static final TruffleString CANNOT_BE_USED_IN_MAIN_INTERP = tsLiteral("_imp._override_multi_interp_extensions_check() cannot be used in the main interpreter"); + public static final TruffleString IMPORT_STAR_NO_DICT_OR_ALL = tsLiteral("from-import-* object has no __dict__ and no __all__"); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/ImportStarNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/ImportStarNode.java index 4b36baf96a..b14dd41adf 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/ImportStarNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/ImportStarNode.java @@ -25,30 +25,32 @@ */ package com.oracle.graal.python.nodes.bytecode; +import static com.oracle.graal.python.builtins.PythonBuiltinClassType.ImportError; import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TypeError; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___ALL__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DICT__; +import static com.oracle.graal.python.nodes.SpecialMethodNames.T_KEYS; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.tsArray; +import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.function.PArguments; -import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.lib.IteratorExhausted; import com.oracle.graal.python.lib.PyIterNextNode; +import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.lib.PyObjectGetAttr; import com.oracle.graal.python.lib.PyObjectGetItem; import com.oracle.graal.python.lib.PyObjectGetIter; +import com.oracle.graal.python.lib.PyObjectLookupAttr; import com.oracle.graal.python.lib.PyObjectSetItem; import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PRaiseNode; +import com.oracle.graal.python.nodes.builtins.ListNodes.FastConstructListNode; import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; -import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; -import com.oracle.graal.python.nodes.object.GetOrCreateDictNode; import com.oracle.graal.python.nodes.statement.AbstractImportNode; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; -import com.oracle.graal.python.runtime.exception.PException; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.dsl.Bind; @@ -80,7 +82,9 @@ void doImport(VirtualFrame frame, TruffleString moduleName, int level, @CachedLibrary(limit = "1") InteropLibrary interopLib, @Cached ImportName importNameNode, @Cached PyObjectSetItem setItemNode, - @Cached GetOrCreateDictNode getDictNode, + @Cached PyObjectLookupAttr lookupAttrNode, + @Cached PyObjectCallMethodObjArgs callMethodNode, + @Cached FastConstructListNode materializeKeysNode, @Cached PyObjectGetAttr getAttrNode, @Cached PyObjectGetIter getIterNode, @Cached PyIterNextNode nextNode, @@ -89,8 +93,7 @@ void doImport(VirtualFrame frame, TruffleString moduleName, int level, @Cached InlinedConditionProfile javaImport, @Cached CastToTruffleStringNode castToTruffleStringNode, @Cached TruffleString.CodePointLengthNode codePointLengthNode, - @Cached TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode, - @Cached IsBuiltinObjectProfile isAttributeErrorProfile) { + @Cached TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { Object importedModule = importModule(frame, moduleName, PArguments.getGlobals(frame), T_IMPORT_ALL, level, importNameNode); Object locals = PArguments.getSpecialArgument(frame); if (locals == null) { @@ -115,37 +118,42 @@ void doImport(VirtualFrame frame, TruffleString moduleName, int level, throw new IllegalStateException(e); } } else { - try { - Object attrAll = getAttrNode.execute(frame, inliningTarget, importedModule, T___ALL__); + Object attrAll = lookupAttrNode.execute(frame, inliningTarget, importedModule, T___ALL__); + if (attrAll != PNone.NO_VALUE) { int n = sizeNode.execute(frame, inliningTarget, attrAll); try { for (int i = 0; i < n; i++) { Object attrName = getItemNode.execute(frame, inliningTarget, attrAll, i); - writeAttributeToLocals(frame, inliningTarget, moduleName, (PythonModule) importedModule, locals, attrName, true, castToTruffleStringNode, codePointLengthNode, + writeAttributeToLocals(frame, inliningTarget, moduleName, importedModule, locals, attrName, true, castToTruffleStringNode, codePointLengthNode, codePointAtIndexNode, getAttrNode, setItemNode); } } finally { LoopNode.reportLoopCount(inliningTarget, n); } - } catch (PException e) { - e.expectAttributeError(inliningTarget, isAttributeErrorProfile); - assert importedModule instanceof PythonModule; - Object keysIterator = getIterNode.execute(frame, inliningTarget, getDictNode.execute(inliningTarget, importedModule)); + } else { + Object dict = lookupAttrNode.execute(frame, inliningTarget, importedModule, T___DICT__); + if (dict == PNone.NO_VALUE) { + throw PRaiseNode.raiseStatic(this, ImportError, ErrorMessages.IMPORT_STAR_NO_DICT_OR_ALL); + } + Object keys = callMethodNode.execute(frame, inliningTarget, dict, T_KEYS); + // Like PyMapping_Keys, consume the keys before reading any exported attributes. + keys = materializeKeysNode.execute(frame, inliningTarget, keys); + Object keysIterator = getIterNode.execute(frame, inliningTarget, keys); while (true) { Object key; try { key = nextNode.execute(frame, inliningTarget, keysIterator); - } catch (IteratorExhausted e1) { + } catch (IteratorExhausted e) { break; } - writeAttributeToLocals(frame, inliningTarget, moduleName, (PythonModule) importedModule, locals, key, false, castToTruffleStringNode, codePointLengthNode, + writeAttributeToLocals(frame, inliningTarget, moduleName, importedModule, locals, key, false, castToTruffleStringNode, codePointLengthNode, codePointAtIndexNode, getAttrNode, setItemNode); } } } } - private void writeAttributeToLocals(VirtualFrame frame, Node inliningTarget, TruffleString moduleName, PythonModule importedModule, Object locals, Object attrName, boolean fromAll, + private void writeAttributeToLocals(VirtualFrame frame, Node inliningTarget, TruffleString moduleName, Object importedModule, Object locals, Object attrName, boolean fromAll, CastToTruffleStringNode castToTruffleStringNode, TruffleString.CodePointLengthNode cpLenNode, TruffleString.CodePointAtIndexUTF32Node cpAtIndexNode, PyObjectGetAttr getAttr, PyObjectSetItem dictWriteNode) { try { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java index 5475ab39ae..a3d3f2a1cd 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java @@ -1956,6 +1956,73 @@ public Object readlinkat(int dirFd, Object path) throws PosixException { } } + @Override + @TruffleBoundary + public Object getFinalPathName(Object path) throws PosixException { + try { + String pathname = pathToJavaString(path); + if (pathname.isEmpty()) { + throw new PosixErrnoException(OSErrorEnum.ENOENT.getNumber(), OSErrorEnum.ENOENT.getMessage(), 3); + } + String result = getTruffleFile(pathname).getCanonicalFile().getPath(); + if (result.startsWith("\\\\?\\")) { + return result; + } + return result.startsWith("\\\\") ? "\\\\?\\UNC\\" + result.substring(2) : "\\\\?\\" + result; + } catch (Exception e) { + throw windowsPathException(e); + } + } + + @Override + @TruffleBoundary + public Object findFirstFile(Object path) throws PosixException { + try { + String pathname = pathToJavaString(path); + if (pathname.isEmpty()) { + throw new PosixErrnoException(OSErrorEnum.ENOENT.getNumber(), OSErrorEnum.ENOENT.getMessage(), 3); + } + // Let the filesystem resolve casing and short names, preserving the link's own name + // even for dangling symlinks. Only literal paths are needed by ntpath.realpath. + TruffleFile file = getTruffleFile(pathname).getCanonicalFile(LinkOption.NOFOLLOW_LINKS); + if (file.getParent() != null) { + return file.getName(); + } + throw new PosixErrnoException(OSErrorEnum.ENOENT.getNumber(), OSErrorEnum.ENOENT.getMessage(), 2); + } catch (Exception e) { + throw windowsPathException(e); + } + } + + // Unlike other emulated APIs, these Windows path helpers must synthesize winerror because + // ntpath.realpath inspects it to choose its fallback behavior. + private static PosixException windowsPathException(Exception e) { + int errno; + TruffleString message; + if (e instanceof PosixErrnoException error) { + if (error.getWinerror() != null) { + return error; + } + errno = error.getErrorCode(); + message = error.getMessageAsTruffleString(); + } else { + ErrorAndMessagePair error = OSErrorEnum.fromException(e); + errno = error.oserror.getNumber(); + message = error.message; + } + int winerror; + if (errno == OSErrorEnum.ENOENT.getNumber()) { + winerror = 2; + } else if (errno == OSErrorEnum.EACCES.getNumber()) { + winerror = 5; + } else if (errno == OSErrorEnum.EINVAL.getNumber()) { + winerror = 123; + } else { + winerror = 1; + } + return new PosixErrnoException(errno, message, winerror); + } + @Override @TruffleBoundary public void kill(long pid, int signal) throws PosixException { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java index 4669d6501d..87d148bb52 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java @@ -846,6 +846,28 @@ public final Object readlinkat(int dirFd, Object path) throws PosixException { } } + @Override + @TruffleBoundary + public final Object getFinalPathName(Object path) throws PosixException { + logEnter("getFinalPathName", "%s", path); + try { + return logExit("getFinalPathName", "%s", delegate.getFinalPathName(path)); + } catch (PosixException e) { + throw logException("getFinalPathName", e); + } + } + + @Override + @TruffleBoundary + public final Object findFirstFile(Object path) throws PosixException { + logEnter("findFirstFile", "%s", path); + try { + return logExit("findFirstFile", "%s", delegate.findFirstFile(path)); + } catch (PosixException e) { + throw logException("findFirstFile", e); + } + } + @Override @TruffleBoundary public final void kill(long pid, int signal) throws PosixException { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java index 732e036472..91a525ebe1 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java @@ -362,6 +362,12 @@ abstract static class PosixNativeFunctionInvoker { @DowncallSignature(returnType = SINT64, argumentTypes = {SINT32, POINTER, POINTER, SINT64}) abstract long call_readlinkat(int dirFd, long path, long buf, long size); + @DowncallSignature(returnType = SINT64, argumentTypes = {POINTER, POINTER, SINT64}) + abstract long call_getfinalpathname(long path, long buf, long size); + + @DowncallSignature(returnType = SINT64, argumentTypes = {POINTER, POINTER, SINT64}) + abstract long call_findfirstfile(long path, long buf, long size); + @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32}) abstract int get_inheritable(int fd); @@ -1715,6 +1721,46 @@ public Object readlinkat(int dirFd, Object path) throws PosixException { } } + @Override + @TruffleBoundary + public Object getFinalPathName(Object path) throws PosixException { + return windowsPathName(path, false); + } + + @Override + @TruffleBoundary + public Object findFirstFile(Object path) throws PosixException { + return windowsPathName(path, true); + } + + private Object windowsPathName(Object path, boolean findFirst) throws PosixException { + long pathPtr = pathToNativeCString(path); + try { + int capacity = 260; + while (true) { + long buffer = NativeMemory.mallocByteArrayOrNull((long) capacity * 2); + try { + long length = findFirst ? posixNativeFunctionInvoker.call_findfirstfile(pathPtr, buffer, capacity) + : posixNativeFunctionInvoker.call_getfinalpathname(pathPtr, buffer, capacity); + if (length < 0) { + throw getErrnoAndThrowPosixException(); + } + if (length >= capacity) { + capacity = Math.toIntExact(length); + continue; + } + byte[] result = new byte[Math.toIntExact(length * 2)]; + NativeMemory.readByteArrayElements(buffer, 0, result, 0, result.length); + return new WidePath(result); + } finally { + NativeMemory.free(buffer); + } + } + } finally { + NativeMemory.free(pathPtr); + } + } + @Override @TruffleBoundary public void kill(long pid, int signal) throws PosixException { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java index a7c729e138..668955cd5c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixConstantsWin32.java @@ -40,7 +40,7 @@ */ package com.oracle.graal.python.runtime; -// Auto generated by gen_native_cfg.py at 2026-11-02 16:06:19.794569 +// Auto generated by gen_native_cfg.py at 2026-11-03 14:47:00.966353 // on Windows QUICKEM-GVMHCD3 11 10.0.22631 AMD64 Intel64 Family 6 Model 198 Stepping 2, GenuineIntel class PosixConstantsWin32 { @@ -184,8 +184,8 @@ static void getConstants(PosixConstants.Registry constants) { constants.put("IPPROTO_SCTP", 132); constants.put("IPPROTO_RAW", 255); constants.put("SHUT_RD", 0); - constants.put("SHUT_WR", 0); - constants.put("SHUT_RDWR", 0); + constants.put("SHUT_WR", 1); + constants.put("SHUT_RDWR", 2); constants.put("SO_DEBUG", 1); constants.put("SO_ACCEPTCONN", 2); constants.put("SO_REUSEADDR", 4); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupport.java index 5a6b6a762f..9ceb8d4def 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupport.java @@ -1186,4 +1186,12 @@ public record WindowsVersion(int major, int minor, int build, int platform, Truf @TruffleBoundary public abstract Object readlinkat(int dirFd, Object path) throws PosixException; + + /** Windows GetFinalPathNameByHandleW, returning an extended absolute path. */ + @TruffleBoundary + public abstract Object getFinalPathName(Object path) throws PosixException; + + /** Windows FindFirstFileW, returning the name of the matching directory entry. */ + @TruffleBoundary + public abstract Object findFirstFile(Object path) throws PosixException; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java index ecf69d93ed..dee9a07c4b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java @@ -651,6 +651,20 @@ public final Object readlinkat(int dirFd, Object path) throws PosixException { return nativePosixSupport.readlinkat(dirFd, path); } + @Override + @TruffleBoundary + public final Object getFinalPathName(Object path) throws PosixException { + checkNotInPreInitialization(); + return nativePosixSupport.getFinalPathName(path); + } + + @Override + @TruffleBoundary + public final Object findFirstFile(Object path) throws PosixException { + checkNotInPreInitialization(); + return nativePosixSupport.findFirstFile(path); + } + @Override @TruffleBoundary public final void raise(int signal) throws PosixException { diff --git a/graalpython/graalpy-pyconfig/pyconfig_template.h b/graalpython/graalpy-pyconfig/pyconfig_template.h index a0caa128f8..b5aa29b005 100644 --- a/graalpython/graalpy-pyconfig/pyconfig_template.h +++ b/graalpython/graalpy-pyconfig/pyconfig_template.h @@ -61,6 +61,16 @@ # endif #endif +// CPython defines MS_WIN32 on both 32-bit and 64-bit Windows. +#ifdef MS_WINDOWS +# ifndef MS_WIN32 +# define MS_WIN32 +# endif +# if defined(_WIN64) && !defined(MS_WIN64) +# define MS_WIN64 +# endif +#endif + /* If Cython is involved, avoid accesses to internal structures. While we are * supporting this in many cases, it still involves overhead. */ #define CYTHON_USE_TYPE_SLOTS 0 diff --git a/graalpython/lib-graalpython/patches/cffi-1.16.0.patch b/graalpython/lib-graalpython/patches/cffi-1.16.0.patch index 3d6daaedd7..ac6ecb4c14 100644 --- a/graalpython/lib-graalpython/patches/cffi-1.16.0.patch +++ b/graalpython/lib-graalpython/patches/cffi-1.16.0.patch @@ -1,3 +1,16 @@ +diff --git a/setup.py b/setup.py +index f9cabaa..e2711bb 100644 +--- a/setup.py ++++ b/setup.py +@@ -135,7 +135,7 @@ if sys.platform == "win32" and uses_msvc(): + _filenames = [filename.lower() for filename in os.listdir(COMPILE_LIBFFI)] + _filenames = [filename for filename in _filenames + if filename.endswith('.c')] +- if sys.maxsize > 2**32: ++ if True: # GraalPy only supports 64-bit platforms. + # 64-bit: unlist win32.c, and add instead win64.obj. If the obj + # happens to get outdated at some point in the future, you need to + # rebuild it manually from win64.asm. diff --git a/src/c/_cffi_backend.c b/src/c/_cffi_backend.c index c5ea2cb..b797540 100644 --- a/src/c/_cffi_backend.c diff --git a/graalpython/lib-graalpython/patches/cffi-2.1.0.patch b/graalpython/lib-graalpython/patches/cffi-2.1.0.patch new file mode 100644 index 0000000000..4b0b6dc2ef --- /dev/null +++ b/graalpython/lib-graalpython/patches/cffi-2.1.0.patch @@ -0,0 +1,96 @@ +diff --git a/setup.py b/setup.py +index 50b487e..1081471 100644 +--- a/setup.py ++++ b/setup.py +@@ -141,7 +141,7 @@ if sys.platform == "win32" and uses_msvc(): + _filenames = [filename.lower() for filename in os.listdir(COMPILE_LIBFFI)] + _filenames = [filename for filename in _filenames + if filename.endswith('.c')] +- if sys.maxsize > 2**32: ++ if True: # GraalPy only supports 64-bit platforms. + # 64-bit: unlist win32.c, and add instead win64.obj. If the obj + # happens to get outdated at some point in the future, you need to + # rebuild it manually from win64.asm. +diff --git a/src/c/_cffi_backend.c b/src/c/_cffi_backend.c +index c1f583c..42d4d56 100644 +--- a/src/c/_cffi_backend.c ++++ b/src/c/_cffi_backend.c +@@ -2521,19 +2521,22 @@ _cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[]) + Py_ssize_t start, stop; + CTypeDescrObject *ct; + +- start = PyLong_AsSsize_t(slice->start); ++ PyObject* slicestart = PySlice_Start(slice); ++ start = PyLong_AsSsize_t(slicestart); + if (start == -1 && PyErr_Occurred()) { +- if (slice->start == Py_None) ++ if (slicestart == Py_None) + PyErr_SetString(PyExc_IndexError, "slice start must be specified"); + return NULL; + } +- stop = PyLong_AsSsize_t(slice->stop); ++ PyObject* slicestop = PySlice_Stop(slice); ++ stop = PyLong_AsSsize_t(slicestop); + if (stop == -1 && PyErr_Occurred()) { +- if (slice->stop == Py_None) ++ if (slicestop == Py_None) + PyErr_SetString(PyExc_IndexError, "slice stop must be specified"); + return NULL; + } +- if (slice->step != Py_None) { ++ PyObject* slicestep = PySlice_Step(slice); ++ if (slicestep != Py_None) { + PyErr_SetString(PyExc_IndexError, "slice with step not supported"); + return NULL; + } +diff --git a/src/c/misc_thread_common.h b/src/c/misc_thread_common.h +index 766dcb2..723bec3 100644 +--- a/src/c/misc_thread_common.h ++++ b/src/c/misc_thread_common.h +@@ -339,45 +339,7 @@ static PyThreadState *get_current_ts(void) + + static PyGILState_STATE gil_ensure(void) + { +- /* Called at the start of a callback. Replacement for +- PyGILState_Ensure(). +- */ +- PyGILState_STATE result; +- PyThreadState *ts = PyGILState_GetThisThreadState(); +- //fprintf(stderr, "%p: gil_ensure(), tstate=%p, tls=%p\n", get_cffi_tls(), ts, get_cffi_tls()); +- +- if (ts != NULL) { +- ts->gilstate_counter++; +- if (ts != get_current_ts()) { +- /* common case: 'ts' is our non-current thread state and +- we have to make it current and acquire the GIL */ +- PyEval_RestoreThread(ts); +- //fprintf(stderr, "%p: gil_ensure(), tstate=%p MADE CURRENT\n", get_cffi_tls(), ts); +- return PyGILState_UNLOCKED; +- } +- else { +- //fprintf(stderr, "%p: gil_ensure(), tstate=%p ALREADY CURRENT\n", get_cffi_tls(), ts); +- return PyGILState_LOCKED; +- } +- } +- else { +- /* no thread state here so far. */ +- result = PyGILState_Ensure(); +- assert(result == PyGILState_UNLOCKED); +- +- ts = PyGILState_GetThisThreadState(); +- //fprintf(stderr, "%p: gil_ensure(), made a new tstate=%p\n", get_cffi_tls(), ts); +- assert(ts != NULL); +- assert(ts == get_current_ts()); +- assert(ts->gilstate_counter >= 1); +- +- /* Use the ThreadCanary mechanism to keep 'ts' alive until the +- thread really shuts down */ +- thread_canary_register(ts); +- +- assert(ts == PyGILState_GetThisThreadState()); +- return result; +- } ++ return PyGILState_Ensure(); + } + + static void gil_release(PyGILState_STATE oldstate) diff --git a/graalpython/lib-graalpython/patches/cffi-2.1.1.patch b/graalpython/lib-graalpython/patches/cffi-2.1.1.patch new file mode 100644 index 0000000000..8f2d4e9817 --- /dev/null +++ b/graalpython/lib-graalpython/patches/cffi-2.1.1.patch @@ -0,0 +1,82 @@ +diff --git a/setup.py b/setup.py +index 50b487e..1081471 100644 +--- a/setup.py ++++ b/setup.py +@@ -141,7 +141,7 @@ if sys.platform == "win32" and uses_msvc(): + _filenames = [filename.lower() for filename in os.listdir(COMPILE_LIBFFI)] + _filenames = [filename for filename in _filenames + if filename.endswith('.c')] +- if sys.maxsize > 2**32: ++ if True: # GraalPy only supports 64-bit platforms. + # 64-bit: unlist win32.c, and add instead win64.obj. If the obj + # happens to get outdated at some point in the future, you need to + # rebuild it manually from win64.asm. +diff --git a/src/c/_cffi_backend.c b/src/c/_cffi_backend.c +index 05331f5..6f2b5e2 100644 +--- a/src/c/_cffi_backend.c ++++ b/src/c/_cffi_backend.c +@@ -2521,19 +2521,22 @@ _cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[]) + Py_ssize_t start, stop; + CTypeDescrObject *ct; + +- start = PyLong_AsSsize_t(slice->start); ++ PyObject* slicestart = PySlice_Start(slice); ++ start = PyLong_AsSsize_t(slicestart); + if (start == -1 && PyErr_Occurred()) { +- if (slice->start == Py_None) ++ if (slicestart == Py_None) + PyErr_SetString(PyExc_IndexError, "slice start must be specified"); + return NULL; + } +- stop = PyLong_AsSsize_t(slice->stop); ++ PyObject* slicestop = PySlice_Stop(slice); ++ stop = PyLong_AsSsize_t(slicestop); + if (stop == -1 && PyErr_Occurred()) { +- if (slice->stop == Py_None) ++ if (slicestop == Py_None) + PyErr_SetString(PyExc_IndexError, "slice stop must be specified"); + return NULL; + } +- if (slice->step != Py_None) { ++ PyObject* slicestep = PySlice_Step(slice); ++ if (slicestep != Py_None) { + PyErr_SetString(PyExc_IndexError, "slice with step not supported"); + return NULL; + } +diff --git a/src/c/misc_thread_common.h b/src/c/misc_thread_common.h +index ca12d4d..02cb813 100644 +--- a/src/c/misc_thread_common.h ++++ b/src/c/misc_thread_common.h +@@ -320,31 +320,7 @@ static void restore_errno_only(void) + + static PyGILState_STATE gil_ensure(void) + { +- /* Called at the start of a callback. */ +- PyGILState_STATE result; +- PyThreadState *ts = PyGILState_GetThisThreadState(); +- //fprintf(stderr, "%p: gil_ensure(), tstate=%p, tls=%p\n", get_cffi_tls(), ts, get_cffi_tls()); +- +- if (ts != NULL) { +- /* common case: this thread already has a thread state */ +- return PyGILState_Ensure(); +- } +- else { +- /* no thread state here so far. */ +- result = PyGILState_Ensure(); +- assert(result == PyGILState_UNLOCKED); +- +- ts = PyGILState_GetThisThreadState(); +- //fprintf(stderr, "%p: gil_ensure(), made a new tstate=%p\n", get_cffi_tls(), ts); +- assert(ts != NULL); +- +- /* Use the ThreadCanary mechanism to keep 'ts' alive until the +- thread really shuts down */ +- thread_canary_register(ts); +- +- assert(ts == PyGILState_GetThisThreadState()); +- return result; +- } ++ return PyGILState_Ensure(); + } + + static void gil_release(PyGILState_STATE oldstate) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index c0475c73ad..10294fb565 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -19,10 +19,20 @@ patch = 'category-encoders.patch' license = 'BSD-3-Clause' [[cffi.rules]] -version = '>= 1.16.0' +version = '>= 1.16.0, < 2.1.0' patch = 'cffi-1.16.0.patch' license = 'MIT' +[[cffi.rules]] +version = '== 2.1.0' +patch = 'cffi-2.1.0.patch' +license = 'MIT-0' + +[[cffi.rules]] +version = '== 2.1.1' +patch = 'cffi-2.1.1.patch' +license = 'MIT-0' + [[cloudpickle.rules]] version = '>= 2.2.1' patch = 'cloudpickle-2.2.1.patch' diff --git a/graalpython/python-libposix/src/posix.c b/graalpython/python-libposix/src/posix.c index 415d73aebc..dab9723de3 100644 --- a/graalpython/python-libposix/src/posix.c +++ b/graalpython/python-libposix/src/posix.c @@ -2235,6 +2235,51 @@ GP_EXPORT int64_t call_readlinkat(int32_t dirFd, const wchar_t *path, wchar_t *b } return characters; } +/* Return the required capacity (including NUL) when the output buffer is too small. */ +GP_EXPORT int64_t call_getfinalpathname(const wchar_t *path, wchar_t *buf, uint64_t size) { + win_path_t prepared; + if (win_path_init(&prepared, path) != 0) { + return -1; + } + HANDLE handle = CreateFileW(prepared.value, 0, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + DWORD error = GetLastError(); + win_path_clear(&prepared); + if (handle == INVALID_HANDLE_VALUE) { + set_win_errno(error); + return -1; + } + DWORD length = GetFinalPathNameByHandleW(handle, buf, (DWORD) size, VOLUME_NAME_DOS); + error = GetLastError(); + CloseHandle(handle); + if (length == 0) { + set_win_errno(error); + return -1; + } + return length; +} + +GP_EXPORT int64_t call_findfirstfile(const wchar_t *path, wchar_t *buf, uint64_t size) { + win_path_t prepared; + if (win_path_init(&prepared, path) != 0) { + return -1; + } + WIN32_FIND_DATAW data; + HANDLE handle = FindFirstFileW(prepared.value, &data); + DWORD error = GetLastError(); + win_path_clear(&prepared); + if (handle == INVALID_HANDLE_VALUE) { + set_win_errno(error); + return -1; + } + FindClose(handle); + size_t length = wcslen(data.cFileName); + if (length >= size) { + return length + 1; + } + memcpy(buf, data.cFileName, length * sizeof(wchar_t)); + return length; +} + GP_EXPORT int64_t call_waitpid(int64_t pid, int32_t *status, int32_t options) { return unsupported(); } GP_EXPORT int32_t call_wcoredump(int32_t status) { return 0; } GP_EXPORT int32_t call_wifcontinued(int32_t status) { return 0; } @@ -3326,6 +3371,18 @@ int64_t call_readlinkat(int32_t dirFd, const char *path, char *buf, uint64_t siz CAPTURE_ERRNO_AND_RETURN(-1, readlinkat(dirFd, path, buf, size)); } +int64_t call_getfinalpathname(const char *path, char *buf, uint64_t size) { + errno = ENOSYS; + capture_errno(); + return -1; +} + +int64_t call_findfirstfile(const char *path, char *buf, uint64_t size) { + errno = ENOSYS; + capture_errno(); + return -1; +} + int64_t call_waitpid(int64_t pid, int32_t *status, int32_t options) { CAPTURE_ERRNO_AND_RETURN(-1, waitpid(pid, status, options)); } diff --git a/mx.graalpython/verify_patches.py b/mx.graalpython/verify_patches.py index 6e7df79154..f5d369af52 100644 --- a/mx.graalpython/verify_patches.py +++ b/mx.graalpython/verify_patches.py @@ -49,6 +49,7 @@ ALLOWED_LICENSES = { 'UPL', # https://spdx.org/licenses/UPL-1.0.html 'MIT', # https://spdx.org/licenses/MIT.html + 'MIT-0', # https://spdx.org/licenses/MIT-0.html 'BSD-3-Clause', # https://spdx.org/licenses/BSD-3-Clause.html 'BSD-2-Clause', # https://spdx.org/licenses/BSD-2-Clause.html 'Apache-2.0', # https://spdx.org/licenses/Apache-2.0.html diff --git a/scripts/gen_native_cfg.py b/scripts/gen_native_cfg.py index ff1febe8ca..03c79e50a5 100644 --- a/scripts/gen_native_cfg.py +++ b/scripts/gen_native_cfg.py @@ -91,6 +91,9 @@ # include # include # include +# define SHUT_RD SD_RECEIVE +# define SHUT_WR SD_SEND +# define SHUT_RDWR SD_BOTH # ifndef S_IFLNK # define S_IFLNK 0xA000 # endif @@ -391,9 +394,9 @@ i IPPROTO_RAW [shutdownHow] -0 i SHUT_RD -0 i SHUT_WR -0 i SHUT_RDWR + i SHUT_RD + i SHUT_WR + i SHUT_RDWR [socketOptions] i SO_DEBUG