configure: Add check for O_DIRECTORY definition in fcntl.h

This is needed by sudoedit to avoid a potential issue on systems
without O_DIRECTORY where the wrong version of the DIR_OPEN_FLAGS
macro could be used.

Author: millert

config.h.in

@@ -168,6 +168,10 @@
    */
 #undef HAVE_DECL_NSIG
 
+/* Define to 1 if you have the declaration of 'O_DIRECTORY', and to 0 if you
+   don't. */
+#undef HAVE_DECL_O_DIRECTORY
+
 /* Define to 1 if you have the declaration of 'passwdexpiredx', and to 0 if
    you don't. */
 #undef HAVE_DECL_PASSWDEXPIREDX

configure

@@ -29511,6 +29511,21 @@ esac
 fi
 
 
+ac_fn_check_decl "$LINENO" "O_DIRECTORY" "ac_cv_have_decl_O_DIRECTORY" "
+$ac_includes_default
+#include <fcntl.h>
+
+" "$ac_c_undeclared_builtin_options" "CFLAGS"
+if test "x$ac_cv_have_decl_O_DIRECTORY" = xyes
+then :
+  ac_have_decl=1
+else case e in #(
+  e) ac_have_decl=0 ;;
+esac
+fi
+printf "%s\n" "#define HAVE_DECL_O_DIRECTORY $ac_have_decl" >>confdefs.h
+
+
 ac_fn_check_decl "$LINENO" "errno" "ac_cv_have_decl_errno" "
 $ac_includes_default
 #include <errno.h>

configure.ac

@@ -3387,6 +3387,14 @@ AC_INCLUDES_DEFAULT
 #include <signal.h>
 ])
 
+dnl
+dnl Check for O_DIRECTORY declaration in fcntl.h
+dnl
+AC_CHECK_DECLS([O_DIRECTORY], [], [], [
+AC_INCLUDES_DEFAULT
+#include <fcntl.h>
+])
+
 dnl
 dnl Check for errno declaration in errno.h
 dnl

include/sudo_compat.h

@@ -145,7 +145,7 @@
 #endif
 
 /* Older systems lack O_DIRECTORY. */
-#ifndef O_DIRECTORY
+#if defined(HAVE_DECL_O_DIRECTORY) && !HAVE_DECL_O_DIRECTORY
 # define O_DIRECTORY	0
 #endif
 

src/sudo_edit.h

@@ -24,18 +24,18 @@
  * Use O_SEARCH/O_PATH and/or O_DIRECTORY where possible.
  */
 #if defined(O_SEARCH)
-# if defined(O_DIRECTORY)
+# if defined(HAVE_DECL_O_DIRECTORY) && HAVE_DECL_O_DIRECTORY
 #  define DIR_OPEN_FLAGS	(O_SEARCH|O_DIRECTORY)
 # else
 #  define DIR_OPEN_FLAGS	(O_SEARCH)
 # endif
 #elif defined(O_PATH)
-# if defined(O_DIRECTORY)
+# if defined(HAVE_DECL_O_DIRECTORY) && HAVE_DECL_O_DIRECTORY
 #  define DIR_OPEN_FLAGS	(O_PATH|O_DIRECTORY)
 # else
 #  define DIR_OPEN_FLAGS	(O_PATH)
 # endif
-#elif defined(O_DIRECTORY)
+#elif defined(HAVE_DECL_O_DIRECTORY) && HAVE_DECL_O_DIRECTORY
 # define DIR_OPEN_FLAGS		(O_RDONLY|O_DIRECTORY)
 #else
 # define DIR_OPEN_FLAGS		(O_RDONLY|O_NONBLOCK)