Support F_GETFL and F_SETFL for fcntl

src/shims/files.rs

@@ -196,6 +196,20 @@ pub trait FileDescription: std::fmt::Debug + FileDescriptionExt {
     fn as_unix<'tcx>(&self, _ecx: &MiriInterpCx<'tcx>) -> &dyn UnixFileDescription {
         panic!("Not a unix file descriptor: {}", self.name());
     }
+
+    /// Helper function for fcntl(F_GETFL) to get fd's flag value.
+    fn get_flags<'tcx>(&self, _ecx: &mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, Scalar> {
+        throw_unsup_format!("cannot get flag value for {}", self.name());
+    }
+
+    /// Helper function for fcntl(F_SETFL) to set flag value.
+    fn set_flags<'tcx>(
+        &self,
+        _flag: i32,
+        _ecx: &mut MiriInterpCx<'tcx>,
+    ) -> InterpResult<'tcx, Scalar> {
+        throw_unsup_format!("cannot set flag value for {}", self.name());
+    }
 }
 
 impl FileDescription for io::Stdin {

src/shims/unix/fd.rs

@@ -9,6 +9,7 @@ use rustc_abi::Size;
 use crate::helpers::check_min_vararg_count;
 use crate::shims::files::FileDescription;
 use crate::shims::unix::linux_like::epoll::EpollReadyEvents;
+use crate::shims::unix::unnamed_socket::AnonSocket;
 use crate::shims::unix::*;
 use crate::*;
 
@@ -141,6 +142,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
         let f_getfd = this.eval_libc_i32("F_GETFD");
         let f_dupfd = this.eval_libc_i32("F_DUPFD");
         let f_dupfd_cloexec = this.eval_libc_i32("F_DUPFD_CLOEXEC");
+        let f_getfl = this.eval_libc_i32("F_GETFL");
+        let f_setfl = this.eval_libc_i32("F_SETFL");
 
         // We only support getting the flags for a descriptor.
         match cmd {
@@ -175,6 +178,35 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     this.set_last_error_and_return_i32(LibcError("EBADF"))
                 }
             }
+            cmd if cmd == f_getfl => {
+                // Check if this is a valid open file descriptor.
+                let Some(fd) = this.machine.fds.get(fd_num) else {
+                    return this.set_last_error_and_return_i32(LibcError("EBADF"));
+                };
+
+                // We only support F_GETFL for socketpair and pipe.
+                let anonsocket_fd = fd.downcast::<AnonSocket>().ok_or_else(|| {
+                    err_unsup_format!("fcntl: only socketpair / pipe are supported for F_GETFL")
+                })?;
+
+                anonsocket_fd.get_flags(this)
+            }
+            cmd if cmd == f_setfl => {
+                // Check if this is a valid open file descriptor.
+                let Some(fd) = this.machine.fds.get(fd_num) else {
+                    return this.set_last_error_and_return_i32(LibcError("EBADF"));
+                };
+
+                // We only support F_SETFL for socketpair and pipe.
+                let anonsocket_fd = fd.downcast::<AnonSocket>().ok_or_else(|| {
+                    err_unsup_format!("fcntl: only socketpair / pipe are supported for F_SETFL")
+                })?;
+
+                let [flag] = check_min_vararg_count("fcntl(fd, F_SETFL, ...)", varargs)?;
+                let flag = this.read_scalar(flag)?.to_i32()?;
+
+                anonsocket_fd.set_flags(flag, this)
+            }
             cmd if this.tcx.sess.target.os == "macos"
                 && cmd == this.eval_libc_i32("F_FULLFSYNC") =>
             {

src/shims/unix/unnamed_socket.rs

@@ -20,9 +20,18 @@ use crate::*;
 /// be configured in the real system.
 const MAX_SOCKETPAIR_BUFFER_CAPACITY: usize = 212992;
 
+#[derive(Debug, PartialEq)]
+enum AnonSocketType {
+    Socketpair,
+    // Read end of the pipe.
+    PipeRead,
+    // Write end of the pipe.
+    PipeWrite,
+}
+
 /// One end of a pair of connected unnamed sockets.
 #[derive(Debug)]
-struct AnonSocket {
+pub struct AnonSocket {
     /// The buffer we are reading from, or `None` if this is the writing end of a pipe.
     /// (In that case, the peer FD will be the reading end of that pipe.)
     readbuf: Option<RefCell<Buffer>>,
@@ -40,7 +49,10 @@ struct AnonSocket {
     /// A list of thread ids blocked because the buffer was full.
     /// Once another thread reads some bytes, these threads will be unblocked.
     blocked_write_tid: RefCell<Vec<ThreadId>>,
-    is_nonblock: bool,
+    /// Whether this is a non-blocking socket or not.
+    is_nonblock: Cell<bool>,
+    // Differentiate between different AnonSocket fd type.
+    fd_type: AnonSocketType,
 }
 
 #[derive(Debug)]
@@ -63,7 +75,10 @@ impl AnonSocket {
 
 impl FileDescription for AnonSocket {
     fn name(&self) -> &'static str {
-        "socketpair"
+        match self.fd_type {
+            AnonSocketType::Socketpair => "socketpair",
+            AnonSocketType::PipeRead | AnonSocketType::PipeWrite => "pipe",
+        }
     }
 
     fn close<'tcx>(
@@ -110,6 +125,57 @@ impl FileDescription for AnonSocket {
     fn as_unix<'tcx>(&self, _ecx: &MiriInterpCx<'tcx>) -> &dyn UnixFileDescription {
         self
     }
+
+    fn get_flags<'tcx>(&self, ecx: &mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, Scalar> {
+        let mut flags = 0;
+
+        // Add flag for file access mode.
+        match self.fd_type {
+            AnonSocketType::Socketpair => {
+                flags |= ecx.eval_libc_i32("O_RDWR");
+            }
+            AnonSocketType::PipeRead => {
+                flags |= ecx.eval_libc_i32("O_RDONLY");
+            }
+            AnonSocketType::PipeWrite => {
+                flags |= ecx.eval_libc_i32("O_WRONLY");
+            }
+        }
+
+        // Add flag for blocking status.
+        if self.is_nonblock.get() {
+            flags |= ecx.eval_libc_i32("O_NONBLOCK");
+        }
+
+        interp_ok(Scalar::from_i32(flags))
+    }
+
+    fn set_flags<'tcx>(
+        &self,
+        mut flag: i32,
+        ecx: &mut MiriInterpCx<'tcx>,
+    ) -> InterpResult<'tcx, Scalar> {
+        // FIXME: File access mode and file creation flags should be ignored.
+
+        let o_nonblock = ecx.eval_libc_i32("O_NONBLOCK");
+
+        // TODO: this is kind of bad and subtle, add a comment in the pr
+        if flag & o_nonblock == o_nonblock {
+            // If there is O_NONBLOCK flag
+            self.is_nonblock.set(true);
+            flag &= !o_nonblock;
+        } else {
+            // If there is no O_NONBLOCK flag
+            self.is_nonblock.set(false);
+        }
+
+        // Fail if there is unsupported flag.
+        if flag & o_nonblock != 0 {
+            throw_unsup_format!("fcntl: only O_NONBLOCK is supported for F_SETFL")
+        }
+
+        interp_ok(Scalar::from_i32(0))
+    }
 }
 
 /// Write to AnonSocket based on the space available and return the written byte size.
@@ -141,7 +207,7 @@ fn anonsocket_write<'tcx>(
     // Let's see if we can write.
     let available_space = MAX_SOCKETPAIR_BUFFER_CAPACITY.strict_sub(writebuf.borrow().buf.len());
     if available_space == 0 {
-        if self_ref.is_nonblock {
+        if self_ref.is_nonblock.get() {
             // Non-blocking socketpair with a full buffer.
             return finish.call(ecx, Err(ErrorKind::WouldBlock.into()));
         } else {
@@ -223,7 +289,7 @@ fn anonsocket_read<'tcx>(
             // Socketpair with no peer and empty buffer.
             // 0 bytes successfully read indicates end-of-file.
             return finish.call(ecx, Ok(0));
-        } else if self_ref.is_nonblock {
+        } else if self_ref.is_nonblock.get() {
             // Non-blocking socketpair with writer and empty buffer.
             // https://linux.die.net/man/2/read
             // EAGAIN or EWOULDBLOCK can be returned for socket,
@@ -407,15 +473,17 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             peer_lost_data: Cell::new(false),
             blocked_read_tid: RefCell::new(Vec::new()),
             blocked_write_tid: RefCell::new(Vec::new()),
-            is_nonblock: is_sock_nonblock,
+            is_nonblock: Cell::new(is_sock_nonblock),
+            fd_type: AnonSocketType::Socketpair,
         });
         let fd1 = fds.new_ref(AnonSocket {
             readbuf: Some(RefCell::new(Buffer::new())),
             peer_fd: OnceCell::new(),
             peer_lost_data: Cell::new(false),
             blocked_read_tid: RefCell::new(Vec::new()),
             blocked_write_tid: RefCell::new(Vec::new()),
-            is_nonblock: is_sock_nonblock,
+            is_nonblock: Cell::new(is_sock_nonblock),
+            fd_type: AnonSocketType::Socketpair,
         });
 
         // Make the file descriptions point to each other.
@@ -475,15 +543,17 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             peer_lost_data: Cell::new(false),
             blocked_read_tid: RefCell::new(Vec::new()),
             blocked_write_tid: RefCell::new(Vec::new()),
-            is_nonblock,
+            is_nonblock: Cell::new(is_nonblock),
+            fd_type: AnonSocketType::PipeRead,
         });
         let fd1 = fds.new_ref(AnonSocket {
             readbuf: None,
             peer_fd: OnceCell::new(),
             peer_lost_data: Cell::new(false),
             blocked_read_tid: RefCell::new(Vec::new()),
             blocked_write_tid: RefCell::new(Vec::new()),
-            is_nonblock,
+            is_nonblock: Cell::new(is_nonblock),
+            fd_type: AnonSocketType::PipeWrite,
         });
 
         // Make the file descriptions point to each other.

tests/fail-dep/libc/fcntl_fsetfl_while_blocking.rs

@@ -0,0 +1,21 @@
+//@ignore-target: windows # File handling is not implemented yet
+//~^ ERROR: deadlock: the evaluated program deadlocked
+//@compile-flags: -Zmiri-preemption-rate=0
+use std::thread;
+
+/// If an O_NONBLOCK flag is set while the fd is blocking, that fd will not be woken up.
+fn main() {
+    let mut fds = [-1, -1];
+    let res = unsafe { libc::pipe(fds.as_mut_ptr()) };
+    assert_eq!(res, 0);
+    let mut buf: [u8; 5] = [0; 5];
+    let _thread1 = thread::spawn(move || {
+        // Add O_NONBLOCK flag while pipe is still block on read.
+        let new_flag = libc::O_NONBLOCK;
+        let res = unsafe { libc::fcntl(fds[0], libc::F_SETFL, new_flag) };
+        assert_eq!(res, 0);
+    });
+    // Main thread will block on read.
+    let _res = unsafe { libc::read(fds[0], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) };
+    //~^ ERROR: deadlock: the evaluated program deadlocked
+}

tests/fail-dep/libc/fcntl_fsetfl_while_blocking.stderr

@@ -0,0 +1,19 @@
+error: deadlock: the evaluated program deadlocked
+   |
+   = note: the evaluated program deadlocked
+   = note: (no span available)
+   = note: BACKTRACE on thread `unnamed-ID`:
+
+error: deadlock: the evaluated program deadlocked
+  --> tests/fail-dep/libc/fcntl_fsetfl_while_blocking.rs:LL:CC
+   |
+LL |     let _res = unsafe { libc::read(fds[0], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) };
+   |                                                                                              ^ the evaluated program deadlocked
+   |
+   = note: BACKTRACE:
+   = note: inside `main` at tests/fail-dep/libc/fcntl_fsetfl_while_blocking.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 2 previous errors
+