fs/vfs: Add ioctldir for volume ioctls via the mountpoint directory.

FIOC_REFORMAT, FIOC_OPTIMIZE, FIOC_INTEGRITY and FIOC_DUMP act on a volume,
not on any one file, but the only route into a file system has been the
per-file ioctl method.  A caller therefore has to open an unrelated file
just to name the volume it means.

For nxffs that is not merely awkward, it is a dead end.  nxffs_ioctl()
refuses FIOC_REFORMAT while any file on the volume is open:

    if (volume->ofiles)
      {
        ferr("ERROR: Open files\n");
        ret = -EBUSY;

and every open file is on that list (nxffs_open.c).  The descriptor used to
issue the command is itself such a file, so the check can never pass and
FIOC_REFORMAT is unreachable through the only interface that exposes it.

Add an optional ioctldir method to struct mountpt_operations, reached by
issuing the ioctl on a descriptor for the mountpoint directory:

    fd = open("/mnt/nxffs", O_RDONLY | O_DIRECTORY);
    ioctl(fd, FIOC_REFORMAT, 0);

It takes the same (mountpt, dir) pair as opendir/readdir/rewinddir, so it
reads as a member of the directory-operations family; the file system
recovers the volume from the mountpoint inode and may ignore dir.  The
member is placed at the end of the structure rather than beside the other
directory operations on purpose: every file system initialises
mountpt_operations positionally, so a member inserted mid-structure would
force all of them to add a slot for a method they do not implement.
Appending keeps the change to one file system.

dir_ioctl() gives that method the first chance at every command when the
directory belongs to a mounted volume and the file system provides one, and
falls back to its own handling of FIOC_FILEPATH and BIOC_FLUSH when the file
system answers -ENOTTY.  Trying the file system first is what lets a file
system override a command the VFS would otherwise answer generically; the
-ENOTTY fallback is what keeps the generic answers available to everyone
else.  A file system that leaves the method NULL is unaffected: the VFS
answers exactly as before.

The existing per-file method could not simply be reused for this.  It takes
a struct file, and every implementation that has an ioctl -- fat, romfs,
tmpfs, spiffs among them -- asserts on filep->f_priv and dereferences it,
so handing it a directory descriptor with no open file behind it would
fault.  Making the entry point separate keeps that contract intact and
makes support explicit rather than assumed.

nxffs implements it, which is what makes its FIOC_REFORMAT reachable.  The
per-file path is left in place and both share one implementation, so
nothing that works today stops working.  spiffs, which has the same shape
of volume commands, can follow.

Measured on sim:nxffs, with one file written to the volume and then the
same sequence of ioctls issued on a file descriptor, on a descriptor for the
mountpoint directory, and on a descriptor for a pseudo file system directory.
Before:

    FIOC_REFORMAT via file fd:  ret=-1 errno=16 (EBUSY, as expected)
    FIOC_REFORMAT via dir fd:   ret=-1 errno=25
    FIOC_FILEPATH via dir fd:   ret=0  "/mnt/nxffs//"
    BIOC_FLUSH    via dir fd:   ret=0
    bogus cmd     via dir fd:   ret=-1 errno=25
    FIOC_FILEPATH via /dev fd:  ret=0  "/dev//"
    bogus cmd     via /dev fd:  ret=-1 errno=25
    name still in the raw MTD image afterwards: yes

After:

    FIOC_REFORMAT via file fd:  ret=-1 errno=16 (EBUSY, as expected)
    FIOC_REFORMAT via dir fd:   ret=0
    FIOC_FILEPATH via dir fd:   ret=0  "/mnt/nxffs//"
    BIOC_FLUSH    via dir fd:   ret=0
    bogus cmd     via dir fd:   ret=-1 errno=25
    FIOC_FILEPATH via /dev fd:  ret=0  "/dev//"
    bogus cmd     via /dev fd:  ret=-1 errno=25
    name still in the raw MTD image afterwards: no

Only the FIOC_REFORMAT line on the directory descriptor changes, and the raw
MTD image confirms the volume really was erased.  FIOC_FILEPATH and
BIOC_FLUSH on a directory still answer even though nxffs is now consulted
ahead of them, an unrecognised command is still refused rather than
forwarded blindly, and a directory in the pseudo file system, which has no
ioctldir at all, is untouched.

Assisted-by: Claude Code:claude-opus-4-8
Assisted-by: Claude Code:claude-fable-5
Assisted-by: Claude Code:claude-opus-5
Signed-off-by: Marco Casaroli <marco.casaroli@gmail.com>

fs/nxffs/nxffs.h

@@ -1094,6 +1094,8 @@ ssize_t nxffs_read(FAR struct file *filep, FAR char *buffer, size_t buflen);
 ssize_t nxffs_write(FAR struct file *filep, FAR const char *buffer,
                     size_t buflen);
 int nxffs_ioctl(FAR struct file *filep, int cmd, unsigned long arg);
+int nxffs_ioctldir(FAR struct inode *mountpt, FAR struct fs_dirent_s *dir,
+                   int cmd, unsigned long arg);
 
 int nxffs_dup(FAR const struct file *oldp, FAR struct file *newp);
 int nxffs_fstat(FAR const struct file *filep, FAR struct stat *buf);

fs/nxffs/nxffs_initialize.c

@@ -86,7 +86,9 @@ const struct mountpt_operations g_nxffs_operations =
   NULL,              /* rmdir -- no directories */
   NULL,              /* rename -- cannot rename in place if name is longer */
   nxffs_stat,        /* stat */
-  NULL               /* chstat */
+  NULL,              /* chstat */
+  NULL,              /* syncfs */
+  nxffs_ioctldir     /* ioctldir */
 };
 
 /****************************************************************************

fs/nxffs/nxffs_ioctl.c

@@ -38,33 +38,23 @@
 #include "nxffs.h"
 
 /****************************************************************************
- * Public Functions
+ * Private Functions
  ****************************************************************************/
 
 /****************************************************************************
- * Name: nxffs_ioctl
+ * Name: nxffs_volume_cmd
  *
  * Description:
- *   Standard mountpoint ioctl method.
+ *   Carry out an ioctl against the volume.  Shared by the per-file and the
+ *   per-volume entry points, which differ only in how they name the volume.
  *
  ****************************************************************************/
 
-int nxffs_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
+static int nxffs_volume_cmd(FAR struct nxffs_volume_s *volume, int cmd,
+                            unsigned long arg)
 {
-  FAR struct nxffs_volume_s *volume;
   int ret;
 
-  finfo("cmd: %d arg: %08lx\n", cmd, arg);
-
-  /* Sanity checks */
-
-  DEBUGASSERT(filep->f_priv != NULL);
-
-  /* Recover the file system state from the open file */
-
-  volume = filep->f_inode->i_private;
-  DEBUGASSERT(volume != NULL);
-
   /* Get exclusive access to the volume.  Note that the volume lock
    * protects the open file list.
    */
@@ -73,7 +63,7 @@ int nxffs_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
   if (ret < 0)
     {
       ferr("ERROR: nxmutex_lock failed: %d\n", ret);
-      goto errout;
+      return ret;
     }
 
   /* Only a reformat and optimize commands are supported */
@@ -113,6 +103,62 @@ int nxffs_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
 
 errout_with_lock:
   nxmutex_unlock(&volume->lock);
-errout:
   return ret;
 }
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: nxffs_ioctl
+ *
+ * Description:
+ *   Standard mountpoint ioctl method.
+ *
+ ****************************************************************************/
+
+int nxffs_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
+{
+  FAR struct nxffs_volume_s *volume;
+
+  finfo("cmd: %d arg: %08lx\n", cmd, arg);
+
+  /* Sanity checks */
+
+  DEBUGASSERT(filep->f_priv != NULL);
+
+  /* Recover the file system state from the open file */
+
+  volume = filep->f_inode->i_private;
+  DEBUGASSERT(volume != NULL);
+
+  return nxffs_volume_cmd(volume, cmd, arg);
+}
+
+/****************************************************************************
+ * Name: nxffs_ioctldir
+ *
+ * Description:
+ *   ioctl method for a descriptor on the mountpoint directory.
+ *   FIOC_REFORMAT is only reachable this way: it refuses to run while any
+ *   file on the volume is open, and the per-file method is itself such a
+ *   file.  The open directory is not needed -- the volume is recovered from
+ *   the mountpoint inode -- so dir is unused.
+ *
+ ****************************************************************************/
+
+int nxffs_ioctldir(FAR struct inode *mountpt, FAR struct fs_dirent_s *dir,
+                   int cmd, unsigned long arg)
+{
+  FAR struct nxffs_volume_s *volume;
+
+  UNUSED(dir);
+
+  finfo("cmd: %d arg: %08lx\n", cmd, arg);
+
+  volume = mountpt->i_private;
+  DEBUGASSERT(volume != NULL);
+
+  return nxffs_volume_cmd(volume, cmd, arg);
+}

fs/vfs/fs_dir.c

@@ -556,15 +556,34 @@ static off_t dir_seek(FAR struct file *filep, off_t offset, int whence)
 static int dir_ioctl(FAR struct file *filep, int cmd, unsigned long arg)
 {
   FAR struct fs_dirent_s *dir = filep->f_priv;
-  int ret = OK;
+  int ret = -ENOTTY;
 
-  if (cmd == FIOC_FILEPATH)
+#ifndef CONFIG_DISABLE_MOUNTPOINT
+  /* If this directory belongs to a mounted volume whose file system offers
+   * volume-wide commands, give it the first chance:  it is the one route to
+   * them that does not require an unrelated file to be open.  Anything the
+   * file system does not recognize falls through to the VFS defaults.
+   */
+
+  if (INODE_IS_MOUNTPT(dir->fd_root) &&
+      dir->fd_root->u.i_mops != NULL &&
+      dir->fd_root->u.i_mops->ioctldir != NULL)
     {
-      strlcpy((FAR char *)(uintptr_t)arg, dir->fd_path, PATH_MAX);
+      ret = dir->fd_root->u.i_mops->ioctldir(dir->fd_root, dir, cmd, arg);
     }
-  else if (cmd != BIOC_FLUSH)
+#endif
+
+  if (ret == -ENOTTY)
     {
-      ret = -ENOTTY;
+      if (cmd == FIOC_FILEPATH)
+        {
+          strlcpy((FAR char *)(uintptr_t)arg, dir->fd_path, PATH_MAX);
+          ret = OK;
+        }
+      else if (cmd == BIOC_FLUSH)
+        {
+          ret = OK;
+        }
     }
 
   return ret;