fs/hostfs: Return ENOTTY for unsupported ioctl

host_ioctl() reports unsupported ioctl requests from hostfs backends. Use
-ENOTTY for that case instead of -ENOSYS so callers can distinguish an
unsupported ioctl request from a missing host operation.

Keep the other host operation stubs returning -ENOSYS; this change is limited
to ioctl semantics. Update the ARM, ARM64, RISC-V, Xtensa and Windows sim
hostfs stubs to match that behavior.

Testing:

  Host: Ubuntu 22.04 x86_64

  - git diff --check

  - make distclean

  - ./tools/configure.sh -l -a ../nuttx-apps sim:nsh

  - make -j16

  - printf 'help\npoweroff\n' | timeout 20s ./nuttx

  - make distclean

  - ./tools/configure.sh -a ../nuttx-apps sabre-6quad:knsh

  - make -j16

Assisted-by: Claude:Claude-Fable-5
Signed-off-by: Lingao Meng <menglingao@xiaomi.com>

arch/arm/src/common/arm_hostfs.c

@@ -232,7 +232,9 @@ off_t host_lseek(int fd, off_t pos, off_t offset, int whence)
 
 int host_ioctl(int fd, int request, unsigned long arg)
 {
-  return -ENOSYS;
+  /* Unsupported ioctl requests use ENOTTY so VFS can apply fallbacks. */
+
+  return -ENOTTY;
 }
 
 void host_sync(int fd)

arch/arm64/src/common/arm64_hostfs.c

@@ -232,7 +232,9 @@ off_t host_lseek(int fd, off_t pos, off_t offset, int whence)
 
 int host_ioctl(int fd, int request, unsigned long arg)
 {
-  return -ENOSYS;
+  /* Unsupported ioctl requests use ENOTTY so VFS can apply fallbacks. */
+
+  return -ENOTTY;
 }
 
 void host_sync(int fd)

arch/risc-v/src/common/riscv_hostfs.c

@@ -232,7 +232,9 @@ off_t host_lseek(int fd, off_t pos, off_t offset, int whence)
 
 int host_ioctl(int fd, int request, unsigned long arg)
 {
-  return -ENOSYS;
+  /* Unsupported ioctl requests use ENOTTY so VFS can apply fallbacks. */
+
+  return -ENOTTY;
 }
 
 void host_sync(int fd)

arch/sim/src/sim/win/sim_hostfs.c

@@ -224,7 +224,9 @@ nuttx_off_t host_lseek(int fd, nuttx_off_t pos, nuttx_off_t offset,
 
 int host_ioctl(int fd, int request, unsigned long arg)
 {
-  return -ENOSYS;
+  /* Unsupported ioctl requests use ENOTTY so VFS can apply fallbacks. */
+
+  return -ENOTTY;
 }
 
 /****************************************************************************

arch/xtensa/src/common/xtensa_hostfs.c

@@ -140,7 +140,9 @@ off_t host_lseek(int fd, off_t pos, off_t offset, int whence)
 
 int host_ioctl(int fd, int request, unsigned long arg)
 {
-  return -ENOSYS;
+  /* Unsupported ioctl requests use ENOTTY so VFS can apply fallbacks. */
+
+  return -ENOTTY;
 }
 
 void host_sync(int fd)