arch/sim: walk frame-pointer chain for non-running tasks in up_backtrace

The previous up_backtrace() relied entirely on host_backtrace() (a thin
wrapper around glibc's backtrace()), which can only unwind the host
thread that calls it.  As a result, when assert / dump_tasks() walked the
task list and called sched_dumpstack() for every task, every task other
than the currently-running one returned a zero-length backtrace, and the
output was silently dropped.  In practice this meant that on sim only
the crashing task ever produced a usable trace.

Fix this by walking the frame-pointer chain ourselves whenever the
target tcb is not the running task.  Because sim's setjmp/longjmp is
provided by NuttX itself (libs/libc/machine/sim/arch_setjmp_*.S) and not
by host libc, the rbp/rsp/rip (or arm fp/sp/pc) saved in tcb->xcp.regs
are plain unmangled pointers, identical across Linux, macOS and Windows
hosts.  The frame layout ([fp]=prev fp, [fp+1]=return address) is also
shared by every host ABI sim supports (x86, x86_64, ARM, ARM64).

The walker validates that fp lies inside the task's stack and is
properly aligned, and stops when fp[0] is NULL, so a corrupted stack
cannot make us read out of bounds.  The running-task path is unchanged
and still uses host_backtrace() so DWARF unwinding through host
libraries continues to work.

Requires CONFIG_FRAME_POINTER=y so the compiler emits a usable fp link.

Signed-off-by: Lingao Meng <menglingao@xiaomi.com>

arch/sim/src/sim/sim_backtrace.c

@@ -25,38 +25,146 @@
  ****************************************************************************/
 
 #include <nuttx/arch.h>
+#include <nuttx/irq.h>
 #include <sched/sched.h>
 
+#include <setjmp.h>
+#include <stdint.h>
 #include <string.h>
 
 #include "sim_internal.h"
 
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: backtrace_fp
+ *
+ * Description:
+ *   Walk the frame-pointer chain to recover return addresses for a task
+ *   that is not currently running on the host thread.  This relies on the
+ *   uniform frame layout used by every host ABI sim supports (x86, x86_64,
+ *   ARM, ARM64):
+ *     [fp + 0] = previous fp
+ *     [fp + 1] = return address
+ *   where one slot is sizeof(uintptr_t).
+ *
+ *   Because sim's setjmp implementation is provided by NuttX itself
+ *   (libs/libc/machine/sim/arch_setjmp_*.S) rather than by host libc, the
+ *   saved fp/sp/pc in tcb->xcp.regs are plain pointers and contain no
+ *   pointer-mangling, so the chain can be followed directly on Linux,
+ *   macOS and Windows hosts.
+ *
+ ****************************************************************************/
+
+nosanitize_address
+static int backtrace_fp(uintptr_t *base, uintptr_t *limit,
+                        uintptr_t *fp, uintptr_t *pc,
+                        void **buffer, int size, int *skip)
+{
+  int i = 0;
+
+  if (pc != NULL)
+    {
+      if ((*skip)-- <= 0)
+        {
+          buffer[i++] = pc;
+        }
+    }
+
+  while (i < size)
+    {
+      /* Validate fp lies within [base, limit) and is properly aligned. */
+
+      if (fp < base || fp >= limit ||
+          ((uintptr_t)fp & (sizeof(uintptr_t) - 1)) != 0)
+        {
+          break;
+        }
+
+      if (fp[0] == 0)
+        {
+          break;
+        }
+
+      if ((*skip)-- <= 0)
+        {
+          buffer[i++] = (void *)fp[1];
+        }
+
+      fp = (uintptr_t *)fp[0];
+    }
+
+  return i;
+}
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
 
+/****************************************************************************
+ * Name: up_backtrace
+ *
+ * Description:
+ *   up_backtrace() returns a backtrace for the TCB, in the array pointed to
+ *   by buffer.  A backtrace is the series of currently active function
+ *   calls for the program.  Each item in the array pointed to by buffer is
+ *   of type void *, and is the return address from the corresponding stack
+ *   frame.
+ *
+ *   For the running task we keep using host_backtrace() (which wraps
+ *   glibc's backtrace()) so that we benefit from DWARF unwinding through
+ *   host libraries.  For any other task host_backtrace() cannot help -- it
+ *   only knows how to walk the host thread that is calling it -- so we
+ *   instead walk the frame-pointer chain starting from the registers saved
+ *   in tcb->xcp.regs by setjmp() at the last context switch.  This requires
+ *   CONFIG_FRAME_POINTER=y so that the compiler emits a usable fp link.
+ *
+ ****************************************************************************/
+
 nosanitize_address
 int up_backtrace(struct tcb_s *tcb, void **buffer, int size, int skip)
 {
-  void *buf[skip + size];
-  int ret = 0;
-  int i;
+  struct tcb_s *rtcb = running_task();
+  irqstate_t flags;
+  int ret;
 
-  if (tcb == running_task())
+  if (size <= 0 || buffer == NULL)
     {
+      return 0;
+    }
+
+  if (tcb == NULL)
+    {
+      tcb = rtcb;
+    }
+
+  if (tcb == rtcb)
+    {
+      void *buf[skip + size];
+
       ret = host_backtrace(buf, skip + size);
+      if (ret <= skip)
+        {
+          return ret < 0 ? ret : 0;
+        }
+
+      ret -= skip;
+      memcpy(buffer, &buf[skip], ret * sizeof(void *));
+      return ret;
     }
 
-  if (ret <= skip)
-    {
-      return ret < 0 ? ret : 0;
-    }
+  flags = enter_critical_section();
 
-  ret -= skip;
-  for (i = 0; i < ret; i++)
-    {
-      buffer[i] = buf[skip + i];
-    }
+  ret = backtrace_fp((uintptr_t *)tcb->stack_base_ptr,
+                     (uintptr_t *)((uintptr_t)tcb->stack_base_ptr +
+                                   tcb->adj_stack_size),
+                     (uintptr_t *)tcb->xcp.regs[JB_FP],
+                     (uintptr_t *)tcb->xcp.regs[JB_PC],
+                     buffer, size, &skip);
+
+  leave_critical_section(flags);
 
   return ret;
 }