From fa7662721d656b1c8458956d1e4c7befb40bbbc4 Mon Sep 17 00:00:00 2001
From: Peter Bee <bijunda1@xiaomi.com>
Date: Wed, 11 Nov 2020 15:49:04 +0800
Subject: [PATCH] libc/stdlib: Fix range check in strtoul(l)

The previous implementation of strtoul(l) is flawed. The range check
assumed that when overflow happens, the truncated value is smaller than
the original value. As a counter example, passing "10000000000" to
strtol will not trigger ERANGE, but return a truncated value. This patch
adds more accurate range checks.

Change-Id: I239e034e390b4974157ed6efa17110f2e74904cf
Signed-off-by: Peter Bee <bijunda1@xiaomi.com>
---
 libs/libc/stdlib/lib_strtoul.c  | 15 +++++++++------
 libs/libc/stdlib/lib_strtoull.c | 15 +++++++++------
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/libs/libc/stdlib/lib_strtoul.c b/libs/libc/stdlib/lib_strtoul.c
index 29ca5c4ecec..1ee1de21e57 100644
--- a/libs/libc/stdlib/lib_strtoul.c
+++ b/libs/libc/stdlib/lib_strtoul.c
@@ -69,8 +69,9 @@
 unsigned long strtoul(FAR const char *nptr, FAR char **endptr, int base)
 {
   unsigned long accum = 0;
-  unsigned long prev;
+  unsigned long limit;
   int value;
+  int last_digit;
   char sign = 0;
 
   if (nptr)
@@ -98,22 +99,24 @@ unsigned long strtoul(FAR const char *nptr, FAR char **endptr, int base)
         }
       else
         {
+          limit = ULONG_MAX / base;
+          last_digit = ULONG_MAX % base;
+
           /* Accumulate each "digit" */
 
           while (lib_isbasedigit(*nptr, base, &value))
             {
-              prev = accum;
-              accum = accum * base + value;
-              nptr++;
-
               /* Check for overflow */
 
-              if (accum < prev)
+              if (accum > limit || (accum == limit && value > last_digit))
                 {
                   set_errno(ERANGE);
                   accum = ULONG_MAX;
                   break;
                 }
+
+              accum = accum * base + value;
+              nptr++;
             }
 
           if (sign == '-')
diff --git a/libs/libc/stdlib/lib_strtoull.c b/libs/libc/stdlib/lib_strtoull.c
index 4e27a1fab42..5f8265026a9 100644
--- a/libs/libc/stdlib/lib_strtoull.c
+++ b/libs/libc/stdlib/lib_strtoull.c
@@ -71,8 +71,9 @@
 unsigned long long strtoull(FAR const char *nptr, FAR char **endptr, int base)
 {
   unsigned long long accum = 0;
-  unsigned long long prev;
+  unsigned long long limit;
   int value;
+  int last_digit;
   char sign = 0;
 
   if (nptr)
@@ -100,22 +101,24 @@ unsigned long long strtoull(FAR const char *nptr, FAR char **endptr, int base)
         }
       else
         {
+          limit = ULLONG_MAX / base;
+          last_digit = ULLONG_MAX % base;
+
           /* Accumulate each "digit" */
 
           while (lib_isbasedigit(*nptr, base, &value))
             {
-              prev = accum;
-              accum = accum * base + value;
-              nptr++;
-
               /* Check for overflow */
 
-              if (accum < prev)
+              if (accum > limit || (accum == limit && value > last_digit))
                 {
                   set_errno(ERANGE);
                   accum = ULLONG_MAX;
                   break;
                 }
+
+              accum = accum * base + value;
+              nptr++;
             }
 
           if (sign == '-')