From 72ec53ca262b827efaa3c53e8f8daed117efde72 Mon Sep 17 00:00:00 2001
From: Alex Shinn <alexshinn@gmail.com>
Date: Thu, 30 Jan 2025 11:44:23 +0900
Subject: [PATCH] More thorough checks for SEXP_MIN_FIXNUM/-1.

Closes #1006.
---
 eval.c               |  4 ++--
 include/chibi/sexp.h |  1 +
 lib/chibi/json.c     |  2 +-
 sexp.c               | 10 ++++++++++
 vm.c                 | 15 +++++++++++----
 5 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/eval.c b/eval.c
index d71e7df4..8a7f0de2 100644
--- a/eval.c
+++ b/eval.c
@@ -1947,8 +1947,8 @@ sexp sexp_inexact_to_exact (sexp ctx, sexp self, sexp_sint_t n, sexp z) {
       res = sexp_xtype_exception(ctx, self, "exact: not an integer", z);
 #endif
 #if SEXP_USE_BIGNUMS
-    } else if ((sexp_flonum_value(z) > SEXP_MAX_FIXNUM)
-               || sexp_flonum_value(z) < SEXP_MIN_FIXNUM) {
+    } else if ((sexp_flonum_value(z) > (double)SEXP_MAX_FIXNUM)
+               || sexp_flonum_value(z) < (double)SEXP_MIN_FIXNUM) {
       res = sexp_double_to_bignum(ctx, sexp_flonum_value(z));
 #endif
     } else {
diff --git a/include/chibi/sexp.h b/include/chibi/sexp.h
index 0858886e..3ebbd6d3 100644
--- a/include/chibi/sexp.h
+++ b/include/chibi/sexp.h
@@ -1079,6 +1079,7 @@ SEXP_API sexp sexp_make_unsigned_integer(sexp ctx, sexp_luint_t x);
 #define sexp_negate_flonum(x) sexp_flonum_value(x) = -(sexp_flonum_value(x))
 #endif
 
+/* TODO: Doesn't support x == SEXP_MIN_FIXNUM. */
 #define sexp_negate(x)                                  \
   if (sexp_flonump(x))                                  \
     sexp_negate_flonum(x);                              \
diff --git a/lib/chibi/json.c b/lib/chibi/json.c
index b0bda443..4e6e22c2 100644
--- a/lib/chibi/json.c
+++ b/lib/chibi/json.c
@@ -65,7 +65,7 @@ sexp json_read_number (sexp ctx, sexp self, sexp in) {
     res *= pow(10.0, scale_sign * scale);
   }
   if (ch != EOF) sexp_push_char(ctx, ch, in);
-  return (inexactp || fabs(res) > SEXP_MAX_FIXNUM) ?
+  return (inexactp || fabs(res) > (double)SEXP_MAX_FIXNUM) ?
     sexp_make_flonum(ctx, sign * res) :
     sexp_make_fixnum(sign * res);  /* always return inexact? */
 }
diff --git a/sexp.c b/sexp.c
index 5ea2c4ea..1e2efeac 100644
--- a/sexp.c
+++ b/sexp.c
@@ -2890,6 +2890,13 @@ sexp sexp_make_ratio (sexp ctx, sexp num, sexp den) {
 sexp sexp_ratio_normalize (sexp ctx, sexp rat, sexp in) {
   sexp tmp;
   sexp_gc_var2(num, den);
+  if (sexp_exact_negativep(sexp_ratio_denominator(rat))) {
+    /* Prevent overflow in the sexp_negate. */
+    if (sexp_ratio_numerator(rat) == sexp_make_fixnum(SEXP_MIN_FIXNUM))
+      sexp_ratio_numerator(rat) = sexp_fixnum_to_bignum(ctx, sexp_ratio_numerator(rat));
+    sexp_negate(sexp_ratio_numerator(rat));
+    sexp_negate(sexp_ratio_denominator(rat));
+  }
   num = sexp_ratio_numerator(rat), den = sexp_ratio_denominator(rat);
   if (den == SEXP_ZERO)
     return sexp_read_error(ctx, "zero denominator in ratio", rat, in);
@@ -2909,6 +2916,9 @@ sexp sexp_ratio_normalize (sexp ctx, sexp rat, sexp in) {
   sexp_ratio_numerator(rat)
     = sexp_quotient(ctx, sexp_ratio_numerator(rat), num);
   if (sexp_exact_negativep(sexp_ratio_denominator(rat))) {
+    /* Prevent overflow in the sexp_negate. */
+    if (sexp_ratio_numerator(rat) == sexp_make_fixnum(SEXP_MIN_FIXNUM))
+      sexp_ratio_numerator(rat) = sexp_fixnum_to_bignum(ctx, sexp_ratio_numerator(rat));
     sexp_negate(sexp_ratio_numerator(rat));
     sexp_negate(sexp_ratio_denominator(rat));
   }
diff --git a/vm.c b/vm.c
index 2ebd98db..5ad9452b 100644
--- a/vm.c
+++ b/vm.c
@@ -1869,7 +1869,12 @@ sexp sexp_apply (sexp ctx, sexp proc, sexp args) {
       if (sexp_flonum_value(_ARG1) == trunc(sexp_flonum_value(_ARG1)))
         _ARG1 = sexp_make_fixnum(sexp_flonum_value(_ARG1));
 #else
-      _ARG1 = sexp_fx_div(tmp1, tmp2);
+      if (tmp1 == sexp_make_fixnum(SEXP_MIN_FIXNUM) && tmp2 == SEXP_NEG_ONE) {
+        _ARG1 = sexp_fixnum_to_bignum(ctx, tmp1);
+        sexp_negate_exact(_ARG1);
+      } else {
+        _ARG1 = sexp_fx_div(tmp1, tmp2);
+      }
 #endif
 #endif
     }
@@ -1896,9 +1901,11 @@ sexp sexp_apply (sexp ctx, sexp proc, sexp args) {
     if (sexp_fixnump(tmp1) && sexp_fixnump(tmp2)) {
       if (tmp2 == SEXP_ZERO)
         sexp_raise("divide by zero", SEXP_NULL);
-      tmp = _ARG1 = sexp_fx_div(tmp1, tmp2);
-      if ((sexp_sint_t)tmp1 < 0 && (sexp_sint_t)tmp2 < 0 && (sexp_sint_t)tmp < 0) {
-        _ARG1 = sexp_quotient(ctx, tmp1=sexp_fixnum_to_bignum(ctx, tmp1), tmp2);
+      if (tmp1 == sexp_make_fixnum(SEXP_MIN_FIXNUM) && tmp2 == SEXP_NEG_ONE) {
+        _ARG1 = sexp_fixnum_to_bignum(ctx, tmp1);
+        sexp_negate_exact(_ARG1);
+      } else {
+        _ARG1 = sexp_fx_div(tmp1, tmp2);
       }
     }
 #if SEXP_USE_BIGNUMS