Merge pull request #5019 from smowton/smowton/fix/unpack-rec-max-bytes

byte-operators unpack_rec: enforce max-bytes if supplied

Author: tautschnig

regression/cbmc/byte_update14/test.c

@@ -0,0 +1,12 @@
+#include <assert.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv)
+{
+  int array[10];
+  char *ptr = argc % 2 ? (char *)&array[0] : (char *)&argc;
+  ptr[argc] = 'a';
+  ptr[1] = (char)argc;
+  assert(ptr[1] == (char)argc);
+  assert(array[1] == (char)argc);
+}

regression/cbmc/byte_update14/test.desc

@@ -0,0 +1,8 @@
+CORE broken-smt-backend
+test.c
+
+^VERIFICATION FAILED$
+^\[main.assertion\.1\] line 10.*SUCCESS$
+^\[main.assertion\.2\] line 11.*FAILURE$
+^EXIT=10$
+^SIGNAL=0$

regression/cbmc/dynamic_size1/stack_object.desc

@@ -1,4 +1,4 @@
-CORE broken-smt-backend
+CORE
 stack_object.c
 --pointer-check
 ^EXIT=0$

regression/cbmc/pointer-function-parameters-struct-mutual-recursion/test.desc

@@ -1,4 +1,4 @@
-CORE broken-smt-backend
+CORE
 main.c
 --function func --min-null-tree-depth 10 --max-nondet-tree-depth 3 --pointer-check
 ^EXIT=0$

regression/cbmc/pointer-function-parameters-struct-simple-recursion-2/test.desc

@@ -1,4 +1,4 @@
-CORE broken-smt-backend
+CORE
 main.c
 --function func --min-null-tree-depth 10 --max-nondet-tree-depth 2 --pointer-check
 ^EXIT=0$

regression/cbmc/pointer-function-parameters-struct-simple-recursion/test.desc

@@ -1,4 +1,4 @@
-CORE broken-smt-backend
+CORE
 main.c
 --function func --min-null-tree-depth 10 --max-nondet-tree-depth 3 --pointer-check
 ^EXIT=0$

src/solvers/lowering/byte_operators.cpp

@@ -539,11 +539,24 @@ static exprt unpack_array_vector(
 
     // recursively unpack each element so that we eventually just have an array
     // of bytes left
-    exprt sub = unpack_rec(element, little_endian, {}, max_bytes, ns, true);
+
+    const optionalt<mp_integer> element_max_bytes =
+      max_bytes
+        ? std::min(mp_integer{el_bytes}, *max_bytes - byte_operands.size())
+        : optionalt<mp_integer>{};
+    const std::size_t element_max_bytes_int =
+      element_max_bytes ? numeric_cast_v<std::size_t>(*element_max_bytes)
+                        : el_bytes;
+
+    exprt sub =
+      unpack_rec(element, little_endian, {}, element_max_bytes, ns, true);
     exprt::operandst sub_operands =
-      instantiate_byte_array(sub, 0, el_bytes, ns);
+      instantiate_byte_array(sub, 0, element_max_bytes_int, ns);
     byte_operands.insert(
       byte_operands.end(), sub_operands.begin(), sub_operands.end());
+
+    if(max_bytes && byte_operands.size() >= *max_bytes)
+      break;
   }
 
   const std::size_t size = byte_operands.size();
@@ -882,10 +895,21 @@ static exprt unpack_rec(
     // endianness
     auto bits_opt = pointer_offset_bits(src.type(), ns);
     DATA_INVARIANT(bits_opt.has_value(), "basic type should have a fixed size");
+
     mp_integer bits = *bits_opt;
+    mp_integer i = 0;
+
+    if(max_bytes.has_value())
+    {
+      const auto max_bits = *max_bytes * 8;
+      if(little_endian)
+        bits = std::min(bits, max_bits);
+      else
+        i = std::max(mp_integer{0}, bits - max_bits);
+    }
 
     exprt::operandst byte_operands;
-    for(mp_integer i=0; i<bits; i+=8)
+    for(; i < bits; i += 8)
     {
       extractbits_exprt extractbits(
         typecast_exprt::conditional_cast(
@@ -1004,7 +1028,7 @@ exprt lower_byte_extract(const byte_extract_exprt &src, const namespacet &ns)
     src.id() == ID_byte_extract_big_endian);
   const bool little_endian = src.id() == ID_byte_extract_little_endian;
 
-  // determine an upper bound of the number of bytes we might need
+  // determine an upper bound of the last byte we might need
   auto upper_bound_opt = size_of_expr(src.type(), ns);
   if(upper_bound_opt.has_value())
   {
@@ -1041,6 +1065,7 @@ exprt lower_byte_extract(const byte_extract_exprt &src, const namespacet &ns)
     if(element_bits.has_value() && *element_bits >= 1 && *element_bits % 8 == 0)
     {
       auto num_elements = numeric_cast<std::size_t>(array_type.size());
+      // XXX: This can't be right -- unpacked is a byte array, whereas array_type may have larger elements
       if(!num_elements.has_value() && unpacked.op().id() == ID_array)
         num_elements = unpacked.op().operands().size();
 
@@ -1221,11 +1246,10 @@ exprt lower_byte_extract(const byte_extract_exprt &src, const namespacet &ns)
     size_bits = op0_bits;
   }
 
-  mp_integer num_elements =
-    (*size_bits) / 8 + (((*size_bits) % 8 == 0) ? 0 : 1);
+  mp_integer num_bytes = (*size_bits) / 8 + (((*size_bits) % 8 == 0) ? 0 : 1);
 
   // get 'width'-many bytes, and concatenate
-  const std::size_t width_bytes = numeric_cast_v<std::size_t>(num_elements);
+  const std::size_t width_bytes = numeric_cast_v<std::size_t>(num_bytes);
   exprt::operandst op;
   op.reserve(width_bytes);