Stack Protection and Smashing using GCC

I am reading Smashing the Stack for Fun and Profit (in particular, this post refers to the “Buffer Overflows” section). The article is written for
a 32-bit machine however I am working on a 64-bit for which I take account in
my examples. One particular example is causing some issues that I cannot
explain. example3.c has the functionality of overwriting the return address to skip
an instruction in the main function. Here is my code:

#include <stdio.h>

void function(int a, int b, int c)
{
  char buf1[5];
  char buf2[10];
  int *retptr;

  retptr = (void*)(buf2 + 40);
  (*retptr) += 8;
}

int main(void)
{
  int x;

  x = 0;
  function(1,2,3);
  x = 1;
  printf("%dn", x);
  return 0;
}

I compile this program with gcc v4.8.2 with the following command:

gcc example3.c -o example3

Note that by default the gcc compiler appears to implement some stack
protection such as address space layout randomisation and stack canaries. I have taken into account these safety measures in my calculation of the ret
pointer value. Here
is the corresponding assembly produced by
gcc example3.c -S -o stack-protection.s:

    .file   "example3.c"
    .text
    .globl  function
    .type   function, @function
function:
.LFB0:
    .cfi_startproc
    pushq   %rbp
    .cfi_def_cfa_offset 16
    .cfi_offset 6, -16
    movq    %rsp, %rbp
    .cfi_def_cfa_register 6
    subq    $64, %rsp
    movl    %edi, -52(%rbp)
    movl    %esi, -56(%rbp)
    movl    %edx, -60(%rbp)
    movq    %fs:40, %rax
    movq    %rax, -8(%rbp)
    xorl    %eax, %eax
    leaq    -32(%rbp), %rax
    addq    $40, %rax
    movq    %rax, -40(%rbp)
    movq    -40(%rbp), %rax
    movl    (%rax), %eax
    leal    8(%rax), %edx
    movq    -40(%rbp), %rax
    movl    %edx, (%rax)
    movq    -8(%rbp), %rax
    xorq    %fs:40, %rax
    je  .L2
    call    __stack_chk_fail
.L2:
    leave
    .cfi_def_cfa 7, 8
    ret
    .cfi_endproc
.LFE0:
    .size   function, .-function
    .section    .rodata
.LC0:
    .string "%dn"
    .text
    .globl  main
    .type   main, @function
main:
.LFB1:
    .cfi_startproc
    pushq   %rbp
    .cfi_def_cfa_offset 16
    .cfi_offset 6, -16
    movq    %rsp, %rbp
    .cfi_def_cfa_register 6
    subq    $16, %rsp
    movl    $0, -4(%rbp)
    movl    $3, %edx
    movl    $2, %esi
    movl    $1, %edi
    call    function
    movl    $1, -4(%rbp)
    movl    -4(%rbp), %eax
    movl    %eax, %esi
    movl    $.LC0, %edi
    movl    $0, %eax
    call    printf
    movl    $0, %eax
    leave
    .cfi_def_cfa 7, 8
    ret
    .cfi_endproc
.LFE1:
    .size   main, .-main
    .ident  "GCC: (Ubuntu 4.8.2-19ubuntu1) 4.8.2"
    .section    .note.GNU-stack,"",@progbits

Executing example3 has the desired effect of skipping the second assignment to x and the program outputs 0.

However, if instead I compile using the -fno-stack-protector option:

gcc -fno-stack-protector example3.c -S -o no-stack-protection.s

I receive the following assembly file:

    .file   "example3.c"
    .text
    .globl  function
    .type   function, @function
function:
.LFB0:
    .cfi_startproc
    pushq   %rbp
    .cfi_def_cfa_offset 16
    .cfi_offset 6, -16
    movq    %rsp, %rbp
    .cfi_def_cfa_register 6
    movl    %edi, -36(%rbp)
    movl    %esi, -40(%rbp)
    movl    %edx, -44(%rbp)
    leaq    -32(%rbp), %rax
    addq    $40, %rax
    movq    %rax, -8(%rbp)
    movq    -8(%rbp), %rax
    movl    (%rax), %eax
    leal    8(%rax), %edx
    movq    -8(%rbp), %rax
    movl    %edx, (%rax)
    popq    %rbp
    .cfi_def_cfa 7, 8
    ret
    .cfi_endproc
.LFE0:
    .size   function, .-function
    .section    .rodata
.LC0:
    .string "%dn"
    .text
    .globl  main
    .type   main, @function
main:
.LFB1:
    .cfi_startproc
    pushq   %rbp
    .cfi_def_cfa_offset 16
    .cfi_offset 6, -16
    movq    %rsp, %rbp
    .cfi_def_cfa_register 6
    subq    $16, %rsp
    movl    $0, -4(%rbp)
    movl    $3, %edx
    movl    $2, %esi
    movl    $1, %edi
    call    function
    movl    $1, -4(%rbp)
    movl    -4(%rbp), %eax
    movl    %eax, %esi
    movl    $.LC0, %edi
    movl    $0, %eax
    call    printf
    movl    $0, %eax
    leave
    .cfi_def_cfa 7, 8
    ret
    .cfi_endproc
.LFE1:
    .size   main, .-main
    .ident  "GCC: (Ubuntu 4.8.2-19ubuntu1) 4.8.2"
    .section    .note.GNU-stack,"",@progbits

and the corresponding executable does not produce the desired value of 0 but a
random value which I cannot reconcile with the assembly file.

My mental picture of the stack frame in the -fno-stack-protector case is (sfp = saved frame pointer, ret = return address):

low memory address     buf2 (16 bytes)  buf1 (8 bytes)  retptr (8 bytes)  sfp (8 bytes) ret       high memory address
<---                  [              ][              ][                ][             ][    ] ...
top of stack                                                                                      bottom of stack

My Question:

Am I miscalculating the position of the return address in the unprotected case?


Source: c#

Leave a Reply