1

At my university we have a class that concerns some kernel programming. I stumbled upon oops message that was generated by faulty module and I am wondering about how can I call stack.

I have something like this when I perform some steps to get segmentation fault:

  469.541182] kernel BUG at mm/slub.c:2969!
[  469.541187] invalid opcode: 0000 [#1] SMP 
[  469.541193] last sysfs file: /sys/devices/LNXSYSTM:00/LNXSYBUS:00/ACPI0003:00/power_supply/ACAD/online
[  469.541198] Modules linked in: broken_module binfmt_misc vsock vmmemctl acpiphp snd_ens1371 gameport snd_ac97_codec ac97_bus snd_pcm_oss snd_mixer_oss snd_pcm snd_seq_dummy snd_seq_oss snd_seq_midi snd_rawmidi snd_seq_midi_event snd_seq snd_timer snd_seq_device ppdev snd vmci fbcon tileblit lp parport_pc intel_agp soundcore psmouse font bitblit softcursor serio_raw parport snd_page_alloc i2c_piix4 agpgart vga16fb vgastate shpchp pcnet32 mptspi mptscsih floppy mii mptbase scsi_transport_spi vmxnet vmw_pvscsi vmxnet3 [last unloaded: broken_module]
[  469.541259] 
[  469.541267] Pid: 2463, comm: cat Tainted: G        W  (2.6.32.46+drm33.20 #3) VMware Virtual Platform
[  469.541274] EIP: 0060:[<c01fa7a5>] EFLAGS: 00210246 CPU: 0
[  469.541286] EIP is at kfree+0xf5/0x100
[  469.541290] EAX: 00000000 EBX: c1912360 ECX: 00000000 EDX: 0889b000
[  469.541295] ESI: 0889b000 EDI: f47fdf98 EBP: f47fdf3c ESP: f47fdf24
[  469.541299]  DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
[  469.541305] Process cat (pid: 2463, ti=f47fc000 task=f69af2c0 task.ti=f47fc000)
[  469.541308] Stack:
[  469.541310]  00000060 00210206 ecc27e00 00000000 0889b000 f47fdf98 f47fdf64 f817b1d9
[  469.541320] <0> 00000000 f47fdf5c bfe4ff3c 00008000 00000023 ecc27200 00008000 0889b000
[  469.541330] <0> f47fdf8c c020609f f47fdf98 f47fdfb4 0804ce10 f817b170 c0583760 ecc27200
[  469.541340] Call Trace:
[  469.541351]  [<f817b1d9>] ? broken_read+0x69/0xf0 [broken_module]
[  469.541360]  [<c020609f>] ? vfs_read+0x9f/0x1a0
[  469.541366]  [<f817b170>] ? broken_read+0x0/0xf0 [broken_module]
[  469.541375]  [<c0583760>] ? do_page_fault+0x160/0x3a0
[  469.541383]  [<c0206252>] ? sys_read+0x42/0x70
[  469.541391]  [<c010340c>] ? syscall_call+0x7/0xb
[  469.541397]  [<c0580000>] ? mutex_lock_interruptible+0x0/0x40
[  469.541400] Code: 8b 03 e9 78 ff ff ff 8b 3d 30 a2 78 c0 85 ff 0f 84 48 ff ff ff 8b 0f 83 c7 04 89 f2 89 d8 ff d1 8b 0f 85 c9 75 f1 e9 32 ff ff ff <0f> 0b eb fe 8d b4 26 00 00 00 00 55 89 e5 0f 1f 44 00 00 83 e8 
[  469.541510] EIP: [<c01fa7a5>] kfree+0xf5/0x100 SS:ESP 0068:f47fdf24
[  469.541531] ---[ end trace 6acfcaf41ffa7314 ]---

I see that error was encountered during cat process (which makes sense because last step I performed was cat /dev/broken) and there was a problem in kfree function.

I have some problems reading call trace.

The first thing is, why do I have broken_read function listed twice? If I look at the source code there is no recursive call from broken_read to broken_read. Also I am wondering how can I read 0x69/0xf0. As far as I know the first value is the offset from the start of the function (in this case broken_read) but I don't know how to interpret the second value (0xf0).

Now, when I perform objdump -S -d -r broken_module.ko I can see

00000170 <broken_read>:

int broken_release(struct inode *inode, struct file *filp) {
    return 0;
}

ssize_t broken_read(struct file *filp, char *user_buf, size_t count, loff_t *f_pos) {
 170:   55                      push   %ebp
 171:   89 e5                   mov    %esp,%ebp
 173:   83 ec 20                sub    $0x20,%esp
 176:   89 5d f4                mov    %ebx,-0xc(%ebp)
 179:   89 75 f8                mov    %esi,-0x8(%ebp)
 17c:   89 7d fc                mov    %edi,-0x4(%ebp)
 17f:   e8 fc ff ff ff          call   180 <broken_read+0x10>
            180: R_386_PC32 mcount
extern void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags);
#else
static __always_inline void *
kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
{
    return kmem_cache_alloc(s, gfpflags);
 184:   b8 7c 04 00 00          mov    $0x47c,%eax
            185: R_386_32   kmalloc_caches
 189:   8b 7d 08                mov    0x8(%ebp),%edi
 18c:   89 d6                   mov    %edx,%esi
 18e:   ba d0 00 00 00          mov    $0xd0,%edx
 193:   e8 fc ff ff ff          call   194 <broken_read+0x24>
            194: R_386_PC32 kmem_cache_alloc
 198:   89 c3                   mov    %eax,%ebx
            if (!s)
                return ZERO_SIZE_PTR;

            ret = kmem_cache_alloc_notrace(s, flags);

            trace_kmalloc(_THIS_IP_, ret, size, s->size, flags);
 19a:   a1 80 04 00 00          mov    0x480,%eax
            19b: R_386_32   kmalloc_caches
 19f:   89 45 f0                mov    %eax,-0x10(%ebp)
    {(unsigned long)__GFP_THISNODE,     "GFP_THISNODE"},    \
    {(unsigned long)__GFP_RECLAIMABLE,  "GFP_RECLAIMABLE"}, \
    {(unsigned long)__GFP_MOVABLE,      "GFP_MOVABLE"}      \
    ) : "GFP_NOWAIT"

TRACE_EVENT(kmalloc,
 1a2:   a1 04 00 00 00          mov    0x4,%eax
            1a3: R_386_32   __tracepoint_kmalloc
 1a7:   85 c0                   test   %eax,%eax
 1a9:   75 70                   jne    21b <broken_read+0xab>
   char* mybuf = NULL;
   int mybuf_size = 100;
   int len, err;

    mybuf = kmalloc(mybuf_size, GFP_KERNEL);
    if (!mybuf) {
 1ab:   85 db                   test   %ebx,%ebx
 1ad:   74 60                   je     20f <broken_read+0x9f>
       broken_exit();
       return -ENOMEM;
    } else {
       fill_buffer(mybuf,mybuf_size);
 1af:   ba 64 00 00 00          mov    $0x64,%edx
 1b4:   89 d8                   mov    %ebx,%eax
 1b6:   e8 fc ff ff ff          call   1b7 <broken_read+0x47>
            1b7: R_386_PC32 fill_buffer
    }

    len = strlen(mybuf);
 1bb:   89 d8                   mov    %ebx,%eax
 1bd:   e8 fc ff ff ff          call   1be <broken_read+0x4e>
            1be: R_386_PC32 strlen
    err = copy_to_user(user_buf, mybuf, len);
 1c2:   89 da                   mov    %ebx,%edx
 1c4:   89 c1                   mov    %eax,%ecx
       return -ENOMEM;
    } else {
       fill_buffer(mybuf,mybuf_size);
    }

    len = strlen(mybuf);
 1c6:   89 45 f0                mov    %eax,-0x10(%ebp)
    err = copy_to_user(user_buf, mybuf, len);
 1c9:   89 f0                   mov    %esi,%eax
 1cb:   e8 fc ff ff ff          call   1cc <broken_read+0x5c>
            1cc: R_386_PC32 copy_to_user
 1d0:   89 c3                   mov    %eax,%ebx
    kfree(user_buf);   
 1d2:   89 f0                   mov    %esi,%eax
 1d4:   e8 fc ff ff ff          call   1d5 <broken_read+0x65>
            1d5: R_386_PC32 kfree

    read_count++;
 1d9:   83 05 00 00 00 00 01    addl   $0x1,0x0
            1db: R_386_32   read_count


    if (!err && *f_pos == 0) {
 1e0:   85 db                   test   %ebx,%ebx
 1e2:   75 1c                   jne    200 <broken_read+0x90>
 1e4:   8b 47 04                mov    0x4(%edi),%eax
 1e7:   0b 07                   or     (%edi),%eax
 1e9:   75 15                   jne    200 <broken_read+0x90>
       return -ENOMEM;
    } else {
       fill_buffer(mybuf,mybuf_size);
    }

    len = strlen(mybuf);
 1eb:   8b 45 f0                mov    -0x10(%ebp),%eax

    read_count++;


    if (!err && *f_pos == 0) {
      *f_pos += len;
 1ee:   89 c2                   mov    %eax,%edx
 1f0:   c1 fa 1f                sar    $0x1f,%edx
 1f3:   89 07                   mov    %eax,(%edi)
 1f5:   89 57 04                mov    %edx,0x4(%edi)
      return len;
 1f8:   eb 08                   jmp    202 <broken_read+0x92>
 1fa:   8d b6 00 00 00 00       lea    0x0(%esi),%esi
 200:   31 c0                   xor    %eax,%eax
    } else {
      return 0;
    }
}
 202:   8b 5d f4                mov    -0xc(%ebp),%ebx
 205:   8b 75 f8                mov    -0x8(%ebp),%esi
 208:   8b 7d fc                mov    -0x4(%ebp),%edi
 20b:   89 ec                   mov    %ebp,%esp
 20d:   5d                      pop    %ebp
 20e:   c3                      ret    
   int mybuf_size = 100;
   int len, err;

    mybuf = kmalloc(mybuf_size, GFP_KERNEL);
    if (!mybuf) {
       broken_exit();
 20f:   e8 fc ff ff ff          call   210 <broken_read+0xa0>
            210: R_386_PC32 broken_exit
 214:   b8 f4 ff ff ff          mov    $0xfffffff4,%eax
       return -ENOMEM;
 219:   eb e7                   jmp    202 <broken_read+0x92>
 21b:   8b 15 10 00 00 00       mov    0x10,%edx
            21d: R_386_32   __tracepoint_kmalloc
 221:   85 d2                   test   %edx,%edx
 223:   89 55 ec                mov    %edx,-0x14(%ebp)
 226:   74 83                   je     1ab <broken_read+0x3b>
 228:   8b 02                   mov    (%edx),%eax
 22a:   89 45 e8                mov    %eax,-0x18(%ebp)
 22d:   8b 55 f0                mov    -0x10(%ebp),%edx
 230:   b8 70 01 00 00          mov    $0x170,%eax
            231: R_386_32   .text
 235:   b9 64 00 00 00          mov    $0x64,%ecx
 23a:   c7 44 24 04 d0 00 00    movl   $0xd0,0x4(%esp)
 241:   00 
 242:   89 14 24                mov    %edx,(%esp)
 245:   89 da                   mov    %ebx,%edx
 247:   ff 55 e8                call   *-0x18(%ebp)
 24a:   83 45 ec 04             addl   $0x4,-0x14(%ebp)
 24e:   8b 45 ec                mov    -0x14(%ebp),%eax
 251:   8b 00                   mov    (%eax),%eax
 253:   85 c0                   test   %eax,%eax
 255:   89 45 e8                mov    %eax,-0x18(%ebp)
 258:   75 d3                   jne    22d <broken_read+0xbd>
 25a:   e9 4c ff ff ff          jmp    1ab <broken_read+0x3b>
 25f:   90                      nop

(I compiled the module with EXTRA_CFLAGS+=g)

now, if I add 0x69 to 0x170 I get 1D9 and by the listing I see

 1d0:   89 c3                   mov    %eax,%ebx
    kfree(user_buf);   
 1d2:   89 f0                   mov    %esi,%eax
 1d4:   e8 fc ff ff ff          call   1d5 <broken_read+0x65>
            1d5: R_386_PC32 kfree

    read_count++;
 1d9:   83 05 00 00 00 00 01    addl   $0x1,0x0
            1db: R_386_32   read_count

so under 1d9 there is variable incrementation. So I am wondering if 1d9 means 'the address under which instruction will be executed when control returns to this function'?

Andna
  • 6,539
  • 13
  • 71
  • 120
  • 1
    The items of the call trace marked with '?' are probably unreliable. That is, some of these may correspond to the actual function calls but some of these may not. The details are here, for example: http://stackoverflow.com/questions/13113384/. So, you are right, no need to look for a recursive calls to `broken_read()`. – Eugene Oct 28 '13 at 08:50
  • 1
    Concerning, `1d9` - yes, the address mentioned in the call trace is the return address, which is greater by sizeof(call near relative) == 5 bytes in this case than the address of the call itself. BTW, it looks like BUG_ON() has been triggered during the execution of kfree(). You can take a look at it, it may provide a better insight into what was going on. – Eugene Oct 28 '13 at 08:55

0 Answers0