As a beginner in assembly x86, I started to code a program, a keygenme like,
like in CTF challenges. So I wrote a very basic one with all strings encoded but xored etc... At the moment it's absolutly easy to reverse but that's not my problem. For copple hours during, I have a segmentation fault at the very early beginning.
As is did not change anything in between, I launched it with Valgrind and he found a brk segment overflow. Could you help me to find the problem ? I am not really familiar with the concept of brk segment yet.
Here is my code :
%define BUFF_IN_SIZE 64
%define PAGE_SIZE 4096
%define SIGTRAP 5
%define PROT_READ 0x1 ; for mmap
%define PROT_WRITE 0x2 ; pour mmap ah non
%macro print 2 ; macro for syscall write
mov eax, 4
mov ebx, 1
mov ecx, %1
mov edx, %2
int 0x80
%endmacro
%macro input 2 ; syscall read
mov eax, 3
mov ebx, 0
mov ecx, %1
mov edx, %2
int 0x80
%endmacro
%macro exit 0
xor ebx, ebx
mov eax, 1
int 0x80
%endmacro
section .bss
buffinp resb 128
bufftemp resb 64
buffhello resb 64
errbuff resb 64
keysclear resw 4
xorkey resw 1
leninput resb 1
xorlen resb 1
errlen resb 1
keyindex resb 1
dummy resb 1
; welcome = "Password : "
; cryflag = encrypted password
; successdone = "This is the flag"
; fail = "Wrong password !"
section .data
cryflag dw 0x14D2, 0x1533, 0x17F4, 0x1735, 0x14B6, 0x17F7, 0x1738, 0x1459, 0x153A, 0x141B, 0x153C, 0x157D, 0x153E, 0x145F, 0x1520, 0x1501
welcome dw 0x0954, 0x0655, 0x0996, 0x0637, 0x0C38, 0x0E19, 0x0C5A, 0x0C5B, 0x0CDC, 0x0FDD, 0x0C7E, 0x0EBF, 0x0640, 0x0501, 0x0642
succesdone dw 0x3721, 0x30A2, 0x3263, 0x3804, 0x3825, 0x3806, 0x3687, 0x3108, 0x30A9, 0x380A, 0x30CB, 0x318C, 0x302D, 0x30EE, 0x380F, 0x3130, 0x3271, 0x3812, 0x3B53, 0x3814
fail dw 0x0C86, 0x0827, 0x0B88, 0x0BA9, 0x0A8A, 0x026B, 0x086C, 0x0A4D, 0x080E, 0x080F, 0x0890, 0x0B91, 0x0832, 0x0AF3, 0x0274, 0x0255
; keys = array of each xor key
keys dw 0x234, 0x3c01, 0x666, 0x1992
debug db "Reached this part !", 0xa, 0
lendbg equ $ - debug
section .text
global strlen
global strcmp
global asciisum
global xordecrypt
global checkvm
global handler
global _start
_start:
; use it only when it will be a ctf
mov eax, 48 ; sys_signal
mov ebx, SIGTRAP ; SIGTRAP for int 0x3
mov ecx, handler ; it exits
int 0x80
; decrypt array of word to array of bytes for each string
mov esi, fail
mov edi, errbuff
mov ebx, 0x666
mov byte [xorlen], 16
call xordecrypt
mov byte [errlen], 16
; second string
mov esi, welcome
mov edi, buffhello
mov ebx, 0x234
mov byte [xorlen], 15
;********************** segmentation fault here
call xordecrypt
; prints the loggin message
print buffhello, [xorlen]
; waiting for the response of user
mov ebx, 0
mov ecx, buffinp
mov edx, BUFF_IN_SIZE
.invalid:
mov eax, 3
int 0x80
cmp byte [ecx + eax - 1], 0xa ; check if LF is last in buffer input
je .exitinput ; yes, the input is valid
mov ecx, dummy ; otherwise flushing the buffer
mov edx, 1 ; and segfault if overflowing dummy byte
jmp .invalid
.exitinput: ; compute size of user input
mov edi, buffinp
call strlen
test al, al
jz .failure
mov byte [leninput], al
; decrypting the harcoded encrypted password here
mov byte [xorlen], al
mov esi, cryflag
mov edi, bufftemp
mov ebx, 0x1992
call xordecrypt
; compares it with the user one
mov esi, buffinp
mov edi, bufftemp
mov ecx, 16 ;[xorlen]
call strcmp
test al, al
jnz .failure
; success in this case
mov esi, succesdone
mov edi, bufftemp
mov ebx, 0x3c01
mov byte [xorlen], 20
call xordecrypt
print edi, [xorlen]
exit
; priting wrong password and exiting
.failure:
print errbuff, [errlen]
exit
; strlen implementation with any control char terminated strings
strlen:
push ecx
xor ecx, ecx
.strlen_next:
cmp [edi + ecx], byte 0x16 ; below 20 so 0, \n all works
jb .strlen_null
inc ecx
jo .strlen_over
jmp .strlen_next
.strlen_over:
print errbuff, [errlen]
exit
.strlen_null:
mov eax, ecx
pop ecx
ret
; basic strcmp without any obfuscation
; eax = 0 if equals
strcmp:
cld
repe cmpsb
jcxz .retcmp
jmp .failcmp
.failcmp: ; fail ? set up ecx on != 0
or ecx, 0x1
jno .retcmp ; jmp like so
xchg ax, ax
.retcmp:
mov eax, ecx
ret
; *************************************
; esi = addresse du word array source
; edi = addresse du byte array dest
; bx = xor key
; ************************************
xordecrypt:
xor ecx, ecx
xor edx, edx
.decrypting:
mov ax, word [esi + edx * 2]
xor ax, bx ; first xoring with key
mov cl, 5
shr ax, cl ; right shift of 5 to get a byte
mov byte [edi + edx], al ; copy the bytes in array destination
inc edx
inc bx ; the key is incremented
cmp edx, [xorlen]
je .xendecrypt
jmp .decrypting
.xendecrypt: ; end decrypt
xor ebx, ebx
ret
; not used here (compute ascii sum of string)
asciisum:
push ebx
xor ecx, ecx
xor ebx, ebx
.summing:
mov al, byte [esi + ecx]
add ebx, eax
inc ecx
cmp ecx, edx
je .retsum
jmp .summing
.retsum:
mov eax, ebx
pop ebx
ret
; looking for guest VM enabled or not
checkvm:
xor eax, eax
or eax, 1
cpuid ; cpuid with eax = 1
mov edx, ecx ; stores ecx value in edx
mov cl, 15
shr edx, cl ; set the vm bit in 16th
and edx, 0x8000 ; masks the lower ones
jnz .vmyes ; ZF == 0 ? VM guest or return
ret
.vmyes: ; VM guest ? exiting
exit
handler: ; handler that not exits when SIGTRAP (for int 0x3)
nop ; waiting a little
xchg ax, ax
ret
; we compute the size of entire file
filesize equ $ - $$
And here is the valgrind report :
==84180== Memcheck, a memory error detector
==84180== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.
==84180== Using Valgrind-3.18.1-42b08ed5bd-20211015 and LibVEX; rerun with -h for copyright info
==84180== Command: ./keygenme
==84180== Parent PID: 1467
==84180==
--84180--
--84180-- Valgrind options:
--84180-- --leak-check=full
--84180-- --show-leak-kinds=all
--84180-- --track-origins=yes
--84180-- --verbose
--84180-- --log-file=xxxvllllllllalgrind-out.txt
--84180-- Contents of /proc/version:
--84180-- Linux version 5.18.0-kali2-amd64 (devel@kali.org) (gcc-11 (Debian 11.3.0-3) 11.3.0, GNU ld (GNU Binutils for Debian) 2.38) #1 SMP PREEMPT_DYNAMIC Debian 5.18.5-1kali1 (2022-06-20)
--84180--
--84180-- Arch and hwcaps: X86, LittleEndian, x86-mmxext-sse1-sse2-sse3-lzcnt
--84180-- Page sizes: currently 4096, max supported 4096
--84180-- Valgrind library directory: /usr/libexec/valgrind
--84180-- Reading syms from /home/hackneolys/Documents/Prog/C/learning/Assembly/keygenme
--84180-- object doesn't have a dynamic symbol table
--84180-- Reading syms from /usr/libexec/valgrind/memcheck-x86-linux
--84180-- Considering /usr/lib/debug/.build-id/d8/9dade77ff7a116220489c1e8df98cd7e123687.debug ..
--84180-- .. build-id is valid
--84180-- object doesn't have a dynamic symbol table
--84180-- Scheduler: using generic scheduler lock implementation.
--84180-- Reading suppressions file: /usr/libexec/valgrind/default.supp
==84180== embedded gdbserver: reading from /tmp/vgdb-pipe-from-vgdb-to-84180-by-hackneolys-on-???
==84180== embedded gdbserver: writing to /tmp/vgdb-pipe-to-vgdb-from-84180-by-hackneolys-on-???
==84180== embedded gdbserver: shared mem /tmp/vgdb-pipe-shared-mem-vgdb-84180-by-hackneolys-on-???
==84180==
==84180== TO CONTROL THIS PROCESS USING vgdb (which you probably
==84180== don't want to do, unless you know exactly what you're doing,
==84180== or are doing some strange experiment):
==84180== /usr/bin/vgdb --pid=84180 ...command...
==84180==
==84180== TO DEBUG THIS PROCESS USING GDB: start GDB like this
==84180== /path/to/gdb ./keygenme
==84180== and then give GDB the following command
==84180== target remote | /usr/bin/vgdb --pid=84180
==84180== --pid is optional if only one valgrind process is running
==84180==
==84180== Invalid read of size 2
==84180== at 0x80491A4: ??? (in /home/hackneolys/Documents/Prog/C/learning/Assembly/keygenme)
==84180== Address 0x804b000 is 0 bytes after the brk data segment limit 0x804b000
==84180==
==84180== Invalid read of size 2
==84180== at 0x80491A6: ??? (in /home/hackneolys/Documents/Prog/C/learning/Assembly/keygenme)
==84180== Address 0x804b002 is 2 bytes after the brk data segment limit 0x804b000
==84180==
==84180== Invalid write of size 1
==84180== at 0x8049198: ??? (in /home/hackneolys/Documents/Prog/C/learning/Assembly/keygenme)
==84180== Address 0x804b000 is 0 bytes after the brk data segment limit 0x804b000
==84180==
==84180==
==84180== Process terminating with default action of signal 11 (SIGSEGV)
==84180== Access not within mapped region at address 0x804C000
==84180== at 0x80491A4: ??? (in /home/hackneolys/Documents/Prog/C/learning/Assembly/keygenme)
==84180== If you believe this happened as a result of a stack
==84180== overflow in your program's main thread (unlikely but
==84180== possible), you can try to increase the size of the
==84180== main thread stack using the --main-stacksize= flag.
==84180== The main thread stack size used in this run was 8388608.
==84180==
==84180== HEAP SUMMARY:
==84180== in use at exit: 0 bytes in 0 blocks
==84180== total heap usage: 0 allocs, 0 frees, 0 bytes allocated
==84180==
==84180== All heap blocks were freed -- no leaks are possible
==84180==
==84180== ERROR SUMMARY: 2389 errors from 3 contexts (suppressed: 0 from 0)
==84180==
==84180== 340 errors in context 1 of 3:
==84180== Invalid write of size 1
==84180== at 0x8049198: ??? (in /home/.../Documents/Prog/C/learning/Assembly/keygenme)
==84180== Address 0x804b000 is 0 bytes after the brk data segment limit 0x804b000
==84180==
==84180==
==84180== 1024 errors in context 2 of 3:
==84180== Invalid read of size 2
==84180== at 0x80491A6: ??? (in /home/.../Documents/Prog/C/learning/Assembly/keygenme)
==84180== Address 0x804b002 is 2 bytes after the brk data segment limit 0x804b000
==84180==
==84180==
==84180== 1025 errors in context 3 of 3:
==84180== Invalid read of size 2
==84180== at 0x80491A4: ??? (in /home/.../Documents/Prog/C/learning/Assembly/keygenme)
==84180== Address 0x804b000 is 0 bytes after the brk data segment limit 0x804b000
==84180==
==84180== ERROR SUMMARY: 2389 errors from 3 contexts (suppressed: 0 from 0)
