I presume that you want as result also an integer.
For the files:
; uname -m -o: x86_64 GNU/Linux
; Calling convention: System V AMD64 ABI
; Name: isqrt.asm
; Assemble: nasm -felf64 isqrt.asm
; Link: gcc -m64 isqrt.o
; Run: ./a.out
; b := value
; a := 0
; while (b-a)>1 => while b > a (integer!)
; b := ((b+a)/2)
; a := (value/b)
; end_while
; result := b
extern printf
%MACRO PRINT 1
; extern printf
mov rdi, %1
mov rsi, rax
xor eax, eax
call printf
%ENDM
section .data
value: dq 1000
fmt1: db `isqrt32: rax=%u\n`,0
fmt2: db `isqrt64: rax=%u\n`,0
fmt3: db `isqrt_fpu: rax=%u\n`,0
section .text
global main:
main:
mov edi, [value]
call isqrt32
PRINT fmt1
mov rdi, [value]
call isqrt64
PRINT fmt2
mov rdi, [value]
call isqrt_fpu
PRINT fmt3
xor al,al ; Exitcode 0
ret
isqrt64:
mov rbx, rdi
xor rax, rax
.while:
cmp rax, rbx
jnb .endwhile
add rbx, rax
shr rbx, 1
mov rax, rdi
xor rdx, rdx
div rbx
jmp .while
.endwhile:
mov rax, rbx
ret
isqrt32:
mov ebx, edi
xor eax, eax
.while:
cmp eax, ebx
jnb .endwhile
add ebx, eax
shr ebx, 1
mov eax, edi
xor edx, edx
div ebx
jmp .while
.endwhile:
mov eax, ebx
ret
isqrt_fpu:
push rdi
fild qword [rsp]
fsqrt
fisttp qword [rsp]
pop rax
ret