Hex Character bit rotation

Question

I wrote the following x86-64 functions to be called in a C program: The first one takes in a 2-digit hexadecimal character and rotates its bit towards the right by one bit, i.e, if '12'(means 0x12 but 0x is not feeded as input) is rotated right by one bit to give '09'(0x09)

.file "rotate_right.s"
.section .rodata
.data
.globl rotate_right
.type   rotate_right, @function
.text
rotate_right:
pushq %rbp
movq %rsp,%rbp

pushq %rsi
pushq %rdi
pushq %rbx

subl $4, %esp
movb 8(%ebp), %al
sarb $1, %al

leave
ret
.size rotate_right, .-rotate_right

Similarly, this function rotates the bits to the left one position, so '12'(0x12) becomes '24'(0x24).

.file "rotate_left.s"
.section .rodata
.data
.globl rotate_left
.type   rotate_left, @function
.text
rotate_left:
pushq %rbp
movq %rsp,%rbp

pushq %rsi
pushq %rdi
pushq %rbx

subl $4, %esp
movb 8(%ebp), %al
sarb $1, %al

leave
ret
.size rotate_left, .-rotate_left

The create_key() function gets a four bit input like 0110 and outputs an 8-bit output as unsigned int i.e 01100110:

.file "create_key.s"
.section .rodata
ask_key:
.string "Enter 4-bit key:"
.data
.globl create_key
.type   create_key, @function
.text
create_key:
pushq %rbp
# stack holding
movq %rsp, %rbp
movl $ask_key, %edi
# printing the ask string
movl $0, %eax
# calling the C functions

call printf
movl $0,%esi
# rsi is set to 0. the key
pushq %rsi
# take it to the stack

# call getchar for getting each key bit
call getchar
popq %rsi
subl $48,%eax
# doing this will give us 1 or 0 if the we input 1 or 0
sall $1,%esi
# shift the key by one bit
orl %eax,%esi
# OR key and the sigle input
pushq %rsi
# push rsi to stack to save the value in rsi

# Do the above operation a total of 4 times
call getchar
popq %rsi
subl $48,%eax
sall $1,%esi
orl %eax,%esi
pushq %rsi

call getchar
popq %rsi
subl $48,%eax
sall $1,%esi
orl %eax,%esi
pushq %rsi

call getchar
popq %rsi
subl $48,%eax
sall $1,%esi
orl %eax,%esi

# copy first 4-bits of the key into %rax
movl %esi,%eax
#left shift the key 4 times
sall $4,%esi
# OR the secont 4-bits into %rax
orl %esi,%eax
leave
ret
# return the values and end the function
.size create_key, .-create_key

Here is the C-program,

#include<stdio.h>
#include<stdlib.h>
#include<math.h>
#include<string.h>

extern unsigned int create_key();
extern unsigned char rotate_left(unsigned char x);
extern unsigned char rotate_right(unsigned char x);

int main(){
/* This Part Takes The Input To Be Ciphered 
    And Prints The Hexadecimal Value*/
    
char word[200], outword[400], xor_hex[400], hex_rot[400], ch, hex[2];
unsigned int i_key, encodant, rotated;
int i, len, j;
/* i_key is the integer equvivalent of the cipher key*/

printf("enter cleartext:");

i = 0;
ch = getchar();
while(ch!='\n'){
    word[i] = ch;
    ++i;
    ch = getchar();
}
fflush(stdin);
len = i;
word[i] = '\0';
printf("%s\n", word);
printf("hex encoding:\n");
for(i = 0; i<len; i++){
    sprintf(outword+(i*2), "%02X", word[i]);
}
for(i=0;i<(2*len);i++){
    if(i%2==0 & i>0)printf(" ");
    if(i%20==0 & i>0)printf("\n");
    printf("%c", outword[i]);
}
printf("\n");

/* This Part Asks For The Cipher Key */
i_key =  create_key();

/* XOR Encoding of the Hex Cyphertext*/
for(i=0;i<len*2;i+=2){
    hex[0] = outword[i];
    hex[1] = outword[i+1];
    encodant = (int)strtol(hex, NULL, 16);
    sprintf(xor_hex+(i), "%02X", (i_key^encodant));
}

/* Encoding the text using bit rotation */
j=1;
for(i=0;i<len*2;i+=2){
    hex[0]=xor_hex[i];
    hex[1]=xor_hex[i+1];
    encodant = (int)strtol(hex, NULL, 16);
    if(j%2==0)rotated = rotate_right(encodant);
    else rotated = rotate_left(encodant);
    j++;
    sprintf(hex_rot+(i), "%02X", rotated);
}

/* Printing The Finished Ciphered Text */
printf("hex ciphertext:\n");
for(i=0;i<(2*len);i++){
    if(i%2==0 & i>0)printf(" ");
    if(i%20==0 & i>0)printf("\n");
    printf("%c", hex_rot[i]);
}
printf("\n");

return 0;
}

The function prototypes can't be changed, i.e rotate functions must be char and have char parameters, the create_key function is good appearently, but my code gives segmentation fault. I don't know what to do in this situation so any help is appreciated.

Answer 1

There is no need to do anything with the stack here. Take the argument from rdi (SysV) or rcx (Win32), put it in al, rotate and return:

    .file "rotate.S"

    .text

    .globl rotate_right
rotate_right:
    mov %rdi, %rax
    shrb $1, %al
    ret

    .globl rotate_left
rotate_left:
    mov %rdi, %rax
    shlb $1, %al
    ret

    .end

Now that is GNU as syntax, it may require some adjustment for AT&T asm.

To test it:

#include <stdio.h>

unsigned char rotate_left(unsigned char);
unsigned char rotate_right(unsigned char);

int main() {
    printf("%02x %02x\n", rotate_left(0x12), rotate_right(0x12));
    return 0;
}

Prints:

24 09