Heap Gives Page Fault

Question

I am now getting a page fault which means i am accessing an invalid address. This is most likely to the higher half kernel design i chose to follow. But i could not see where it leads to a page fault. Here is my kernel.c++

#include "types.h"
#include "gdt.h"
#include "stdio.h"
#include "serial.h"
#include "mem.h"
#include "idt.h"
#include "timer.h"
#include "isr.h"
#include "kbd.h"
#include "mouse.h"
#include "irq.h"
#include "string.h"
#include "terminal.h"
#include "multiboot.h"
#include "pmm.h"
#include "heap.h"




//Call all class constructor
//for global objects before
//calling the kernel
typedef void (*constructor)();
extern "C" constructor start_ctors;
extern "C" constructor end_ctors;
extern "C" void callConstructors()
{
    for(constructor* i = &start_ctors; i != &end_ctors; i++)
       (*i)();
}



extern "C" void kernelMain(uint32_t kernel_virtual_end,
        uint32_t  kernel_physical_end,
        uint32_t placeholder,
        uint32_t kernel_physical_start, uint32_t  kernel_virtual_start,
        multiboot_info_t *multiboot_structure,uint32_t magicnumber
        )


{


       cls();
       printf("******KERNEL INFO********\n");
       printf("KERNEL START VIRTUAL 0x%x\n" , kernel_virtual_start);
       printf("KERNEL START PHYSICAL 0x%x\n" , kernel_physical_start);
       printf("KERNEL END VIRTUAL 0x%x\n" , kernel_virtual_end);
       printf("KERNEL END PHYSICAL 0x%x\n" , kernel_physical_end);
       printf("MAGIC 0x%x\n" , magicnumber);
       printf("*************************");
//     printf("********RAM INFO*********\n");
//
//
//     printf("Memory Upper : %d \n", multiboot_structure->mem_upper);
//     printf("Memory Lower : %d \n", multiboot_structure->mem_lower);
//
//     printf("*************************\n");



       gdt gt;
       IDT idt;
       ISR isr;
       IRQ irq;
       SerialPort sp;
       isr.install_isrs();
       irq.install_irqs();
        Timer timer;
        timer.install_timer();
        KBD kbd;

        kbd.install_kbd_driver();


        MOUSE mouse;
        mouse.install_mouse_driver();
        __asm__ __volatile__ ("sti");


        PhyiscalMemoryManager pmm(multiboot_structure);

        KHEAP       kheap;
        char        *ptr;

        Heap heap((&kheap));
        heap.k_addBlock(&kheap, (0x100000+0xC0000000), (0x100000+0xC0000000));
        //ptr = (char*)heap.k_malloc(&kheap, 256);
        //heap.k_free(&kheap, ptr);

   while(1);
   err:
       while(1);
}

Ok you can see i commented out the last lines.. the k_malloc() there is what is causing all the chaos . for some reason it malloc's() bellow 0xC0100000 , which is the minumum size we can go to for the kernel, because we are using virtual addresses and, the 1MB is Grub Related Stuff. Saying that here is my heap.c++ :

#include "heap.h"


int Heap::k_addBlock(KHEAPLCAB *heap, uintptr_t addr, uint32_t size)
{
        KHEAPBLOCKLCAB          *hb;
        KHEAPHDRLCAB            *hdr;

        hb = (KHEAPBLOCKLCAB*)addr;
        hb->size = size;
        hb->used = 0;
        hb->next = heap->fblock;
        heap->fblock = hb;

        hdr = (KHEAPHDRLCAB*)&hb[1];
        hdr->flagsize = hb->size - (sizeof(KHEAPBLOCKLCAB) + 32);

        ++heap->bcnt;

        hb->lastdsize = 0;
        hb->lastdhdr = 0;

        return 1;
}

/*
    Look behind and forward to see if we can merge back into some chunks.
*/
void Heap::k_free(KHEAPLCAB *heap, void *ptr)
{
    KHEAPHDRLCAB                *hdr, *phdr, *nhdr;
    KHEAPBLOCKLCAB              *hb;
    uint32_t                        sz;
    uint8_t                     fg;


    hdr = (KHEAPHDRLCAB*)ptr;
    hdr[-1].flagsize &= ~0x80000000;


    phdr = 0;
    /* find the block we are located in */
    for (hb = heap->fblock; hb; hb = hb->next)
    {
        if (((uintptr_t)ptr > (uintptr_t)hb) && ((uintptr_t)ptr < (uintptr_t)hb + hb->size))
        {

            hdr = (KHEAPHDRLCAB*)((uintptr_t)ptr - sizeof(KHEAPHDRLCAB));


            hdr->flagsize &= ~0x80000000;

            hb->used -= hdr->flagsize;


            if (hdr->prevsize)
            {
                phdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr - (sizeof(KHEAPHDRLCAB) + hdr->prevsize));
            }
            else
            {
                phdr = 0;
            }

            /* get next header */
            nhdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + hdr->flagsize);
            if ((uintptr_t)nhdr >= ((uintptr_t)hb + hb->size))
            {
                nhdr = 0;
            }

            if (nhdr)
            {
                if (!(nhdr->flagsize & 0x80000000))
                {
                    /* combine with it */
                    hdr->flagsize += sizeof(KHEAPHDRLCAB) + nhdr->flagsize;
                    hb->used -= sizeof(KHEAPHDRLCAB);
                    /* set next header prevsize */
                    nhdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + hdr->flagsize);
                    nhdr->prevsize = hdr->flagsize;
                }
            }


            if (phdr)
            {
                if (!(phdr->flagsize & 0x80000000))
                {
                    phdr->flagsize += sizeof(KHEAPHDRLCAB) + hdr->flagsize;
                    hb->used -= sizeof(KHEAPHDRLCAB);
                    hdr = phdr;
                    nhdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + hdr->flagsize);
                    if ((uintptr_t)nhdr < (uintptr_t)hb + sizeof(KHEAPBLOCKLCAB) + hb->size)
                    {
                        nhdr->prevsize = hdr->flagsize;
                    }
                }
            }

            /* optimization */
            if (hdr->flagsize > hb->lastdsize)
            {
                hb->lastdsize = hdr->flagsize;
                hb->lastdhdr = hdr;
            }

            return;
        }
    }

    printf("uhoh ptr:%p\n", ptr);
    for (hb = heap->fblock; hb; hb = hb->next) {
        printf("lcab free looking at block:%p next:%p ptr:%p end:%p\n", hb, hb->next, ptr, (uintptr_t)hb + hb->size);
        if (((uintptr_t)ptr > (uintptr_t)hb)) {
            printf("above\n");
            if (((uintptr_t)ptr < ((uintptr_t)hb + hb->size))) {
                printf("found\n");
            }
        }
    }
    for (;;);
    /* uhoh.. this should not happen.. */
    return;
}

/*
    This will allocate a chunk of memory by the specified size, and if
    no memory chunk can be found it will return zero.
*/
void* Heap::k_malloc(KHEAPLCAB *heap, uint32_t size)
{
    KHEAPBLOCKLCAB      *hb;
    KHEAPHDRLCAB        *hdr, *_hdr, *phdr;
    uint32_t            sz;
    uint8_t             fg;
    uint32_t            checks;
    uint32_t            bc;

    checks = 0;
    bc =0;


    for (hb = heap->fblock; hb; hb = hb->next) {
        if ((hb->size - hb->used) >= (size + sizeof(KHEAPHDRLCAB))) {
            ++bc;

            hdr = (KHEAPHDRLCAB*)&hb[1];


            phdr = 0;
            while ((uintptr_t)hdr < ((uintptr_t)hb + hb->size)) {
                ++checks;

                fg = hdr->flagsize >> 31;
                sz = hdr->flagsize & 0x7fffffff;

                if (!fg)
                {

                    if (sz >= size)
                    {

                        if (sz > (size + sizeof(KHEAPHDRLCAB) + 16))
                        {

                            _hdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + size);
                            _hdr->flagsize = sz - (size + sizeof(KHEAPHDRLCAB));
                            _hdr->prevsize = size;
                            hdr->flagsize = 0x80000000 | size;
                            hb->used += sizeof(KHEAPHDRLCAB);

                        }
                        else
                        {
                            hdr->flagsize |= 0x80000000;
                        }
                        hb->used += size;


                        return &hdr[1];
                    }
                }
                phdr = hdr;
                hdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + sz);
            }

        }
    }

    return 0;
}


Heap::Heap(KHEAPLCAB *heap)
{
    heap->fblock = 0;
    heap->bcnt = 0;
}

Heap::~Heap()
{

}

The problem is with malloc() i think , because when i coment that out, it works all fine. I am accessing some invalid address in that function :

void* Heap::k_malloc(KHEAPLCAB *heap, uint32_t size)
{
    KHEAPBLOCKLCAB      *hb;
    KHEAPHDRLCAB        *hdr, *_hdr, *phdr;
    uint32_t            sz;
    uint8_t             fg;
    uint32_t            checks;
    uint32_t            bc;

    checks = 0;
    bc =0;


    for (hb = heap->fblock; hb; hb = hb->next) {
        if ((hb->size - hb->used) >= (size + sizeof(KHEAPHDRLCAB))) {
            ++bc;

            hdr = (KHEAPHDRLCAB*)&hb[1];


            phdr = 0;
            while ((uintptr_t)hdr < ((uintptr_t)hb + hb->size)) {
                ++checks;

                fg = hdr->flagsize >> 31;
                sz = hdr->flagsize & 0x7fffffff;

                if (!fg)
                {

                    if (sz >= size)
                    {

                        if (sz > (size + sizeof(KHEAPHDRLCAB) + 16))
                        {

                            _hdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + size);
                            _hdr->flagsize = sz - (size + sizeof(KHEAPHDRLCAB));
                            _hdr->prevsize = size;
                            hdr->flagsize = 0x80000000 | size;
                            hb->used += sizeof(KHEAPHDRLCAB);

                        }
                        else
                        {
                            hdr->flagsize |= 0x80000000;
                        }
                        hb->used += size;


                        return &hdr[1];
                    }
                }
                phdr = hdr;
                hdr = (KHEAPHDRLCAB*)((uintptr_t)&hdr[1] + sz);
            }

        }
    }

    return 0;
}

. Help Would Be Appreciated. Here is My Full Source Code in Github : https://github.com/amanuel2/OS_MIRROR

I'm very confused, What memory region are you expecting this to use `heap.k_addBlock(&kheap, (0x100000+0xC0000000), (0x100000+0xC0000000));` I'm unsure what address range you are hoping to allocate as free. Not sure why you added 0xC0000000 to the length, and do you actually mean to add 0xC0000000 to the base address? Have you created Page Directory and Page Table entries for this memory? As it stands `heap.k_addBlock(&kheap, (0x100000+0xC0000000), (0x100000+0xC0000000));` seems to add a free block going from 0xC0100000 with a length of `0xC0100000` which suggests you are going beyond 4GB? — Michael Petch, Aug 23 '16 at 20:24
@MichaelPetch If i delete the `+0xC0000000` .. it gives me a page fault on the k_addBlock() .. and i meant `heap.k_addBlock(&kheap, (0x0 + 0xC0000000), (0x300000+0xC0000000));` : https://github.com/amanuel2/OS_Mirror/blob/master/kernel.c%2B%2B#L89 . sorry that was an accident . What i want to do is add a 3MB Block , and allocate 256 bytes to that block. Basically the Heap. all the heap code came from here, so i can test before implementing : http://wiki.osdev.org/User:Pancakes/EntryBasedHeapImplementation — amanuel2, Aug 23 '16 at 20:51
That will be because you already mapped memory with an initial page table and directory when you created the higher half kernel. You are aware that you effectively told this code as is to overwrite you kernel as a heap area. What this code lacks is anything to add page tables and page directories that map a page frame to a physical address. It is almost like the code you have here was written for a non-paged OS that mapped the entire 4gb region with a data selector. You have enabled paging here, so you have to take that into account. — Michael Petch, Aug 23 '16 at 20:54
Have you considered starting simpler with a non higher half kernel without paging? You might have jumped into this without the prerequisite understanding of paging, page tables, page directories, page frames etc. — Michael Petch, Aug 23 '16 at 20:55
@MichaelPetch i do understand the concept of paging, pt, pd, and pf... (dont really need to worry about pt since of PAE).. This code i did not fully understand — amanuel2, Aug 23 '16 at 20:57
@MichaelPetch When you say overwriting the kernel thats because i add a block in 0xC000000 - 0xC030000, where kernel is stored right? But that didnt give an error what so ever, which is wierd. Ok now know that shouldnt happen no more, because i a doing ` heap.k_addBlock(&kheap, (0x0 + 0xC010CB00), (0x300000+0xC010CB00));` . And 0xC010CA00 is my kernel end. — amanuel2, Aug 23 '16 at 21:14
It doesn't give an error because you are the one who needs to keep track of what memory is used for what purpose. There is no error because the CPU happily realizes that the memory above 0xC0000000 already has some pages allocated, you then write heap info into that area (potentially overwriting the kernel).You need to pick a region that isn't already being used by your kernel and add paging entries for that region of memory (and their access privileges). If not using PAE you need both page tables and page directories. If PAE is enabled you need both those plus a page directory pointer table. — Michael Petch, Aug 23 '16 at 21:17
On top of everything I said about having to create paging entries, I still do not know why you are adding `(0x300000+0xC010CB00)` in the second parameter. That addblock function takes a base pointer and a LENGTH (not an end address). You shouldn't be adding `0xC010CB00` to it. If you want to add a free block of memory to the heap starting at 0xC010CB00 of length 0x300000 then you'd use something like `heap.k_addBlock(&kheap, 0xC010CB00, 0x300000);` . This code assumes that the memory has already had paging entries added for it. — Michael Petch, Aug 23 '16 at 21:21
@MichaelPetch ok i accidently added 0xC010Cb00 , my head was thinking that was the end address.. i converted it to 260, to represnt a size of 260 bytes now — amanuel2, Aug 23 '16 at 21:25
@MichaelPetch ok so.. The diffrence PAE Makes is that you use 4MB Pages instead of 4KB Page Frames.. the concept of page tables goes away when you use PAE . There are 1024 pages in the page directory, and each of the pages contain 1024 (4096 byte) page frames. Thats the diffrence PAE Brings to the table — amanuel2, Aug 23 '16 at 21:27
@MichaelPetch what i am asking is what do i need do in order to complete this heap proccess , and avoid this page fault im recieving — amanuel2, Aug 23 '16 at 21:29
I've already told you. Before you call the addblock function you need to create page entries for the region you want to use as heap. That addblock function doesn't do that for you. As for PAE - Without PAE a system can actually use 4kb or 4mb pages. With PAE enabled you have a choice between 4kb and 2mb pages. — Michael Petch, Aug 23 '16 at 21:31
I think you are confusing PAE and PSE. Looking at your boot.asm code PAE isn't enabled, but the PSE bit is set. That switched you to 4mb pages but you aren't using PAE. Unless you are changing that somewhere else in your code after kernelMain. — Michael Petch, Aug 23 '16 at 21:36
@MichaelPetch oh... i thought PAE And PSE Were the same thing :/ — amanuel2, Aug 23 '16 at 21:47
If you intend to use 4mb pages(PSE) without PAE then you need to consider that you only have a page directory structure (with upto 1024 entries of 4 bytes each) pointing at 4mb pages. 4mb * 1024 = 4gb as in this [diagram](https://upload.wikimedia.org/wikipedia/commons/thumb/d/d9/X86_Paging_4M.svg/400px-X86_Paging_4M.svg.png) . My guess is that the BareBones kernel you originally looked at decided it was simpler to deal with 4MB page frames than 4KB ones. It is of course valid to create a higher half kernel with 4KB pages but you have more work to set up the page related tables (3 of them) — Michael Petch, Aug 23 '16 at 21:54
@MichaelPetch Whats the type of BootPageDirectory , so i can extern it? https://github.com/amanuel2/OS_Mirror/blob/master/boot.asm#L24 .. `extern "C" uint64_t BootPageDirectory[1024];` , seems fine? — amanuel2, Aug 23 '16 at 22:00
No, more like `extern "C" uint32_t BootPageDirectory[1024]` . Each entry in Bootpage directory is 4 bytes long (32-bits). — Michael Petch, Aug 23 '16 at 22:03
No a 32-bit processor can use memory above 4GB with PAE enabled since those processors that support PAE actually have an extra 4 address lines (you can address upto 64GB of RAM). But with the way non-PAE mode and 4MB page frames work (with a single 1024 entry page directory) you are limited to 4GB (4MB * 1024 = 4GB) or less. Most modern processors these days support PAE. If you enable PAE you can use up to 64GB of ram with either 2MB page frames, or 4KB page frames. Without PAE you can addres 4GB with 4MB pages frames or 4KB page frames. — Michael Petch, Aug 23 '16 at 22:12
You can see how all the page tables work for No-PAE/4MB, No-PAE/4KB, PAE/2MB, and PAE/4KB in this Wiki article on [PAE](https://en.wikipedia.org/wiki/Physical_Address_Extension) — Michael Petch, Aug 23 '16 at 22:19
@MichaelPetch Ok i see.. I am not gonna do PAE Now, because it will confuse me lot . Ill rather stick with this for now, and then when i rewrite my OS , then actually enable and study on PAE. — amanuel2, Aug 23 '16 at 22:27
@MichaelPetch Ok so what i need to do in my constructor is make multiple unused pages used as the heap? — amanuel2, Aug 23 '16 at 22:29
You'd create as many 4MB mappings as you need for your heap. But as you have probably now realized you only have 4MB page size granularity if you intend to use the full 4GB of RAM - meaning the minimum heap space will be 4MB and you can add to it in 4MB chunks. You also want to use memory in regions that the memory map say are actually available. The first 4MB chunk after your kernel would be at 0xC0400000 (and be of length 4MB). You'd create a mapping to a linear address by putting an entry in BootPageDirectory at index 769 (769 << 22 = 0xC0400000) — Michael Petch, Aug 23 '16 at 22:52
@MichaelPetch ok for right now let me just assume the only malloc() i am doing is the 26 byte one, that starts at adress 0xC0450000 . So first in the constructor How would i "create as many 4MB mappings as you need for your heap" . I know BootPageDirectory[0] is the first 4MB Page. But how do i map it? — amanuel2, Aug 23 '16 at 23:30
There are 2 different issues. you need to set up mappings using paging before you can even worry about addblock and then malloc. So first things first. Assume your heap is maximum 4MB (one page) for experimentation, what physical memory address that is available on your system (look at memory map) that starts on a 4MB boundary do you wish to use (in an ideal world you'd write code that looks through the memory map passed by GRUB to find one but for test purposes you can find one. — Michael Petch, Aug 23 '16 at 23:50
I should point out as a clarification (to other readers) that enabling PSE doesn't automatically turn on 4MB pages, it turns on support for them. The actual page size is set in the Page Directory entry. You can in theory use a mixture of 4MB and 4KB page frames with appropriate entries in the page directory entry. — Michael Petch, Aug 23 '16 at 23:52
If you are using QEMU, and the output of your [screenshot](http://imgur.com/VOaTe4V) is correct you have physical memory starting at 0x100000 of length 133038080 . That can fit a number of 4MB pages in it. Since we probably don't want to map the lowest 4MB again (part of our higher half kernel) we can use the 4MB region starting at physical address 0x400000 (this is on a 4MB boundary) up to 0x7FFFFF. We want to map the 4MB page frame starting at physical address 0x400000 is (0x400000>>22)=1 to somewhere in virtual memory. — Michael Petch, Aug 24 '16 at 00:14
The next question is where do we want to map that 4MB page frame. It seems you are intent on placing it right after the kernel page that was already allocated. Your BootPageDirectory already has an entry at index 768. (768<22)=0xC0000000(and is 0x400000 bytes long) . The next 4MB high half memory address starts at 0xC0400000. (0xC0400000>>22)=769. So to map the 4MB page frame at virtual address 0xC0400000 to a physical (linear address) we need to update BootPageDirectory[769] . So the question is with what? — Michael Petch, Aug 24 '16 at 00:22
With no-PAE, and 4MB paging we will want to probably set Read/Write(bit 0) and Present(bit 1) bits along with the 4MB(bit 7).The entire bottom 12 bits are defined in this [diagram](http://wiki.osdev.org/images/9/94/Page_dir.png) . With 4MB paging bit set(bit 7) bits 12-21 are set to zero. The upper 10 bits are the page table number of the physical address which we determined to be **1** for the second 4MB region. (1<<22)|(0x83) = 0x400083 (0x83 = read/write,present,4MB bit values).We assign the value 0x400083 to BootPageDirectory[769]. That will map 0xC0400000 to the 4MB @ physaddress 0x400000 — Michael Petch, Aug 24 '16 at 00:33
More information on paging can be found here: http://wiki.osdev.org/Paging — Michael Petch, Aug 24 '16 at 00:38
@MichaelPetch your making me confused now.. You said at first i needed " you need to set up mappings using paging". So your teling me for expierement map 4MB To Heap.. How do i do that? — amanuel2, Aug 24 '16 at 00:54
Do you understand paging, or did you just copy code from some tuorial and it worked? The entire discussion above is specifically to get the PAGING going for the memory address you will use with the heap.. Next you can call `k_addblock` with the base of 0xC0400000 and length 0x400000 since they are now mapped. Then you should be able to call `malloc`. I really do believe you should have started your OS without paging, until you really understood the concept. — Michael Petch, Aug 24 '16 at 00:57
So I have now told you exactly which BootPageDirectory entry to modify for this case, what value to put in it and what to pass into `k_addblock`. Assuming the code you copied actually works you should be able to call `malloc` after the 2 things above are done. — Michael Petch, Aug 24 '16 at 01:01
I Do indeed understand paging.. i will try my best to comprehend what you are trying to say above — amanuel2, Aug 24 '16 at 01:05
If you understood paging then why did you ask me ` How do i do that? `. If you understand paging it should be very easy to set up the Paging directory and the values inside. — Michael Petch, Aug 24 '16 at 01:11
Ok basically whats happening is your trying to put the HEAP in a physical adress that has 4MB , (4mb for exp.) — amanuel2, Aug 24 '16 at 01:16
The reality is that once you turn on paging (which you do in boot.asm) you can not use any arbitrary memory address without creating paging entries that say "here is a mapping from a virtual address to a physical(linear address) and the memory is present and you can read/write to it". Until you do that - reading or writing any data to it will cause a page fault to occur. — Michael Petch, Aug 24 '16 at 01:20
The heap is just one thing that requires memory. When you want to load apps into memory you will need to know how to create pages for that code to run. — Michael Petch, Aug 24 '16 at 01:21
Personally I'd be using 4KB paging, but I'm not sure if you understand how to build a page directory, and page table to that. 4MB is what your boot.asm was using (and appears to be copied from an online tutorial) simplifies it by using a single array to create mappings and set the page bits. — Michael Petch, Aug 24 '16 at 01:27
its like 10 lines of code.. there is no improvments i can do to that.. and just because i copied some lines of code dosent mean i dont fully understand it — amanuel2, Aug 24 '16 at 01:28
On a side note this is what is troubling me "If you are using QEMU, and the output of your screenshot is correct you have physical memory starting at 0x100000 of length 133038080 . That can fit a number of 4MB pages in it. Since we probably don't want to map the lowest 4MB again (part of our higher half kernel) we can use the 4MB region starting at physical address 0x400000 (this is on a 4MB boundary) up to 0x7FFFFF. We want to map the 4MB page frame starting at physical address 0x400000 is (0x400000>>22)=1 to somewhere in virtual memory. –" . Where did you get 0x400000 from? — amanuel2, Aug 24 '16 at 01:29
Convert 0x400000 to decimal. If you understand paging you'd understand. 0x400000 in case you don't have something that converts is 4Megabytes. Each 4MB page is aligned to a 4MB boundary in memory. You can only map pages that are on a boundary(linear/physical address) that is evenly divisible by 0x400000 (or 4MB). If you were using 4KB pages then each page frame is on a 4K boundary. 4MB = 1024*4096=4194304=0x400000 — Michael Petch, Aug 24 '16 at 01:36
Let me try to pick this up tomorrow morning ... I am not capable of understanding this right now.. — amanuel2, Aug 24 '16 at 01:39
The first 4MB page runs from physical(linear) address 0x00000000 to 0x003FFFFF, second one at 0x00400000 to 0x007FFFFF, the third 0x00800000 to 0x00BFFFFF, and so on. 1024 4MB regions is = 4GB (1024*4*1024*1024) — Michael Petch, Aug 24 '16 at 01:41
I've placed 3 modified files that should get things going: http://www.capp-sysware.com/misc/stackoverflow/39109581/ . I added an `invalidate_page_vm` assembler function to invalidate a page that starts at the given virtual memory address. This is needed when you make changes to the TLB cache. I've also added code to do the mapping I discussed above, and provides some examples of declaring the memory region for the heap using `k_addblock`. The code seems to work here. — Michael Petch, Aug 24 '16 at 03:32
I also modified stdio.c++ to at least put some workarounds for the printing related bugs that can display incorrect memory addresses (and values) when using `%x` . You really should resolve those. — Michael Petch, Aug 24 '16 at 03:43
@MichaelPetch i dont understand how this `BootPageDirectory[769]=0x400083;` is mapping physical address 0x400083 to Virtual address 0xC0400000 . the 83 i know means bit 7,1,and2 has been set but how i dont see virtual address anywhere. And also i see that when i print out `BootPageDirectory[768]` as a decimal i get `227` . All the others are 0 (since we zeroed out the first directory in boot.asm). Why is that? Basically what im asking is, whats so special about the 768'th page? It means the kernel is in 768'th page and is max space it can take is only 4MB? — amanuel2, Aug 24 '16 at 10:58
@MichaelPetch And **If** You respond above as the kernel being 4MB, how is this code not overwriting the kernel? `heap.k_addBlock(&kheap, 0xC0200000, 0x300000); ` . It starts at 0xC0200000 which is already in the lines of the kernel because again if you respond yes to the kernel being 4MB At all times , then its virtual adress is 0xC0000000 - 0xC0400000 . and there is also a comment that says `This heap runs from 0xC0200000 to C04FFFFF - 3MB` . I dont see where that is happening at all :/ — amanuel2, Aug 24 '16 at 11:25
Earlier I didn't have the luxury of knowing exactly how much memory your kernel takes in memory until I actually built your project. Placing it in its own page made sense. But as long as the addresses used don't conflict with something your kernel is already using then you can use it. Earlier in the comments you asked for a 3MB heap which is 0x300000. `heap.k_addBlock(&kheap, 0xC0200000, 0x300000);` sets up a 3MB heap running from start address 0xC0200000. You'll notice in one example in the commented code I showed a way to use `kernel_virtual_end` to find a base address for the heap. — Michael Petch, Aug 24 '16 at 14:36
For the moment 0xC0200000 is beyond the extent of your kernel code and data but as your kernel grows you need to adjust that value. That was why I suggested in the code comments this technique `heap.k_addBlock(&kheap, (kernel_virtual_end+4095)&0xFFFFF000, 0x300000);` which uses `kernel_virtual_end` to place a heap after the kernel. I just chose to align it on a 4KB boundary after the kernel. As your kernel grows the heap will appear after it without overlapping. — Michael Petch, Aug 24 '16 at 14:39
As for the question of the number 768. Take the decimal value(index) 768 and shift it left 22 bits (768<<22). In Hex you get 0xC0000000 . That happens to be the base virtual address of your higher half kernel. And yes, because of the way boot.asm did paging the kernel page was 4MB. It actually allocated a second page to identity map the lower 4MB of memory temporarily until paging was completely set up and then removed it. By the time boot.asm called `kernelMain` there was only a single 4MB page active between 0xC0000000 and 0xC03FFFFF which was where your higher half kernel was. — Michael Petch, Aug 24 '16 at 14:51
And that higher half kernel is mapped to the first 4MB of physical memory between 0x00000000 and 0x003FFFFF. — Michael Petch, Aug 24 '16 at 14:53
As for the value 227 in page entry 768. It started 0x083 for the lower 12 bits in `boot.asm` but as the CPU runs and the page directory gets used the bits may change. In non-PAE using 4MB pages the value 227 contains the flags but also represents the physical address being mapped to. Set the lower 22 bits to all zero and the value you get back will be the physical address in memory that the entry maps to. In the case of 0x000000E3(decimal 227) zeroing out the lower 22 bits gives us a value of 0x00000000 . That is the physical address of the 4MB frame being mapped into that entry (768). — Michael Petch, Aug 24 '16 at 15:10
Much of this is discussed (albeit briefly for the 4MB paging) here: http://wiki.osdev.org/Paging . It does say _"Setting the S bit makes the page directory entry point directly to a 4-MiB page. There is no paging table involved in the address translation. Note: With 4-MiB pages, bits 21 through 12 are reserved! Thus, the physical address must also be 4-MiB-aligned."_ . More information specific to PSE/4MB can be found here: https://en.wikipedia.org/wiki/Page_Size_Extension — Michael Petch, Aug 24 '16 at 15:28
@MichaelPetch I am reading your bottom two comments now.. but before that i have couple questions.. How come the end of my kernel memory is `0xC03FFFFF` ? I mean if you add 4MB To its Base Address it will become `0xC0400000` not `0xC03FFFFF` . I understand almost everything about this part `k_addBlock(&kheap, (kernel_virtual_end+4095)0xFFFFF000, 0x300000);` , you are basically trying to "dynamcally" alocate it by using variables instead of 0xC000000 which is static. But the thing i dont understand is , why the 4095 additon, Each pages have 4KB Seperations? And whats up with the `0xFFFFF000`. — amanuel2, Aug 24 '16 at 16:10
`0xC03FFFFF` is one lower than `0xC0400000` (4MB-1). It is 4MB total when you account for the fact that bytes started counting at 0xC0000000. Consider an array defined in C as as `char arr[10]` . Counting starts at 0 (not 1). So bytes run from 0 to 9 for a 10 byte array. Byte at index 10 in 0 based indexing is the 11th element (one beyond the edge of the array). Same thing applies with the last byte in the page is 0xC03FFFFF isntead of one higher at 0xC0400000 — Michael Petch, Aug 24 '16 at 16:16
The 4KB was somewhat arbitrary. In fact you could have started your heap right at the end of the kernel with `k_addBlock(&kheap, kernel_virtual_end, 0x300000);` . I chose 4KB alignment mainly because in your linker script you were using 4KB alignment for your other sections like text, data, bss. I decided to use such alignment for the heap. — Michael Petch, Aug 24 '16 at 16:20
Technically speaking `kernel_virtual_end` points to the address one past the last used byte of your kernel. By adding 4095 and then anding with 0xFFFFF000 you are rounding up to the next 4096 boundary IF `kernel_virtual_end` wasn't already on a 4KB to begin with. If you had added 4096(instead of 4095) and `kernel_virtual_end` was already on a 4KB boundary you would have wasted 4KB of space. — Michael Petch, Aug 24 '16 at 16:24
This type of help is not really suitable for Stackoverflow question and answer format. You might be better off using IRC chat where OSDev people hang out. They are on the `#osdev` channel on Freenode. You can access it through the web based IRC client: https://kiwiirc.com/client/irc.freenode.net/#osdev — Michael Petch, Aug 24 '16 at 16:48
Ok @MichaelPetch im starting to understand this. First off your comment conceringing` #osdev` , i have been there for quite some time now, and trust me you are much better at explaining topics like this thankyou sir. Again im really starting to understand this, i just have two question (one which i will try to solve on my own) , other one is why am i getting as you can see: https://github.com/amanuel2/OS_Mirror/blob/master/kernel.c%2B%2B , i have two mallocs.. And both of them print out "malloc returned C020001C" ... Why is it the same for both? — amanuel2, Aug 24 '16 at 17:11
Oh sorry I deleted my last comment (it didn't apply). You malloc'ed some space and then you **freed** it. Then you malloc'ed again. The free returned that memory from the first malloc back to the heap to be potentially reused. Remove the free between the mallocs and they should be different. — Michael Petch, Aug 24 '16 at 17:22
@MichaelPetch okay .. Understand now. I need to make some type of function that handles if he freed all of his memory. If not make a panic error. Ok so my last problem is that none of my IRQ Are working anymore.. My keyboard,Mouse, or Timer Driver. I will try to fix this on my own, if i cant for a long time, ill ask if you can give me any sort of clues. Thanks! — amanuel2, Aug 24 '16 at 17:24
I noticed the same thing, and the reason is you removed the line `__asm__ __volatile__ ("sti");` which enables interrupts. You use to do this after the mouse interrupt handler was installed but I noticed one of your git commits removed it. — Michael Petch, Aug 24 '16 at 17:27
@MichaelPetch yes .. wow i accidently removed it trying to comment it out.. my bad. Ok well, i will deeply understand this for the whole day , and master this concept in my head. If i have questions i cant find in internet or couldnt solve i hope i can ask. Thankyou! — amanuel2, Aug 24 '16 at 17:30
@MichaelPetch is the length on entry #3 negative here do to my computer or printf function? http://i.imgur.com/s3alkTF.png — amanuel2, Aug 24 '16 at 19:04
I mentioned this issue in another comment somewhere along the line. Your `printf` (using `%d`) and your `itoa` are written in such a way that output is signed when using `%d`. So if the high bit (sign bit) is set and you use `%d` the result will be a negative value. That seems to be the case here. You could write the length out as hexadecimal using your `%x`. Alternatively you'd have to enhance your `printf` to handle printing unsigned values. `Printf` usually has `%u` for printing unsigned integers. — Michael Petch, Aug 24 '16 at 19:25
PS: 1Megabyte(1MB) = 1024*1024 = 1048576 . I noticed you have `printf("%d MB Of Usable RAM" , bytes_usable/1000000);` Rather than divide by 1000000 you should divide by 1048676. — Michael Petch, Aug 24 '16 at 20:32
@MichaelPetch well i just tested it on my real pc hardware.. Seems like when it added up it added it right.. But before, it told me the second entry was negative.. now i had to change it to hex , to check if the total MB Was right. printf gets wrong when i try to print a large number like 2 billion ... — amanuel2, Aug 24 '16 at 21:09
@MichaelPetch hmm wierd because dividing by 1000000 gave me the right anwser.. And google converter tells me im right by dividing by 1000000 .. :/ wierd — amanuel2, Aug 24 '16 at 21:12
Well `printf` with `%d` is doing what is expected it just isn't overly useful to print a unsigned number as a signed one. Implementing `%u` in `printf` would solve that issue. Then you'd use `%u` instead of `%d`. — Michael Petch, Aug 24 '16 at 21:12
@MichaelPetch yeah yea im still doing research implemenetation on that — amanuel2, Aug 24 '16 at 21:13
@MichaelPetch oh the only change i have to make for `%u` is in the var_arg i will do `va_arg(arg,uint32_t);` instead of `va_art(ard,int)` — amanuel2, Aug 24 '16 at 21:15
About the only industry that divides by 1000000 is the Harddrive industry and that is a bit of a sham. For everyone else on a computer 1Megabyte is 1024 * 1024. One of you screenshots (maybe it was in QEMU) showed an unusual number of 133MB and I thought that was strange. Then I realized you divided by 10000000. If you had divided by 1048576 then it would have come out very close to 128MB (which I believe is the memory configuration for a default QEMU session). — Michael Petch, Aug 24 '16 at 21:16
@MichaelPetch Ohhh thats why i was having this 5 byte shy Memory in VB.. Okay i see — amanuel2, Aug 24 '16 at 21:17
@MichaelPetch ok seems like we've fixed the unsigned integer printing.. The same Hex that printed negative before prints the right number when i do %u now .. commit changes : https://github.com/amanuel2/OS_Mirror/commit/2249037d2221c89952f837603e19c7f43dab780b — amanuel2, Aug 24 '16 at 21:52
I see you changed it to `printf("%d MB Of Usable RAM" , bytes_usable/1048676);` . There is a typo. That should be 1048576, not 1048676. You could also write it as `printf("%d MB Of Usable RAM" , bytes_usable/(1024*1024));` — Michael Petch, Aug 25 '16 at 00:32
@MichaelPetch oh okey.. reason why i was off by 1MB . Well i have another question.. the question is i dont get some type of error here: https://github.com/amanuel2/OS_Mirror/blob/master/heap.c%2B%2B#L226 .. You can see i am overwriting 0x400083 from page 769 , to 770 . I guess i dont really understand that part of the proccess .. Like can you show me a little example of how i would allocate 8MB? so i can see how to allocate to multiple pages? Thanks — amanuel2, Aug 25 '16 at 00:57
You only need to pass the virtual address and the physical address (both must be evenly divisible by 0x400000 (4MB) when using 4MB pages as you are here. You don't need the 3rd argument at all so remove it. The index is based upon the VirtualAddress. Take the Virtual Address and shift it right 22 bits. That will be the index you use for BootPageDirectory. — Michael Petch, Aug 25 '16 at 01:58
@MichaelPetch ok i see just the opposite way to get the page index. Well i have this wierd thing going on where when i do this `heap.k_addBlock(&kheap, (kernel_virtual_end+4095)&0xFFFFF000, 0x300000);` , it gives me a page fault.. If i remove the &0xFFFFF000 OR the +4095 , it becomes fine. Or if i remove both it becomes fine.. But if all three of them are included i get this wierd page fault :/ . Didnt notice it until now since i was using qemu , which i booted from binary file.. when i tried it on vb , it gave me a page fault as it booted from ISO — amanuel2, Aug 25 '16 at 02:16
@MichaelPetch And instead of `pmm.map_physical_virtual(0x400000,0xC0400000);` , i would do `pmm.map_physical_virtual(0x800000,0xC0800000);` if i wanted a 8MB to be reserved for my Heap which would be from 0xC0000000-0x800000,0xC0800000 ? If this is a yes, then basically what this function does , is sets up where we can allocate our heap to.. the whole box.. I will paint a diagram , and just tell me if i have a correct illustration or not — amanuel2, Aug 25 '16 at 02:19
I can only imagine there is a bug in that heap code you have. I don't have time to debug it, but It appears it may require the memory area to be set to zero before it starts building a heap data structure. As the first line of your `k_addblock` routine add `Lib::Mem::memset((void *)addr, 0, size);` to zero out the memory first. Then try it with `heap.k_addBlock(&kheap, (kernel_virtual_end+4095)&0xFFFFF000, 0x300000);` . Parts of the memory area just beyond `kernel_virtual_end` isn't used by your kernel but may be non-zero because of debug information and other GCC info loaded there. — Michael Petch, Aug 25 '16 at 02:35
@MichaelPetch Sorry took long, had to go somewhere. Um, well i tought of this well, and this is what i understood of this Memory Management Scheme.. Is this correct idea? Anything wrong? http://i.imgur.com/Wvfp4iY.png (Zoom Out to see more) — amanuel2, Aug 25 '16 at 03:04
It is more or less correct. What is a bit wrong is inside the heap space. 0x100=256 bytes (not 0x10c), but for every chunk of memory you request there is a small piece of data that the heap places in front of the pointer it returns. That data is used to keep track of memory usage within the heap so it can search for free space when you malloc and to reclaim space when it is freed. — Michael Petch, Aug 25 '16 at 03:20
@MichaelPetch one thing that is kinda bothering me though is as you can see the kernel end is 0xC010CB20 . 0x100 is 256 bytes you said. So if you add 0x100 to 0xC010CC20 . If thats the end of the k_malloc() , then how is it saying that "malloc returned 0xC010CB3C" ? thats more than the end address of that specific malloc :/ ... — amanuel2, Aug 25 '16 at 03:34
As I said there is data before every malloc'ed item. You allocated 256 bytes, but the heap had to add an additional number of bytes just before the pointer it returned. — Michael Petch, Aug 25 '16 at 03:35
@MichaelPetch oh the header that gets the flag and size. ok i get it — amanuel2, Aug 25 '16 at 03:38
@MichaelPetch Ok well, here is my improved version. I have to go sleep now. Thanks! http://i.imgur.com/oAkKu13.png — amanuel2, Aug 25 '16 at 03:41
@MichaelPetch well they said one thing i have missing is dynamically grow the heap.. And they said you dont need this `pmm.map_physical_virtual(0x400000,0xC0400000);`... Tried to explain it here: http://stackoverflow.com/questions/39152751/how-to-implement-dynamically-growing-heap — amanuel2, Aug 25 '16 at 18:55

Heap Gives Page Fault

0 Answers0