Navigate up not back using fragments

Question

I am creating an Android app using one main activity and multiple fragments. I am using the navigation drawer which has two items in it - Fragment A and Fragment B. From Fragment A you can go to 2 lower level fragments and the same for Fragment B. In my case there is a button in Fragment B.1 and Fragment B.2 which allows you to go directly to Fragment A.3. (see image below)

This all works well with the back stack however I want to also implement the home Up button. Usually if you are using multiple activities you would use the NavUtils class but I can't use this for fragments. Instead I have implemented the solution found here. Unfortunately this does not follow the navigation pattern correctly in my case as when I click on the button in Fragment B.2 to go to Fragment A.3 and then click the the up button it takes me back to Fragment B.2 instead of up to Fragment A.2.

The navigation pattern I am talking about is shown here: i.stack.imgur.com/EsoTl.png

My app navigation looks like this: i.stack.imgur.com/8WjBr.png

The code:

private FragmentManager.OnBackStackChangedListener
        mOnBackStackChangedListener = new FragmentManager.OnBackStackChangedListener() {
    @Override
    public void onBackStackChanged() {
        setActionBarArrowDependingOnFragmentsBackStack();
    }
};


@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.main);

    // get the title of the current page
    title = drawerTitle = getTitle();

    // load slide menu item titles
    navMenuTitles = getResources().getStringArray(R.array.nav_menu_items);

    navMenuLayout = (DrawerLayout) findViewById(R.id.nav_menu_layout);
    navMenuList = (ListView) findViewById(R.id.nav_menu_list);

    navMenuItems = new ArrayList<NavMenuItem>();

    // Instantiate a NavMenuItem object and add it to an ArrayList.
    for (int i = 0; i < navMenuTitles.length; i++){
        navMenuItems.add(new NavMenuItem(navMenuTitles[i]));
    }

    // setting the nav menu list adapter
    adapter = new NavMenuListAdapter(getApplicationContext(),navMenuItems);
    navMenuList.setAdapter(adapter);

    // enabling action bar app icon and behaving it as toggle button
    getActionBar().setDisplayHomeAsUpEnabled(true);
    getActionBar().setHomeButtonEnabled(true);

    // setup the onItemClickListener to load correct page when user clicks on navigation item
    navMenuList.setOnItemClickListener(new AdapterView.OnItemClickListener() {
        @Override
        public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
            displayFragment(position);

        }
    });


    mDrawerToggle = new ActionBarDrawerToggle(this, navMenuLayout,
            R.drawable.ic_drawer, //nav menu toggle icon
            R.string.app_name, // nav drawer open - description for accessibility
            R.string.app_name // nav drawer close - description for accessibility
    ){
        public void onDrawerClosed(View view) {
            getActionBar().setTitle(title);
        }

        public void onDrawerOpened(View drawerView) {
            getActionBar().setTitle(drawerTitle);
        }
    };

    navMenuLayout.setDrawerListener(mDrawerToggle);
    getSupportFragmentManager().addOnBackStackChangedListener(mOnBackStackChangedListener);

    // if this is the first time the app is being opened then open the first page from the list.
    if (savedInstanceState == null) {
        // on first time display view for first nav item
        displayFragment(0);
    }

}

@Override
protected void onDestroy() {
    getSupportFragmentManager().removeOnBackStackChangedListener(mOnBackStackChangedListener);
    super.onDestroy();
}

private void setActionBarArrowDependingOnFragmentsBackStack() {
    int backStackEntryCount =
            getSupportFragmentManager().getBackStackEntryCount();
    mDrawerToggle.setDrawerIndicatorEnabled(backStackEntryCount == 0);
}

@Override
public boolean onOptionsItemSelected(MenuItem item) {

    if (mDrawerToggle.isDrawerIndicatorEnabled() &&
            mDrawerToggle.onOptionsItemSelected(item)) {
        return true;
    } else if (item.getItemId() == android.R.id.home &&
            getSupportFragmentManager().popBackStackImmediate()) {
        return true;
    } else {
        switch (item.getItemId()) {
                // handle other items in actionbar
            default:
                return super.onOptionsItemSelected(item);
        }
    }
}