JTable autoresize issue

Question

I'm creating a program in which user can record and analyze the start time and end time of an activity. I think it is having an autoresize problem: at the start of the program, the table shrinks weirdly. But when I click on any cell, it becomes normal.

Below is my code

import java.awt.*;
import javax.swing.*;
import javax.swing.border.EtchedBorder;
import javax.swing.table.*;
import java.awt.event.*;

public class TestTimeLog {
private static int COLUMN_NUM = 4;
private int rowNum = 11;
private JTable table;
private JPanel tablePanel;
public static int FRAME_WIDTH = 700;
public static int FRAME_HEIGHT = 800;

/*
 * Panel that holds the table
 */
public JPanel TimeLogPanel() {

    JPanel timeLogPanel = new JPanel();
    // Create border layout with horizontal gap and vertical gap between components
    timeLogPanel.setLayout(new BorderLayout(20, 20));

    // Set border with parameters of top, left, bottom and right spacing
    timeLogPanel.setBorder(BorderFactory.createEmptyBorder(20, 40, 40, 20));
    timeLogPanel.add(tablePanel, BorderLayout.CENTER);

    // Create table panel and set layout
    tablePanel = new JPanel();
    tablePanel.setLayout(new BorderLayout());

    // Add a component listener to be able to calculate the panel size, thus
    // calculate width and height of column
    tablePanel.addComponentListener(new ComponentListener() {

        // Override component resized method and change cell width and height
        @Override
        public void componentResized(ComponentEvent e) {
            if (tablePanel.getHeight() / rowNum >= 1) {
                table.setRowHeight(tablePanel.getHeight() / rowNum);
            }
            table.setAutoResizeMode(JTable.AUTO_RESIZE_OFF);
            table.getColumnModel().getColumn(0).setPreferredWidth(tablePanel.getWidth() / 5);
            table.getColumnModel().getColumn(1).setPreferredWidth(tablePanel.getWidth() / 5);
            table.getColumnModel().getColumn(2).setPreferredWidth(tablePanel.getWidth() / 2);
            table.getColumnModel().getColumn(3).setPreferredWidth(tablePanel.getWidth() / 10);
        }

        // Below methods are not used but must be there, as a requirement when adding a
        // listener
        @Override
        public void componentMoved(ComponentEvent e) {
        }

        @Override
        public void componentShown(ComponentEvent e) {
        }

        @Override
        public void componentHidden(ComponentEvent e) {
        }
    });

    // Create a new table with a table model (defined in a different method below)
    table = new JTable(tableModel());

    // Change appearance
    table.setShowGrid(true);
    table.setGridColor(Color.BLACK);
    table.setSelectionBackground(Color.WHITE);
    table.setSelectionForeground(Color.BLACK);

    // Set a raised border
    table.setBorder(new EtchedBorder(EtchedBorder.RAISED));

    // Add table to JScrollPane which is added to table panel
    JScrollPane scrollPane = new JScrollPane(table);
    tablePanel.add(scrollPane);

    return timeLogPanel;
}

/*
 * This method overrides a table model so that it shows desired column names
 */
public DefaultTableModel tableModel() {
    DefaultTableModel model = new DefaultTableModel(rowNum, COLUMN_NUM) {

        private static final long serialVersionUID = 1L;
        String[] columnNames = { "Start Time", "End Time", "Activity", "Priority" };

        @Override
        public String getColumnName(int index) {
            return columnNames[index];
        }
    };
    return model;
}

/*
 * This main method creates a JFrame that holds the JPanel
 */
public void main(String[] args) {

    // Create a new frame
    JFrame guiFrame = new JFrame("Time Log Program");

    // Set size of the frame
    guiFrame.setSize(FRAME_WIDTH, FRAME_HEIGHT);

    // Add the panel
    guiFrame.add(TimeLogPanel());

    // Exit normally on closing the window
    guiFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);

    // Show the frame
    guiFrame.setVisible(true);
    }
}

Somewhere in the code, I did turn off auto resize mode. But the weird display still happens, and I've go not idea how to fix it.

I appreciate any suggestions. Thanks!

EDIT: Also, half of the time I run the program, it will show up perfectly right (a normal table), and half of other time a shrank table appears. It's so weird.