Click and drag with fabric js and canvas - many objects creating a delay in canvas updates

Question

Creating an app using react and html5 canvas. It is a simple grid of fabric rectangle objects. On a click or click and drag the cells will change color. The grid is resizable and the coloring works fine for 20x20 grid, but when I update to 50x50 there is a delay and some of the cells in the path don't update. I'm assume I am doing something wrong as far as rendering/handling the color change. Any help is appreciated!

class Grid extends Component {
    constructor(props){
        super(props);
        this.state = {
            height: 500,
            width:500,
            gaugeFactor: 1.4,
            mousedown: false
        }
    }

    componentDidMount(){
        this.init();
    }

    init = () => {
        var canvas = new fabric.Canvas('canvas', {renderOnAddRemove: false});
        var grid = [];
    
        canvas.selection = false;
        fabric.Object.prototype.objectCaching = false;
        
        this.generateGrid(canvas, grid);
        this.allowColoring(canvas);

        canvas.renderAll();
        this.setState({ canvas });
        this.setState({ grid: grid });
      };
    

    generateGrid = (canvas, grid) => {
        var xPos = -1
        var yPos = -1
        var x = 0, y = 0;
        while(y < this.props.rows){
            var row = [];
            while(x < this.props.stitches){

                var rect = new fabric.Rect({
                    left: xPos,
                    top: yPos,
                    fill: 'white',
                    width: this.state.height/this.props.rows * this.props.gaugeFactor,
                    height: this.state.height/this.props.rows,
                    stroke: 'black',
                    strokeWidth: 1,
                    selectable: false,
                    hoverCursor: 'default',
                    
                });
                
                row.push(rect);

      
                canvas.add(rect);
                xPos += this.state.height/this.props.rows * this.props.gaugeFactor;
                x += 1;
            }
            yPos += this.state.height/this.props.rows;
            xPos = -1;
            y += 1;
            x = 0;
            grid.push(row);
        }
    }

    allowColoring = (canvas) => {
        canvas.on('mouse:down', (options) => {
            this.setState({mousedown: true})
                if(options.target){
                    options.target.set('fill', this.props.activeColor);
                }
        });

        canvas.on('mouse:up', () => {
            this.setState({mousedown: false})
        });

        canvas.on('mouse:over', (options)=>{
            if(options.target){
                if(this.state.mousedown){
                    options.target.set('fill', this.props.activeColor);
                    canvas.renderAll();
                }
            }
        });
    }
    
    updateGauge = () => {
        var grid = this.state.grid;
        var canvas = this.state.canvas;

        var x = 0, y = 0;
        var xPos = -1, yPos = -1;

        while(y < this.props.rows){
            while(x < this.props.stitches){
             
                grid[y][x].set({
                    left: xPos,
                    top: yPos,
                    width: this.state.height/this.props.rows * this.props.gaugeFactor,
                    height: this.state.height/this.props.rows,
                });
                grid[y][x].setCoords();
         
                xPos += this.state.height/this.props.rows * this.props.gaugeFactor;
                x += 1;
            }
            yPos += this.state.height/this.props.rows;
            xPos = -1;
            y += 1;
            x = 0;
        }
        canvas.renderAll();
    }

    clearCanvas = () => {
        var grid = this.state.grid;
        var canvas = this.state.canvas;

        var x = 0, y = 0;

        while(y < this.props.rows){
            while(x < this.props.stitches){
       
                grid[y][x].set({
                    fill: '#ffffff',
    
                });
    
                x += 1;
            }
            y += 1;
            x = 0;
        }
        
        canvas.renderAll();
    }
    render() {
        return (
            <div id="canvasContainer" style = {containerStyle}>
                <canvas id="canvas"  width="1000" height="1000" style = {canvasStyle}></canvas>
            </div>
        )
    }
}
const containerStyle = {
        width: '800px', 
        height: '500px', 
        overflow: 'auto', 
        border: '1px solid'
    }
    
    const canvasStyle = {
        border: '1px solid #000000'
    }

UPDATE I was able to fix this issue by making a custom build on fabric since I really only needed the fabric.rect. NOTE: If you do this, make sure you also check the Animation box lest you waste half our day like me