Why doesn't removeEventListener remove associated function

Question

Hello I am tring to make a svg editor where I can create and drag the rectangles I create. Each time the mouseDown event gets fired I add 2 eventListeners to the svg element (mouseMove and mouseUp) and each time mouseUp gets called I try to remove these events, but because the remove doesn't work the element has multiple mouseMove and mouseUp listeners attached to it. I can't figure out what's wrong. MouseDown does 2 things, first it can draw the rectangles and then if createMode is false it let me drag them

   this.domElement.addEventListener("mousedown", (event) => {
    if (event.button === 0 && this.createMode === true) {
        document.body.style.cursor = 'crosshair'
        const rect = document.createElementNS(this.svgns, 'rect');
        //rect.setAttribute('tabindex', "1");
        let first_mouseX = event.clientX;
        let first_mouseY = event.clientY;

        const drawRect = (event) => {

            let mouseX = event.clientX;
            let mouseY = event.clientY;
            const width = Math.abs(mouseX - first_mouseX);
            const height = Math.abs(mouseY - first_mouseY);

            if (mouseX > first_mouseX) {
                mouseX = first_mouseX;
            }

            if (mouseY > first_mouseY) {
                mouseY = first_mouseY;
            }

            rect.setAttribute('x', mouseX - this.domElement.getBoundingClientRect().left)
            rect.setAttribute('y', mouseY - this.domElement.getBoundingClientRect().top);
            rect.setAttribute('width', width);
            rect.setAttribute('height', height);
            rect.setAttribute('stroke', "black");
            rect.setAttribute('fill', 'white');
            rect.setAttribute('stroke-width', 2);
            this.rect = rect;


            this.domElement.appendChild(rect);
        }

        const endDraw = () => {
            this.domElement.removeEventListener("mousemove", drawRect);
            this.domElement.removeEventListener("mouseup", endDraw);

            if (this.rect !== null) {
                this.rect.addEventListener('click', () => {
                    this.createMode = false;
                    document.body.style.cursor = "auto"
                    event.target.focus()
                })

                this.rect.addEventListener('blur', () => {
                    this.stergeClick();

                })

                this.rect.addEventListener("focus", (event) => {
                    this.stergeClick();
                    this.adaugaClick(event.target)
                })
                this.rect.focus();
                this.rect = null;
            }

        }

        this.domElement.addEventListener("mouseup", endDraw)
        this.domElement.addEventListener("mousemove", drawRect)
    } else if (this.createMode === false) {

        const startDrag = (event) => {
            this.selectedItem = {}
            this.selectedItem.item = event.target
            this.selectedItem.offsetX = event.clientX - this.selectedItem.item.getAttribute('x');
            this.selectedItem.offsetY = event.clientY - this.selectedItem.item.getAttribute('y');
        }

        startDrag(event)

        const drag = (event) => {

            if (Object.keys(this.selectedItem).length > 0 && this.createMode === false && this.selectedItem.item.nodeName !== "svg") {
                this.stergeClick();
                this.adaugaClick(this.selectedItem.item);
                this.selectedItem.item.setAttribute('x', event.clientX - this.selectedItem.offsetX);
                this.selectedItem.item.setAttribute('y', event.clientY - this.selectedItem.offsetY);
            } else {
                return false
            }
        }

        const endDrag = (ev) => {
            if (Object.keys(this.selectedItem).length > 0 && this.selectedItem.item.nodeName !== "svg") {
                this.stergeClick();
                this.adaugaClick(this.selectedItem.item);
                ev.target.removeEventListener('mouseup', drag);
                ev.target.removeEventListener('mousemove', endDrag);
                this.domElement.removeEventListener('mouseup', drag);
                this.domElement.removeEventListener('mousemove', endDrag);
                this.selectedItem = {};
            }
        }

        this.domElement.addEventListener("mousemove", drag);
        this.domElement.addEventListener("mouseup", endDrag);

    }

})

}