C# how to handle 1 million lines without crashing the program?

Question

I have a file with 1 million lines. My program is made to check line by line if it contains one of the words that the user requests to be removed. If the line contains the word, it has to be removed and added into the list. Everytime I press Start, the program freezes and nothing shows up in the listbox. However if I add like 1k it will clean the file and display the new file in list. What's the best way to handle this?

My code :

    using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.IO;

namespace BoringAssComboTool
{
    public partial class Form2 : Form
    {
        List<String> list;
        string myFile = null;
        string[] line = null;
        int linecount = 0;
        public Form2()
        {
            InitializeComponent();
        }

        private void groupBox1_Enter(object sender, EventArgs e)
        {

        }

        private void label1_Click(object sender, EventArgs e)
        {

        }

        private void textBox1_TextChanged(object sender, EventArgs e)
        {

        }
        public void button1_Click(object sender, EventArgs e)
        {

            OpenFileDialog openFileDialog = new OpenFileDialog();
            openFileDialog.RestoreDirectory = true;
            openFileDialog.Multiselect = false;
            openFileDialog.Filter = "Combo List (*.txt)|*.txt";
            openFileDialog.FilterIndex = 1;
            if (openFileDialog.ShowDialog() == DialogResult.OK)
            {
                myFile = openFileDialog.FileName;
                MessageBox.Show("You selected " + openFileDialog.FileName);



                linecount = File.ReadAllLines(myFile).Length;
                label3.Text = "Total loaded : " + linecount;
                line = File.ReadAllLines(myFile);
                //MessageBox.Show(line[0]);
                //MessageBox.Show(line[4]);
                list = File.ReadAllLines(myFile).ToList();
              //  StreamReader sr = new StreamReader(myFile);
            }

        }

        private void button3_Click(object sender, EventArgs e)
        {
            int removedlines = 0;


            string domainSplitted = textBox1.Text;
            string[] splitDomain = domainSplitted.Split(',');
            //MessageBox.Show(splitDomain[2]);
            //MessageBox.Show(splitDomain[3]);
            //MessageBox.Show(line[2]);
            int sizeOfArray = splitDomain.Length;
            // MessageBox.Show("Length is " + sizeOfArray);
            for (int x = 0; x < sizeOfArray - 1; x++)
            {
              //  for (int i = 0; i < linecount - 1; i++) 
              for ( int i = linecount - 1; i>=0; i--)
                {
                    if (line[i].Contains(splitDomain[x]))
                    {

                        list.RemoveAt(i);
                        string[] lines = list.ToArray();
                        removedlines++;
                        label4.Text = "Total Removed = " + removedlines;
                        listBox1.DataSource = list;
                        // MessageBox.Show("there is a match on " + line[i]);
                    }
                }
            }
          // listBox1.DataSource = list;
        }

        private void pictureBox3_Click(object sender, EventArgs e)
        {
            this.WindowState = FormWindowState.Minimized;
        }

        private void pictureBox2_Click(object sender, EventArgs e)
        {
            Application.Exit();
        }



        private void label3_Click(object sender, EventArgs e)
        {

        }

        private void button2_Click(object sender, EventArgs e)
        {
            // System.IO.File.WriteAllLines("Cleaned Combo File.txt", list);
            SaveFileDialog save = new SaveFileDialog();

            save.FileName = "Cleaned Combo File.txt";

            save.Filter = "Text File | *.txt";

            if (save.ShowDialog() == DialogResult.OK)

            {

                StreamWriter writer = new StreamWriter(save.OpenFile());

                for (int i = 0; i < listBox1.Items.Count; i++)

                {

                    writer.WriteLine(listBox1.Items[i].ToString());

                }

                writer.Dispose();

                writer.Close();

                MessageBox.Show("Saved succesfully");

            }
        }
        // MessageBox.Show("==" + line[27]);

        protected override void WndProc(ref Message m)
        {
            base.WndProc(ref m);
            if (m.Msg == WM_NCHITTEST)
                m.Result = (IntPtr)(HT_CAPTION);
        }

        private const int WM_NCHITTEST = 0x84;
        private const int HT_CLIENT = 0x1;
        private const int HT_CAPTION = 0x2;

        private void Form2_Load(object sender, EventArgs e)
        {

        }
    }
    }

Answer 1

You should not really read all the text into memory. Instead change your program to read line-by-line. You can then append each updated/fixed line into a temp file. After the processing is done, if necessary, overwrite the source file with the temp file.

Here is how you can read file line-by-line:

using (StreamReader sr = new StreamReader(path)) 
{
    while (sr.Peek() >= 0) 
    {
        Console.WriteLine(sr.ReadLine());
    }
}

Here is documentation for StreamReader/StreamWriter classes.

In addition to that, I'd suggest you to switch to async APIs. StreamReader has a ReadLineAsync API too, to satisfy that need. This will allow to avoid lengthy delays of the UI thread.