1

I am trying to fill my SQL Server database with 3 different .txt files I accomplished this using regular for but it takes a long time. Then since the order does not matter (in my database) I tried to use Parallel.ForEach but I get this error error

Here is my code

using System;
using System.Windows.Forms;
using System.Data.SqlClient;
using System.Data;

namespace LogwithDataBase
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        SqlConnection con = new SqlConnection(@"Data Source=.\SQLEXPRESS;Initial Catalog=LogsUploaded;Integrated Security=SSPI");

        void filldata()
        {
            SqlDataAdapter da = new SqlDataAdapter("Select *From LogsTable", con);
            DataSet ds = new DataSet();
            con.Open();

            da.Fill(ds, "LogsTable");

            dataGridView1.DataSource = ds.Tables["LogsTable"];
            dataGridView1.Columns[1].DefaultCellStyle.Format = "yyyy-MM-dd HH:mm:ss.fff";
            con.Close();
        }

        private void datab_connect_Click(object sender, EventArgs e)
        {
            con.Open();
            IEnumerable<string> lines1 = File.ReadLines(openFileDialog1.FileName); //HWLog

            Parallel.ForEach(lines1, line1 =>
            {
                string timeStmp = line1.Split('\t')[0];
                string atmIdd = line1.Split('\t')[1];
                string fileTyp = "HWLog";
                string evenTex = line1.Split('\t')[2];
                string query = "INSERT INTO LogsTable (timestampof_log,atmID,fileType,eventText) values (@timeStmp, @atmIdd, @fileTyp, @evenTex)";
                SqlCommand cmd = new SqlCommand(query, con);
                cmd.Parameters.AddWithValue("@timeStmp", DateTime.Parse(timeStmp));
                cmd.Parameters.AddWithValue("@atmIdd", atmIdd);
                cmd.Parameters.AddWithValue("@fileTyp", fileTyp);
                cmd.Parameters.AddWithValue("@evenTex", evenTex);
                cmd.ExecuteNonQuery();
            });

            IEnumerable<string> lines2 = File.ReadLines(openFileDialog2.FileName); //SWLog
            Parallel.ForEach(lines2, line2 =>
            {
                string timeStmp = line2.Split('\t')[0];
                string atmIdd = line2.Split('\t')[1];
                string fileTyp = "SWLog";
                string evenTex = line2.Split('\t')[2];
                string query = "INSERT INTO LogsTable (timestampof_log,atmID,fileType,eventText) values (@timeStmp, @atmIdd, @fileTyp, @evenTex)";
                SqlCommand cmd = new SqlCommand(query, con);
                cmd.Parameters.AddWithValue("@timeStmp", DateTime.Parse(timeStmp));
                cmd.Parameters.AddWithValue("@atmIdd", atmIdd);
                cmd.Parameters.AddWithValue("@fileTyp", fileTyp);
                cmd.Parameters.AddWithValue("@evenTex", evenTex);
                cmd.ExecuteNonQuery();
            });

            IEnumerable<string> lines3 = File.ReadLines(openFileDialog3.FileName); //JournalLog
            Parallel.ForEach(lines3, line3 =>
            {
                string timeStmp = line3.Split('\t')[0];
                string atmIdd = line3.Split('\t')[1];
                string fileTyp = "JournalLog";
                string evenTex = line3.Split('\t')[2];
                string query = "INSERT INTO LogsTable (timestampof_log, atmID, fileType, eventText) VALUES (@timeStmp, @atmIdd, @fileTyp, @evenTex)";

                SqlCommand cmd = new SqlCommand(query, con);

                cmd.Parameters.AddWithValue("@timeStmp", DateTime.Parse(timeStmp));
                cmd.Parameters.AddWithValue("@atmIdd", atmIdd);
                cmd.Parameters.AddWithValue("@fileTyp", fileTyp);
                cmd.Parameters.AddWithValue("@evenTex", evenTex);

                cmd.ExecuteNonQuery();
            });

            string querycb = "SELECT DISTINCT fileType FROM LogsTable"; //COMBOBOXA DATA ÇEKMEK
            SqlCommand cmdcb = new SqlCommand(querycb, con); //COMBOBOXA DATA ÇEKMEK
            SqlDataReader drcb = cmdcb.ExecuteReader(); //COMBOBOXA DATA ÇEKMEK
            List<string> list = new List<string>(); //COMBOBOXA DATA ÇEKMEK
            list.Add(""); //COMBOBOXA DATA ÇEKMEK

            while (drcb.Read())
            {
                list.Add(drcb["fileType"].ToString());
            }

            drcb.Close();

            comboBox1.DataSource = list; //COMBOBOXA DATA ÇEKMEK
            con.Close();

            filldata();
        }
    }
}    

I couldn't solve this problem and I don't want to go back to regular for

marc_s
  • 732,580
  • 175
  • 1,330
  • 1,459
U.Deniz A.
  • 13
  • 6
  • You do understand what "IO bound" means, right? – Dai Aug 11 '22 at 05:22
  • 3
    You should not be calling `ExecuteNonQuery` in a loop to begine with, especially creating a new command object every time. You ought to be populating a `DataTable` and then saving it in a single batch using a `SqlDataAdapter` or even `SqlBulkCopy`. – John Aug 11 '22 at 05:24
  • I have an opinion, is it because execute query should be completed before paralleling lines ? If its like this how do we make this parallel or how can I do this faster ? – U.Deniz A. Aug 11 '22 at 05:25

2 Answers2

2

In terms of performance i would suggest to use sqlbulkcopy Just fill a DataTable with all the lines and insert all at once, its much faster than make a single query for each line in the file thant could be really really long.

    public static void BulkInsert(IEnumerable<string> lines)
    {
        DataTable dt = new DataTable();
        dt.TableName = "LogsTable";
        
        dt.Columns.Add("timestampof_log");
        dt.Columns.Add("atmID");
        dt.Columns.Add("fileType");
        dt.Columns.Add("eventText");

        foreach (var line in lines)
        {
            string timeStmp = line.Split('\t')[0];
            string atmIdd = line.Split('\t')[1];
            string fileTyp = "JournalLog";
            string evenTex = line.Split('\t')[2];

            dt.Rows.Add(timeStmp, atmIdd, fileTyp, evenTex);

        }

        BulkInsert(dt);
    }

    public static void BulkInsert(DataTable table)
    {
        using (var bulkInsert = new SqlBulkCopy(ConnectionString))
        {
            bulkInsert.DestinationTableName = table.TableName;
            bulkInsert.WriteToServer(table);
        }
    }
Hytac
  • 147
  • 1
  • 7
  • 2
    The only caveat which you should point out in your answer, is that `SqlBulkCopy` does not cause triggers to fire. If the table has triggers that need to be fired, then the solution would be a User Defined Table Type – Jonathan Willcock Aug 11 '22 at 05:33
  • @JonathanWillcock what you mean by triggers that need to fired? Is it like my filldata() I use it in every button click but I use it at the end – U.Deniz A. Aug 11 '22 at 06:02
  • Have a look [trigger](https://learn.microsoft.com/en-us/sql/t-sql/statements/create-trigger-transact-sql?view=sql-server-ver16) – Hytac Aug 11 '22 at 06:08
  • @Hytac I believe this method copies data from another datatable/dataset but I have an .txt file its not a data set I have a log file – U.Deniz A. Aug 11 '22 at 06:15
  • @U.DenizA. I have edited my answer, check it now. – Hytac Aug 11 '22 at 06:52
  • @Hytac isn't it same process if I use for each loop at the beginning ? – U.Deniz A. Aug 11 '22 at 07:46
  • @U.DenizA. What do you mean with "at the beginning"? – Hytac Aug 11 '22 at 07:50
  • @Hytac I will have three different .txt files which have different filetypes so I think there should be three foreach and isn't the using foreach will be the same as mine ? – U.Deniz A. Aug 11 '22 at 07:55
  • @U.DenizA. I provided this as an example to show my point. If you have 3 files then you have to iterate 3 times. The problem with your code is that you are executing one query per iteration and that takes a really long time. In my example i am commiting all the rows at once wich is much faster. – Hytac Aug 11 '22 at 08:21
1

Simplest way - open sql connection in each parallel thread.

Parallel.Foreach SQL querying sometimes results in Connection more detailed here

Yuri Myakotin
  • 61
  • 1
  • 3
  • 4
    Databasses are IO-bound, not CPU-bound - using additional threads won't be faster - and in-fact will likely slow things down. – Dai Aug 11 '22 at 05:28
  • Depends of size and and database implementation. Some times all needed data already cached in memory. – Yuri Myakotin Aug 11 '22 at 05:46
  • When I do that it gives this error " System.InvalidCastException: 'Unable to cast object of type 'System.Data.ProviderBase.DbConnectionClosedConnecting' to type 'System.Data.SqlClient.SqlInternalConnectionTds'.' " Original code is ' ` Parallel.ForEach(lines1, line1 => { con.Open(); same as before con.Close(); });'` – U.Deniz A. Aug 11 '22 at 06:26
  • you have to **create** and open connection in each thread, not just open. – Yuri Myakotin Aug 11 '22 at 06:47
  • @YuriMyakotin now it says timeout period expired maximum pool connection – U.Deniz A. Aug 11 '22 at 07:59
  • Something wrong with connections pool settings or with closing connections. https://stackoverflow.com/questions/670774/how-can-i-solve-a-connection-pool-problem-between-asp-net-and-sql-server?rq=1 – Yuri Myakotin Aug 11 '22 at 10:25