MySQL data insertion efficiency comparison

MySQL data insertion efficiency comparison

When inserting data, I found that I had never considered database performance when I was working on office systems before. Because the amount of data involved was small, the time and efficiency were not obvious. But when the amount of data is very large, so large that 10,000 insertions are required per second, you have to consider your SQL statements. When inserting 100 data, the data insertion methods that can be thought of are:

1: For loop 100 times, inserting data again and again. Connect once and insert 100 times, which is the most time-consuming and consumes the most IO and connection;

2: Combine 100 data insert statements into one SQL statement, then connect once and insert the data. This method is more time-consuming than the first one.

3: Use transactions, insert 100 times, and commit the last transaction; this method is faster than the second method.

4: Use the insert statement itself to insert multiple data;

When the above methods are used on a small amount of data, there is almost no difference and we can't feel it at all. However, when the amount of data is slightly larger, for example, 10,000 pieces of data at a time. The speed and efficiency of insertion come out;

This is the mysql instance class; this instance provides mysql connection and database related operations

public class MySqlInstance
  {
    //Connection string private static string mySqlConnectionStr = "Server = localhost; Database = test; Uid = root; Pwd = password.1;";
    private static MySqlConnection _mysqlConnect;
    private static MySqlConnection mysqlConnect
    {
      get
      {
        if (null == _mysqlConnect)
        {
          _mysqlConnect = new MySqlConnection(mySqlConnectionStr);
        }
        return _mysqlConnect;
      }
    }
    private static MySqlCommand _mysqlCommand;
    private static MySqlCommand mysqlCommand
    {
      get
      {
        if (null == _mysqlCommand)
        {
          _mysqlCommand = mysqlConnect.CreateCommand();
        }
        return _mysqlCommand;
      }
    }
    //Open the connection public static void OpenConnect()
    {
      mysqlConnect.Open();
    }
    //Close the connection public static void CloseConnect()
    {
      mysqlConnect.Close();
    }
    public static MySqlConnection Connection
    {
      get
      {
        return mysqlConnect;
      }
    }
    //Insert data in an anti-injection way //Use transaction 10000 to insert, and commit the transaction once at the end public static int InsertData(string Command, List<MySqlParameter> Params)
    {
      //Program time monitoring Stopwatch sw = new Stopwatch();
      //Program timing startssw.Start();
      OpenConnect();
      //Transaction start MySqlTransaction trans = mysqlConnect.BeginTransaction();
      mysqlCommand.CommandText = Command;
      mysqlCommand.Parameters.AddRange(Params.ToArray());
      int count = 0;
      for (int i = 0; i < 10000; i++)
      {
        if (mysqlCommand.ExecuteNonQuery() > 0)
          count++;
      }
      //Transaction commit trans.Commit();
      CloseConnect();
      mysqlCommand.Parameters.Clear();
      //Timer stops sw.Stop();
      TimeSpan ts2 = sw.Elapsed;
      Console.WriteLine(ts2.TotalMilliseconds);
      return count;
    }
    //The query results in MySqlDataReader. You cannot close the connection if you want to use it. public static MySqlDataReader SelectData(string sql)
    {
      Stopwatch sw = new Stopwatch();
      sw.Start();
      // OpenConnect();
      MySqlCommand newcommond = new MySqlCommand(sql, mysqlConnect);
      MySqlDataReader data = newcommond.ExecuteReader();
      // CloseConnect();
      sw.Stop();
      TimeSpan ts2 = sw.Elapsed;
      Console.WriteLine(ts2.TotalMilliseconds);
      return data;
    }
    /// <summary>
    /// The query results in a data set/// </summary>
    /// <param name="sql"></param>
    /// <returns></returns>
    public static DataSet SelectDataSet(string sql)
    {
      MySqlCommand newcommond = new MySqlCommand(sql, mysqlConnect);
      MySqlDataAdapter adapter = new MySqlDataAdapter();
      adapter.SelectCommand = newcommond;
      DataSet ds = new DataSet();
      adapter.Fill(ds);
      return ds;
    }
    //Unsafe insert with injection public static int InsertDataSql(string sql)
    {
      // OpenConnect();
      mysqlCommand.CommandText = sql;
      int count = mysqlCommand.ExecuteNonQuery();
      // CloseConnect();
      return count;
    }
    //Safely insert parameters using @
    //Insert 10,000 times without using transactions public static int InsertDataNoTran(string Command, List<MySqlParameter> Params)
    {
      Stopwatch sw = new Stopwatch();
      sw.Start();
      OpenConnect();
      mysqlCommand.CommandText = Command;
      mysqlCommand.Parameters.AddRange(Params.ToArray());
      int count = 0;
      for (int i = 0; i < 10000; i++)
      {
        if (mysqlCommand.ExecuteNonQuery() > 0)
          count++;
      }
      CloseConnect();
      mysqlCommand.Parameters.Clear();
      sw.Stop();
      TimeSpan ts2 = sw.Elapsed;
      Console.WriteLine(ts2.TotalMilliseconds);
      return count;
    }
    //Spell 10,000 insert statements at once and submit them at once public static void test4()
    {
      Stopwatch sw = new Stopwatch();
      sw.Start();
      MySqlInstance.OpenConnect();
      MySqlTransaction tran = MySqlInstance.Connection.BeginTransaction();
      string command = string.Empty;
      for (int i = 0; i < 10000; i++)
      {
        string temp = string.Format("insert into test.testtable(pname,pwd) value ('{0}','{1}'); \r\n", "name" + i, "password." + i);
        command += temp;
      }
      MySqlInstance.InsertDataSql(command);
      tran.Commit();
      MySqlInstance.CloseConnect();
      sw.Stop();
      TimeSpan ts2 = sw.Elapsed;
      Console.WriteLine(ts2.TotalMilliseconds);
    }
 }

Finally, a console program is created to test the three methods, using transaction submission, not using transactions, and concatenating 10,000 inserts to form a transaction, and print out the time consumed. The result is as shown below:

It can be seen that: 10,000 inserts took only 4.7 seconds to commit using transactions, while it took 311 seconds without transactions, and it took 7.3 seconds to assemble 10,000 insert statements. The time taken here is 7.3 seconds. In theory, the execution of database SQL should be similar to that of using transactions. The time taken here is mainly used for string concatenation, which takes more time on the client side.

Paste the test program code:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MySql.Data;
using MySql.Web;
using MySql.Data.MySqlClient;
using System.Diagnostics;
using System.Data;
namespace mysqlDEMO01
{
  Class Program
  {
    static void Main(string[] args)
    {      
      testInsert();
      Console.ReadLine();
    }
    //Use @ to safely insert the anti-injection parameter.
    public static void testInsert()
    {
      List<MySqlParameter> lmp = new List<MySqlParameter>();
      lmp.Add(new MySqlParameter("@pname", "hello2"));
      lmp.Add(new MySqlParameter("@pwd", "1232"));
      string command = "insert into test.testtable(pname,pwd) value(@pname,@pwd); ";
      MySqlInstance.InsertData(command, lmp);
      List<MySqlParameter> lmp2 = new List<MySqlParameter>();
      lmp2.Add(new MySqlParameter("@pname", "hello2"));
      lmp2.Add(new MySqlParameter("@pwd", "1232"));
      MySqlInstance.InsertDataNoTran(command, lmp2);
      test4();
    }
   }
}

Summarize

The above is the full content of this article. I hope that the content of this article will have certain reference learning value for your study or work. Thank you for your support of 123WORDPRESS.COM. If you want to learn more about this, please check out the following links

You may also be interested in:
  • Getting Started with MySQL (IV) Inserting, Updating, and Deleting Data from a Table
  • MySQL data insertion optimization method concurrent_insert
  • Mysql updates certain fields of another table based on data from one table (sql statement)
  • Why is the disk space still occupied after deleting table data in MySQL?
  • Detailed explanation of the idea of ​​MySQL trigger detecting a statement in real time for backup and deletion
  • mysql data insert, update and delete details

<<:  How to install JDK 13 in Linux environment using compressed package

>>:  JavaScript+html to implement front-end page sliding verification (2)

Recommend

JavaScript implements the generation of 4-digit random verification code

This article example shares the specific code for...

Solution to the problem that Navicat cannot remotely connect to MySql server

The solution to the problem that Navicat cannot r...

Summary of learning HTML tags and basic elements

1. Elements and tags in HTML <br />An eleme...

Common functions of MySQL basics

Table of contents 1. Common function classificati...

HTML form tag tutorial (1):

Forms are a major external form for implementing ...

Summary of basic knowledge and operations of MySQL database

This article uses examples to explain the basic k...

Create a movable stack widget function using flutter

This post focuses on a super secret Flutter proje...

MySQL foreign key setting method example

1. Foreign key setting method 1. In MySQL, in ord...

Solving problems encountered when importing and exporting Mysql

background Since I converted all my tasks to Dock...

Detailed example of locating and optimizing slow query sql in MySQL

Table of contents 1. How to locate and optimize s...

CocosCreator implements skill cooling effect

CocosCreator realizes skill CD effect There are s...

Detailed explanation of Vue custom instructions

Table of contents Vue custom directive Custom dir...

How to add vector icons to web font files in web page production

As we all know, there are two types of images in c...

Detailed explanation of Vue life cycle functions

Table of contents Lifecycle Functions Common life...

VSCode+CMake+Clang+GCC environment construction tutorial under win10

I plan to use C/C++ to implement basic data struc...