nio实现文件读取写入数据库或文件

1.nio实现读取大文件，之后分批读取写入数据库

2.nio实现读取大文件，之后分批写入指定文件

[java] view plain copy

print?

1. package com.ally;  
2.   
3. import java.io.File;  
4. import java.io.RandomAccessFile;  
5. import java.nio.ByteBuffer;  
6. import java.nio.channels.FileChannel;  
7. import java.sql.*;  
8.   
9. /** 
10.  * Created by admin on 2016/6/28. 
11.  * 1.nio分批读取sql文件并执行插入数据库 
12.  * 2.读取一个文件写入另外文件 
13.  */  
14. public class TestNio {  
15.     public static void main(String args[]) throws Exception {  
16.         System.err.println("begin");  
17.         long start = System.currentTimeMillis();  
18.         int _5M = 1024 * 1024 * 5;  
19.         File fin = new File("D:\\drug_general_info.sql");  
20.         FileChannel fcin = new RandomAccessFile(fin, "r").getChannel();  
21.         ByteBuffer rBuffer = ByteBuffer.allocate(_5M);  
22.         //将文件读取执行到数据库  
23.         readFileByLine(_5M, fcin, rBuffer);  
24.   
25.         //将文件读取并写入另外文件  
26.         File fout = new File("D:\\mm.sql");  
27.         FileChannel fcout = new RandomAccessFile(fout, "rws").getChannel();  
28.         ByteBuffer wBuffer = ByteBuffer.allocateDirect(_5M);  
29.         saveOtherFile( _5M,  fcin, rBuffer,  fcout,  wBuffer);  
30.         System.err.print((System.currentTimeMillis() - start) / 1000);  
31.     }  
32.   
33.     /** 
34.      * 将一个文件内容写入另外一个 
35.      * @param bufSize 
36.      * @param fcin 
37.      * @param rBuffer 
38.      * @param fcout 
39.      * @param wBuffer 
40.      */  
41.     public static void saveOtherFile(int bufSize, FileChannel fcin,  
42.                                      ByteBuffer rBuffer, FileChannel fcout, ByteBuffer wBuffer){  
43.             Statement pst = null;  
44.             String enterStr = "\n";  
45.             try {  
46.                 byte[] bs = new byte[bufSize];  
47.                 StringBuilder strBuf = new StringBuilder("");  
48.                 String tempString = null;  
49.                 while (fcin.read(rBuffer) != -1) {  
50.                     int rSize = rBuffer.position();  
51.                     rBuffer.rewind();  
52.                     rBuffer.get(bs);  
53.                     rBuffer.clear();  
54.                     tempString = new String(bs, 0, rSize);  
55.                     int fromIndex = 0;  
56.                     int endIndex = 0;  
57.                     int i = 0;  
58.                     while ((endIndex = tempString.indexOf(enterStr, fromIndex)) != -1) {  
59.                         String line = tempString.substring(fromIndex, endIndex);  
60.                         line = strBuf.toString() + line;  
61.                         writeFileByLine(fcout, wBuffer, line);  
62.                         strBuf.delete(0, strBuf.length());  
63.                         fromIndex = endIndex + 1;  
64.   
65.                     }  
66.                 }  
67.             } catch (Exception e) {  
68.                 e.printStackTrace();  
69.             }  
70.   
71.     }  
72.   
73.     /** 
74.      * 读文件写入数据库 
75.      * @param bufSize 
76.      * @param fcin 
77.      * @param rBuffer 
78.      */  
79.     public static void readFileByLine(int bufSize, FileChannel fcin,  
80.                                       ByteBuffer rBuffer) {  
81.         Connection conn = null;  
82.         Statement pst = null;  
83.         String enterStr = "\n";  
84.         try {  
85.             byte[] bs = new byte[bufSize];  
86.             StringBuilder strBuf = new StringBuilder("");  
87.             String tempString = null;  
88.             Class.forName("com.mysql.jdbc.Driver");  
89.             conn = DriverManager.getConnection(  
90.                     "jdbc:mysql://localhost:3306/doctor?useUnicode=true&characterEncoding=utf-8", "root", "root");  
91.             pst = conn.createStatement();  
92.             while (fcin.read(rBuffer) != -1) {  
93.   
94.                 int rSize = rBuffer.position();  
95.                 rBuffer.rewind();  
96.                 rBuffer.get(bs);  
97.                 rBuffer.clear();  
98.                 tempString = new String(bs, 0, rSize);  
99.                 int fromIndex = 0;  
100.                 int endIndex = 0;  
101.                 int i = 0;  
102.                 while ((endIndex = tempString.indexOf(enterStr, fromIndex)) != -1) {  
103.                     String line = tempString.substring(fromIndex, endIndex);  
104.                     line = strBuf.toString() + line;  
105.                     strBuf.delete(0, strBuf.length());  
106.                     fromIndex = endIndex + 1;  
107.                     pst.addBatch(line);  
108.                   /*  System.out.println("-----------------------"); 
109.                     System.out.println(line); 
110.                     System.out.println("-----------------------");*/  
111.                     if (i % 100 == 0) {  
112.                         System.out.println("执行了：" + i);  
113.                         if (i == 2700) {  
114.                             System.out.println("导了：" + i);  
115.                         }  
116.                         int[] flag = pst.executeBatch();  
117.                         System.out.println("结果：" + flag[0]);  
118.                     }  
119.                     i += 1;  
120.                 }  
121.                 // 执行批量更新  
122.                 pst.executeBatch();  
123.   
124.                 if (rSize > tempString.length()) {  
125.                     strBuf.append(tempString.substring(fromIndex,  
126.                             tempString.length()));  
127.                 } else {  
128.                     strBuf.append(tempString.substring(fromIndex, rSize));  
129.                 }  
130.                 //  System.out.println(strBuf.toString());  
131.             }  
132.   
133.         } catch (Exception e) {  
134.             e.printStackTrace();  
135.         } finally {  
136.             try {  
137.                 if (pst != null) {  
138.                     pst.close();  
139.                 }  
140.                 if (conn != null) {  
141.                     conn.close();  
142.                 }  
143.             } catch (SQLException e) {  
144.                 e.printStackTrace();  
145.             }  
146.         }  
147.     }  
148.   
149.     public static void writeFileByLine(FileChannel fcout, ByteBuffer wBuffer,  
150.                                        String line) {  
151.         try {  
152.             fcout.write(wBuffer.wrap(line.getBytes()), fcout.size());  
153.         } catch (Exception e) {  
154.             e.printStackTrace();  
155.         }  
156.     }  
157. }  

package com.ally;
import java.io.File;

import java.io.RandomAccessFile;

import java.nio.ByteBuffer;

import java.nio.channels.FileChannel;

import java.sql.*;
/**

• Created by admin on 2016/6/28.

• 1.nio分批读取sql文件并执行插入数据库

• 2.读取一个文件写入另外文件
  */
  public class TestNio {
  public static void main(String args[]) throws Exception {
  System.err.println("begin");
  long start = System.currentTimeMillis();
  int _5M = 1024 * 1024 * 5;
  File fin = new File("D:\drug_general_info.sql");
  FileChannel fcin = new RandomAccessFile(fin, "r").getChannel();
  ByteBuffer rBuffer = ByteBuffer.allocate(_5M);
  //将文件读取执行到数据库
  readFileByLine(_5M, fcin, rBuffer);

   //将文件读取并写入另外文件
   File fout = new File("D:\\mm.sql");
   FileChannel fcout = new RandomAccessFile(fout, "rws").getChannel();
   ByteBuffer wBuffer = ByteBuffer.allocateDirect(_5M);
   saveOtherFile( _5M,  fcin, rBuffer,  fcout,  wBuffer);
   System.err.print((System.currentTimeMillis() - start) / 1000);
  

  }

  /**

  • 将一个文件内容写入另外一个

  • @param bufSize

  • @param fcin

  • @param rBuffer

  • @param fcout

  • @param wBuffer
    */
    public static void saveOtherFile(int bufSize, FileChannel fcin,
    ByteBuffer rBuffer, FileChannel fcout, ByteBuffer wBuffer){
    Statement pst = null;
    String enterStr = "\n";
    try {
    byte[] bs = new byte[bufSize];
    StringBuilder strBuf = new StringBuilder("");
    String tempString = null;
    while (fcin.read(rBuffer) != -1) {
    int rSize = rBuffer.position();
    rBuffer.rewind();
    rBuffer.get(bs);
    rBuffer.clear();
    tempString = new String(bs, 0, rSize);
    int fromIndex = 0;
    int endIndex = 0;
    int i = 0;
    while ((endIndex = tempString.indexOf(enterStr, fromIndex)) != -1) {
    String line = tempString.substring(fromIndex, endIndex);
    line = strBuf.toString() + line;
    writeFileByLine(fcout, wBuffer, line);
    strBuf.delete(0, strBuf.length());
    fromIndex = endIndex + 1;

             }
         }
     } catch (Exception e) {
         e.printStackTrace();
     }
    

  }

  /**

  • 读文件写入数据库

  • @param bufSize

  • @param fcin

  • @param rBuffer
    */
    public static void readFileByLine(int bufSize, FileChannel fcin,
    ByteBuffer rBuffer) {
    Connection conn = null;
    Statement pst = null;
    String enterStr = "\n";
    try {
    byte[] bs = new byte[bufSize];
    StringBuilder strBuf = new StringBuilder("");
    String tempString = null;
    Class.forName("com.mysql.jdbc.Driver");
    conn = DriverManager.getConnection(
    "jdbc:mysql://localhost:3306/doctor?useUnicode=true&characterEncoding=utf-8", "root", "root");
    pst = conn.createStatement();
    while (fcin.read(rBuffer) != -1) {

         int rSize = rBuffer.position();
         rBuffer.rewind();
         rBuffer.get(bs);
         rBuffer.clear();
         tempString = new String(bs, 0, rSize);
         int fromIndex = 0;
         int endIndex = 0;
         int i = 0;
         while ((endIndex = tempString.indexOf(enterStr, fromIndex)) != -1) {
             String line = tempString.substring(fromIndex, endIndex);
             line = strBuf.toString() + line;
             strBuf.delete(0, strBuf.length());
             fromIndex = endIndex + 1;
             pst.addBatch(line);
           /*  System.out.println("-----------------------");
             System.out.println(line);
             System.out.println("-----------------------");*/
             if (i % 100 == 0) {
                 System.out.println("执行了：" + i);
                 if (i == 2700) {
                     System.out.println("导了：" + i);
                 }
                 int[] flag = pst.executeBatch();
                 System.out.println("结果：" + flag[0]);
             }
             i += 1;
         }
         // 执行批量更新
         pst.executeBatch();
    
         if (rSize > tempString.length()) {
             strBuf.append(tempString.substring(fromIndex,
                     tempString.length()));
         } else {
             strBuf.append(tempString.substring(fromIndex, rSize));
         }
         //  System.out.println(strBuf.toString());
     }
    

    } catch (Exception e) {
    e.printStackTrace();
    } finally {
    try {
    if (pst != null) {
    pst.close();
    }
    if (conn != null) {
    conn.close();
    }
    } catch (SQLException e) {
    e.printStackTrace();
    }
    }
    }

  public static void writeFileByLine(FileChannel fcout, ByteBuffer wBuffer,
  String line) {
  try {
  fcout.write(wBuffer.wrap(line.getBytes()), fcout.size());
  } catch (Exception e) {
  e.printStackTrace();
  }
  }
  }