java - 具有限制行数的结果集 jdbc 驱动程序

Question

我正在尝试创建一个批处理程序。问题是表可能有 200 万行，这可能会产生内存问题。在我的工作中，我使用 DB2，在家里测试时我使用 mysql。

我已经用 eclipse 进行了调试，并且我发现结果集包含所有行。

我在网页中搜索过，但没有找到解决方案。比如我的测试代码是:

    package misCodigos;

    import java.io.FileWriter;
    import java.io.IOException;
    import java.sql.Connection;
    import java.sql.DriverManager;
    import java.sql.PreparedStatement;
    import java.sql.ResultSet;
    import java.sql.SQLException;
    import java.sql.Statement;

    public class testLimitDB {

        // JDBC driver name and database URL
        static final String JDBC_DRIVER = "com.mysql.jdbc.Driver";  
        static final String DB_URL = "jdbc:mysql://localhost/testing";

        //  Database credentials
        static final String USER = "root";
        static final String PASS = "8182";


        private static FileWriter fw=null;

        public static void main(String[] args) {
            // Clase que pretende mostrar cómo insertar filas en java
            Connection conn = null;
            Statement stmt = null;


            try{

                //STEP 2: Register JDBC driver
                Class.forName("com.mysql.jdbc.Driver");

                //STEP 3: Open a connection
                System.out.println("Connecting to database...");
                conn = DriverManager.getConnection(DB_URL,USER,PASS);

                //STEP 4: Execute a query
                //  System.out.println("Creating statement...");
                stmt = conn.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY,
                        java.sql.ResultSet.CONCUR_READ_ONLY);
                String sql = "select * from tablaTest";


                PreparedStatement pstmt;
                pstmt = conn.prepareStatement(sql);

                conn.setAutoCommit(false);
                pstmt.setFetchSize(200);

                ResultSet rs = pstmt.executeQuery(sql);
                System.out.println("Starting to retrieve data. Memory Used: " + getUsedMemorySize());

                //System.out.println(rs.getString(rs.last()));

                /*          rs.last(); 
                int numRows = rs.getRow(); 
                System.out.println("Numero de filas: " + numRows);*/

                fw = new FileWriter("test.out");
                while (rs.next()) {
                    fw.write(rs.getInt(1) + ";" + rs.getString(2) + "\n");
                }
                System.out.println("Done retrieving data => Memory Used: "  + getUsedMemorySize());

                pstmt.close();
                conn.close();
                fw.flush();
                fw.close();

            }catch(Exception e){
                //Handle errors for Class.forName
                e.printStackTrace();
            }finally{
                //finally block used to close resources

                try{

                    if (fw != null){
                        fw.flush();
                        fw.close();      
                    }

                    if(stmt!=null) {
                        stmt.close();
                    }
                }catch(SQLException | IOException se2){
                }// nothing we can do
                try{
                    if(conn!=null) {
                        conn.close();
                    }
                }catch(SQLException se){
                    se.printStackTrace();
                }//end finally try
            }//end try
            System.out.println("Finalizado!");


        }

        public static long getUsedMemorySize() {

            long freeSize = 0L;
            long totalSize = 0L;
            long usedSize = -1L;
            try {
                Runtime info = Runtime.getRuntime();
                freeSize = info.freeMemory();
                totalSize = info.totalMemory();
                usedSize = totalSize - freeSize;
            } catch (Exception e) {
                e.printStackTrace();
            }
            return usedSize;

        }

    }

Could anyone help me?

谢谢

PD:我创建了 db2 的代码。我不确定结果集是否只获取我喜欢的行，因为我在 Eclipse 和结果集变量中使用 Debug模式启动，现在它非常不同，并且看起来是加密的。

import java.io.FileWriter;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

public class testLimitDB {

    // JDBC driver name and database URL
    //static final String JDBC_DRIVER = "com.mysql.jdbc.Driver";  

    static final String JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver";  
    //static final String DB_URL = "jdbc:mysql://localhost/testing";
    static final String DB_URL = "jdbc:db2://localhost:50000/SAMPLE";

    //  Database credentials MYSQL
    //static final String USER = "root";

    static final String USER = "db2admin";
    static final String PASS = "8182";


    private static FileWriter fw=null;

    public static void main(String[] args) {
        // Clase que pretende mostrar cómo insertar filas en java

        Connection conn = null;
        Statement stmt = null;


        try{

            //STEP 2: Register JDBC driver
            Class.forName(JDBC_DRIVER);

            //STEP 3: Open a connection
            System.out.println("Connecting to database...");
            conn = DriverManager.getConnection(DB_URL,USER,PASS);

            //STEP 4: Execute a query
            //  System.out.println("Creating statement...");
            stmt = conn.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY,
                    java.sql.ResultSet.CONCUR_READ_ONLY);
            String sql = "select * from FELIPE.TABLATEST";


            PreparedStatement pstmt;
            pstmt = conn.prepareStatement(sql);

            conn.setAutoCommit(false);
            pstmt.setFetchSize(999);

            ResultSet rs = pstmt.executeQuery();
            System.out.println("Starting to retrieve data. Memory Used: " + getUsedMemorySize());

            //System.out.println(rs.getString(rs.last()));

            /*          rs.last(); 
            int numRows = rs.getRow(); 
            System.out.println("Numero de filas: " + numRows);*/

            fw = new FileWriter("test.out");
            while (rs.next()) {
                fw.write(rs.getInt(1) + ";" + rs.getString(2) + "\n");
            }
            System.out.println("Done retrieving data => Memory Used: "  + getUsedMemorySize());

            pstmt.close();
            conn.close();
            fw.flush();
            fw.close();

        }catch(Exception e){
            //Handle errors for Class.forName
            e.printStackTrace();
        }finally{
            //finally block used to close resources

            try{

                if (fw != null){
                    fw.flush();
                    fw.close();      
                }

                if(stmt!=null) {
                    stmt.close();
                }
            }catch(SQLException | IOException se2){
            }// nothing we can do
            try{
                if(conn!=null) {
                    conn.close();
                }
            }catch(SQLException se){
                se.printStackTrace();
            }//end finally try
        }//end try
        System.out.println("Finalizado!");


    }

    public static long getUsedMemorySize() {

        long freeSize = 0L;
        long totalSize = 0L;
        long usedSize = -1L;
        try {
            Runtime info = Runtime.getRuntime();
            freeSize = info.freeMemory();
            totalSize = info.totalMemory();
            usedSize = totalSize - freeSize;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return usedSize;

    }

}

我再次尝试，我的结论是驱动程序(mysql 和 db2)工作正常。我用jvisualvm进行了测试，我清楚地看到内存随着光标的大小而变化。我会写一份有不同结论的文件，然后将其发布。

Answer 1

我认为您的问题来自于使用没有服务器端游标的 MySQL 进行测试，并且无论您是否设置 FETCH 大小，选择大结果集都不可避免地会导致 OOM。

由于游标，DB2 不会出现同样的问题，并且驱动程序不会抛出 OOM。您需要定义一个非常适合的获取大小。如果您有大量工作，我会使用一个大值。换句话说，当您针对相同的环境进行测试时:)