在java中使用processbuilder执行sqoop命令

Question

我有一个有效的 sqoop 命令，可以将数据从 oracle 导入到 hive。 当我尝试使用进程构建器 API 运行 sqoop 命令时，它会给出

java.io.IOException: error=2, No such file or directory

从 shell 执行时，相同的命令可以正常工作。请告诉我如何解决此问题。

我还尝试将命令拆分为字符串数组和字符串列表。但是，这两种方法都不起作用。

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;


public class SqoopTest {
    static ProcessBuilder processBuilder = null;
    static Process spawnProcess = null;
    static int exitValue;
    static int pid;
    static List<String> commands;

    public static void main(String[] args) {

      runSqoop();



    }

    /*
     * Executes the shell script with a series of sqoop commands.
     * 
     * 
     * 
     * sqoop import -D mapred.child.java.opts='\-Djava.security.egd=file:/dev/../dev/urandom' 
     * --connect 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=173.37)(PORT=1541))(ADDRESS=(PROTOCOL=TCP)(HOST=173.37)(PORT=1541)))(CONNECT_DATA=(SERVICE_NAME=CAFI2PRD)(SERVER=DEDICATED)))' 
     * --username XXCSS_O --password "c******3" --split-by LINE_ITEM_ID -m 10 
     * --query "select LINE_ITEM_ID,LIST_PRICE,SERVICE_VALUE from XXCSS_KTN_REQ_LINE_DETAIL where \$CONDITIONS AND lid_date > to_date('2016-10-13 03:04:18','MM/dd/yyyy hh24:mi:ss') and lid_date <= to_date('2016-10-13 03:04:18','MM/dd/yyyy hh24:mi:ss')" 
     * --map-column-hive LINE_ITEM_ID=BIGINT,LIST_PRICE=BIGINT,SERVICE_VALUE=BIGINT 
     * --null-string '\\\\N' --null-non-string '\\\\N' --hcatalog-home 
     * /opt/mapr/hive/hive-1.2/hcatalog --hcatalog-table XXCSS_KTN_REQ_LINE_DETAIL --hcatalog-database frameworks_dataingestion
     * 
     */
    public static void runSqoop() {
        commands = new ArrayList<String>();
        commands.add("sqoop import -D mapred.child.java.opts=\'\\-Djava.security.egd=file:/dev/../dev/urandom\' --connect \'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=173.37)(PORT=1541))(ADDRESS=(PROTOCOL=TCP)(HOST=173.37)(PORT=1541)))(CONNECT_DATA=(SERVICE_NAME=CAFI2PRD)(SERVER=DEDICATED)))\' --username XXCSS_O --password \"c*****3\" --split-by LINE_ITEM_ID -m 10 --query \"select LINE_ITEM_ID,LIST_PRICE,SERVICE_VALUE from XXCSS_KTN_REQ_LINE_DETAIL where \\$CONDITIONS AND lid_date > to_date(\'2016-10-13 00:00:00\',\'yyyy-MM-dd hh24:mi:ss\') and lid_date <= to_date(\'2016-10-13 03:04:18\',\'yyyy-MM-dd hh24:mi:ss\')\" --map-column-hive LINE_ITEM_ID=BIGINT,LIST_PRICE=BIGINT,SERVICE_VALUE=BIGINT --null-string '\\\\N' --null-non-string '\\\\N' --hcatalog-home /opt/mapr/hive/hive-1.2/hcatalog --hcatalog-table XXCSS_KTN_REQ_LINE_DETAIL --hcatalog-database frameworks_dataingestion");
        processBuilder = new ProcessBuilder(commands);
        try {
            System.out.println("Executing " + commands.toString());
            spawnProcess = processBuilder.start();
            BufferedReader reader = 
                    new BufferedReader(new InputStreamReader(spawnProcess.getErrorStream()));
            StringBuilder builder = new StringBuilder();
            String line = null;
            while ( (line = reader.readLine()) != null) {
               builder.append(line);
               builder.append(System.getProperty("line.separator"));
            }
            String result = builder.toString();
            System.out.println(result);
            try {
                exitValue = spawnProcess.waitFor();
                pid = getPID(spawnProcess);
                System.out.println("The PID is " + pid);
            } catch (InterruptedException e) {
                System.out.println(e.getMessage());
            } catch (Exception e) {
                System.out.println(e.getMessage());
            }
            System.out.println("Process exited with the status :" + exitValue);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /*
     * Retrieves the process id of the executed command.
     * 
     * @param process the executed process
     * 
     * @return PID
     */

    public static int getPID(Process process) {
        try {
            Class<?> processImplClass = process.getClass();
            Field fpid = processImplClass.getDeclaredField("pid");
            if (!fpid.isAccessible()) {
                fpid.setAccessible(true);
            }
            return fpid.getInt(process);
        } catch (Exception e) {
            System.out.println(e.getMessage());
            return -1;
        }
    }

    /*
     * Retrieves the current java process id
     * 
     * @return java process ID
     */

//  @SuppressWarnings("restriction")
//  public static int getCurrentJavaPID() {
//      int jvmpid = 0;
//      java.lang.management.RuntimeMXBean runtime = java.lang.management.ManagementFactory.getRuntimeMXBean();
//      try {
//          java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm");
//          jvm.setAccessible(true);
//          sun.management.VMManagement mgmt = (sun.management.VMManagement) jvm.get(runtime);
//          java.lang.reflect.Method pid_method = mgmt.getClass().getDeclaredMethod("getProcessId");
//          pid_method.setAccessible(true);
//
//          jvmpid = (Integer) pid_method.invoke(mgmt);
//      } catch (Exception e) {
//          e.getMessage();
//      }
//      return jvmpid;
//  }

}

注意：出于安全考虑，我已将 IP 地址和密码替换为虚拟变量。

尝试将命令作为字符串数组传递。但是，它不能识别 JDBC 连接字符串

public static void runSqoop() {
        String[] commands = {"sqoop","import","-D mapred.child.java.opts=\'\\-Djava.security.egd=file:/dev/../dev/urandom\'","--connect", "\'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=173.37)(PORT=1541))(ADDRESS=(PROTOCOL=TCP)(HOST=173.370)(PORT=1541)))(CONNECT_DATA=(SERVICE_NAME=CAFI2PRD)(SERVER=DEDICATED)))\'","--username","XXCSS_O","--password","\"c***3\"", "--split-by","LINE_ITEM_ID","-m","10","--query","\"select LINE_ITEM_ID,LIST_PRICE,SERVICE_VALUE from XXCSS_KTN_REQ_LINE_DETAIL where \\$CONDITIONS AND lid_date > to_date(\'2016-10-13 00:00:00\',\'yyyy-MM-dd hh24:mi:ss\') and lid_date <= to_date(\'2016-10-13 03:04:18\',\'yyyy-MM-dd hh24:mi:ss\')\"","--null-string","'\\\\N'","--null-non-string","'\\\\N'","--hcatalog-home","/opt/mapr/hive/hive-1.2/hcatalog","--hcatalog-table","XXCSS_KTN_REQ_LINE_DETAIL","--hcatalog-database","frameworks_dataingestion"};
        processBuilder = new ProcessBuilder(commands);
        try {
            System.out.println("Executing " + commands.toString());
            spawnProcess = processBuilder.start();
            BufferedReader reader = 
                    new BufferedReader(new InputStreamReader(spawnProcess.getErrorStream()));
            StringBuilder builder = new StringBuilder();
            String line = null;
            while ( (line = reader.readLine()) != null) {
               builder.append(line);
               builder.append(System.getProperty("line.separator"));
            }
            String result = builder.toString();
            System.out.println(result);
            try {
                exitValue = spawnProcess.waitFor();
                pid = getPID(spawnProcess);
                System.out.println("The PID is " + pid);
            } catch (InterruptedException e) {
                System.out.println(e.getMessage());
            } catch (Exception e) {
                System.out.println(e.getMessage());
            }
            System.out.println("Process exited with the status :" + exitValue);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

堆栈跟踪：

cat: /opt/mapr/zookeeper/zookeeperversion: 没有那个文件或目录 16/10/25 07:41:12 INFO sqoop.Sqoop：运行 Sqoop 版本： 1.4.6-mapr-1609 16/10/25 07:41:12 WARN tool.BaseSqoopTool：在命令行上设置密码是不安全的。考虑使用 -P 反而。 25 年 16 月 10 日 07:41:13 错误工具。BaseSqoopTool：出现错误 创建数据库管理器：java.io.IOException: No manager for connect 细绳： 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=173.37)(PORT=1541))(ADDRESS=(PROTOCOL=TCP )(HOST=173.37)(PORT=1541)))(CONNECT_DATA=(SERVICE_NAME=CAFI2PRD)(SERVER=DEDICATED)))' 在 org.apache.sqoop.ConnFactory.getManager(ConnFactory.java:191) 在 org.apache.sqoop.tool.BaseSqoopTool.init(BaseSqoopTool.java:256) 在 org.apache.sqoop.tool.ImportTool.init(ImportTool.java:89) 在 org.apache.sqoop.tool.ImportTool.run(ImportTool.java:593) 在 org.apache.sqoop.Sqoop.run(Sqoop.java:143) 在 org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) 在 org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:179) 在 org.apache.sqoop.Sqoop.runTool(Sqoop.java:218) 在 org.apache.sqoop.Sqoop.runTool(Sqoop.java:227) 在 org.apache.sqoop.Sqoop.main(Sqoop.java:236)

Answer 1

您必须分别提供命令和参数，如下所示

  Process p = new ProcessBuilder("myCommand", "myArg").start();

或

    ProcessBuilder pb =
       new ProcessBuilder("myCommand", "myArg1", "myArg2");
pb.start();

这是example