hadoop - 在HDP 2.3中接收 “Exception in thread ” main“java.lang.NoSuchMethodError”异常，在CDH 4.7中正常工作

Question

堆栈跟踪:

**Exception in thread "main" java.lang.NoSuchMethodError: org.apache.hadoop.util.Shell.getGROUPS_COMMAND()[Ljava/lang/String**<br/>
        at org.apache.hadoop.security.UnixUserGroupInformation.getUnixGroups(UnixUserGroupInformation.java:345)
        at org.apache.hadoop.security.UnixUserGroupInformation.login(UnixUserGroupInformation.java:264)
        at org.apache.hadoop.security.UnixUserGroupInformation.login(UnixUserGroupInformation.java:300)
        at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:192)
        at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:170)
        at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:82)

码:

public static void main(String[] args) throws Exception {

    System.out.println("start");

    OutputStream outputStream = null;
    try {

        String c11 = "c11";
        String c12 = "c12";
        String c13 = "c13";

        String c21 = "c21";
        String c22 = "c22";
        String c23 = "c22";

        Configuration configuration = new Configuration();
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());            
        FileSystem hdfs = FileSystem.get(new URI("hdfs://sandbox.hortonworks.com:8020"), configuration);
        outputStream = hdfs.create(
                new Path("hdfs://sandbox.hortonworks.com:8020/user/hue/oozie/workspaces/my2/windowsCompleted.txt"),
                new Progressable() {
                    @Override
                    public void progress() {
                    }
                });
        System.out.println("HELLO");
        byte[] byt = null;
        String input1 = c11 + "|" + c12 + "|" + c13 + "\n";
        String input2 = c21 + "|" + c22 + "|" + c23;
        byt = input1.toString().getBytes();
        outputStream.write(byt);

        byt = input2.toString().getBytes();
        outputStream.write(byt);

        outputStream.flush();
    } catch (Exception e) {
        e.printStackTrace();
    } finally {         
        if (null != outputStream)
            outputStream.close();
    }
    System.out.println("END");

}

POM.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>BBN</groupId>
<artifactId>BBN</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>BBN</name>
<properties>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
</properties>

<dependencies>
    <!-- https://mvnrepository.com/artifact/com.googlecode.json-simple/json-simple -->
    <dependency>
        <groupId>com.googlecode.json-simple</groupId>
        <artifactId>json-simple</artifactId>
        <version>1.1</version>
    </dependency>

    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
        <version>1.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.postgresql</groupId>
        <artifactId>postgresql</artifactId>
        <version>9.2-1003-jdbc4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-jdbc</artifactId>
        <version>0.14.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase-client</artifactId>
        <version>1.1.2</version>
    </dependency>
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
        <version>1.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-hadoop</artifactId>
        <version>1.7.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-core</artifactId>
        <version>0.20.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-avro</artifactId>
        <version>1.7.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-column</artifactId>
        <version>1.7.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-common</artifactId>
        <version>1.7.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-format</artifactId>
        <version>2.3.0-incubating</version>
    </dependency>
    <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-generator</artifactId>
        <version>1.7.0</version>
    </dependency>
    <dependency>
        <groupId>com.sun.jersey</groupId>
        <artifactId>jersey-bundle</artifactId>
        <version>1.8</version>
    </dependency>
    <dependency>
        <groupId>com.google.code.gson</groupId>
        <artifactId>gson</artifactId>
        <version>2.2.4</version>
    </dependency>
    <dependency>
        <groupId>org.json</groupId>
        <artifactId>json</artifactId>
        <version>20140107</version>
    </dependency>
    <dependency>
        <groupId>com.sun.jersey</groupId>
        <artifactId>jersey-server</artifactId>
        <version>1.8</version>
    </dependency>
    <dependency>
        <groupId>com.sun.jersey</groupId>
        <artifactId>jersey-core</artifactId>
        <version>1.8</version>
    </dependency>
    <dependency>  
        <groupId>jdk.tools</groupId>
        <artifactId>jdk.tools</artifactId>
        <!--<version>1.8.0_101</version>-->
        <version>1.7.0_79</version>
        <scope>system</scope>
        <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
    </dependency>
    <!-- Added log4j for logging -->
    <dependency>
        <groupId>log4j</groupId>
        <artifactId>log4j</artifactId>
        <version>1.2.17</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.3.0</version>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <artifactId>maven-assembly-plugin</artifactId>
            <configuration>
                <archive>
                    <manifest>
                        <mainClass>com.bbn.main.WindowsLoadData</mainClass>
                    </manifest>
                </archive>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
            </configuration>
        </plugin>
    </plugins>
</build>

</project>

Answer 1

hadoop.util.Shell类属于pom.xml中缺少的hadoop-common jar。添加此依赖项和hadoop-common所需的依赖项jar。