java - 找不到除 main 之外的 hadoop 类

Question

我在虚拟机上运行 hadoop Cloudera CHD4.5。在 Manhout in Action 第 9-10 章中，我尝试创建一个自定义的 lucene 分析器。我的分析器在它自己的类中定义。当我创建 JAR 时，类就存在了。当我通过命令行执行时，我不断收到 java.lang.IllegalStateException: java.lang.ClassNotFoundException: my.org.MyAnalyzer。

HADOOP_CLASSPATH 用于将缺失的类传递给客户端，但我的问题是 myanalyzer 类似乎没有传递给 map/reduce JVM。我没有创建自己的 map/reduce 类，也没有从我的主类运行作业。我正在使用现有的象夫类来实现我的目标。在我的代码中，如果我使用 WhitespaceAnalyzer 代替我的分析器，我没有问题。

这是我的分析器类

public final class MyAnalyzer extends Analyzer {

public final StandardAnalyzer stdAnalyzer = new StandardAnalyzer(Version.LUCENE_36);

@Override
 public TokenStream tokenStream(String fieldName, Reader reader) {
    return stdAnalyzer.tokenStream(fieldName, reader);

  }
}

有什么想法吗？

我试过 maven，我得到了相同的结果。但是，我可能在 pom.xml 文件中没有正确的依赖项。

-libjars 不工作，因为我没有在我的代码中使用 GenericOptionsParser 方法。

Answer 1

它现在正在工作。我在我的 pom.xml 中使用了 maven-assembly-plugin 并创建了相应的 job.xml 组装作业。

pom.xml

<project    xmlns="http://maven.apache.org/POM/4.0.0" 
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0       http://maven.apache.org/xsd/maven-4.0.0.xsd">

<modelVersion>4.0.0</modelVersion>

<groupId>my.org</groupId>
<artifactId>maven_NewKmeans</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>

<name>maven_NewKmeans</name>
<url>http://maven.apache.org</url>

<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <hadoop.version>2.0.0-mr1-cdh4.0.1</hadoop.version>

    <mahout.version>0.7</mahout.version>
    <mahout.groupid>org.apache.mahout</mahout.groupid>
    </properties>

<repositories>
    <repository>
        <id>cloudera-releases</id>
        <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
        <releases>
                <enabled>true</enabled>
        </releases>
        <snapshots>
                <enabled>false</enabled>
        </snapshots>
    </repository>
</repositories>
<build>

<plugins> 

    <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>2.3.2</version>
        <configuration>
            <source>1.6</source>
            <target>1.6</target>
        </configuration>
    </plugin>

    <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-resources-plugin</artifactId>
        <version>2.4.3</version>
        <configuration>
            <encoding>UTF-8</encoding>
        </configuration>
    </plugin>

    <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-assembly-plugin</artifactId>
        <executions>
            <execution>
                <id>job</id>
                <phase>package</phase>
                <goals>
                    <goal>single</goal>
                </goals>
                <configuration>
                    <descriptors>
                             <descriptor>src/main/assembly/job.xml</descriptor>
                    </descriptors>
                </configuration>
            </execution>
            <execution>
                <id>my-jar-with-dependencies</id>
                <phase>package</phase>
                <goals>
                    <goal>single</goal>
                </goals>
                <configuration>
                    <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
            </execution>
        </executions>
    </plugin>

</plugins>

<pluginManagement> 
</pluginManagement>

 </build>

<dependencies>

<dependency>
  <groupId>junit</groupId>
  <artifactId>junit</artifactId>
  <version>3.8.1</version>
  <scope>test</scope>
</dependency>

<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-client</artifactId>
  <version>${hadoop.version}</version>
</dependency>

<dependency>
  <groupId>${mahout.groupid}</groupId>
  <artifactId>mahout-core</artifactId>
  <version>${mahout.version}</version>
</dependency>

</dependencies>

</project>

装配作业.xml

<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-   plugin/assembly/1.1.0
http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>job</id>
<formats>
<format>jar</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
  <unpack>true</unpack>
  <scope>runtime</scope>
  <outputDirectory>/</outputDirectory>
  <excludes>
    <exclude>org.apache.hadoop:hadoop-core</exclude>
  </excludes>
</dependencySet>
</dependencySets>
<fileSets>
<fileSet>
  <directory>${basedir}/target/classes</directory>
  <outputDirectory>/</outputDirectory>
  <excludes>
    <exclude>*.jar</exclude>
  </excludes>
</fileSet>
<fileSet>
  <directory>${basedir}/target/classes</directory>
  <outputDirectory>/</outputDirectory>
  <includes>
    <include>driver.classes.default.props</include>
  </includes>
</fileSet>