hadoop - HBase放置在第二次插入时失败

Question

我正在使用spring数据+ hbase将一些值写入HBase数据库。
不幸的是，HbaseTemplate似乎在第一次调用后关闭了连接。

我是Spring和HBase / Hadoop的新手，所以我不知道这是Spring / HBase配置问题还是另一个愚蠢问题

测试类别:

package org.springframework.data.hadoop.samples;

import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import org.springframework.data.hadoop.hbase.TableCallback;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration("/hbase-context.xml")
public class WordCountWorkflowTest {

    @Autowired
    private ApplicationContext ctx;

    @Autowired
    private HbaseTemplate hbaseTemplate;

    @Test
    public void testWorkflowNS() throws Exception {
        if (hbaseTemplate == null) {
            throw new NullPointerException("template null!");
        }
        // Write to HBase
        InnerTableCallback itc = new InnerTableCallback("JustaString", 42);
        hbaseTemplate.execute("Wordcount", itc);
        itc = new InnerTableCallback("Anotherstring", 23);
        // Here the HBase insert fails
        hbaseTemplate.execute("Wordcount", itc);
    }

    @Test
    public void testWorkflowNSSucess() throws Exception {
        System.out.println("done");
    }

    /**
     * This is a Inner class providing access to the HBase Table to store the
     * counted words and number of matches.
     * 
     * */
    class InnerTableCallback implements TableCallback<Object> {

        String foundStr;

        int no;

        /**
         * The constructor just saved the given foundStr/no tuple in inner
         * variables.
         * 
         * @param foundstr
         *            string found in the text
         * @param no
         *            number of found matches
         * @return null
         * */
        public InnerTableCallback(String foundStr, int no) {
            this.foundStr = foundStr;
            this.no = no;
        }

        /**
         * This Method puts the given String and number of found matches into
         * the HBase table the column family is "cf1" and the column is
         * "matches". The rowname is the found string.
         * */
        @Override
        public Object doInTable(HTable table) throws Throwable {
            Put p = new Put(Bytes.toBytes(foundStr));
            // Put operation on hbase shell:
            // hbase(main):005:0> put 'testtable', 'myrow-2', 'colfam1:q2',
            // 'value-2'
            // add(byte[] family, byte[] qualifier, byte[] value)
            p.add(Bytes.toBytes("cf1"), Bytes.toBytes("matches"),
                    Bytes.toBytes(new Integer(no).toString()));
            table.put(p);
            return null;
        }
    }
}

hbase-context.xml:

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:p="http://www.springframework.org/schema/p"
    xmlns:hdp="http://www.springframework.org/schema/hadoop"
    xsi:schemaLocation="
        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
            http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">

    <context:property-placeholder location="classpath:batch.properties,classpath:hadoop.properties"
            ignore-resource-not-found="true" ignore-unresolvable="true" />

    <context:component-scan base-package="org.springframework.data.hadoop.samples" />

    <bean class="org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor"/>

    <bean id="hbaseTemplate" class="org.springframework.data.hadoop.hbase.HbaseTemplate" p:configuration-ref="hbaseConfiguration"/>

    <hdp:hbase-configuration>           
    </hdp:hbase-configuration>
</beans>

堆栈跟踪:

org.springframework.data.hadoop.hbase.HbaseSystemException: org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@61bb0cc0 closed; nested exception is java.io.IOException: org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@61bb0cc0 closed
    at org.springframework.data.hadoop.hbase.HbaseUtils.convertHbaseException(HbaseUtils.java:42)
    at org.springframework.data.hadoop.hbase.HbaseTemplate.convertHbaseAccessException(HbaseTemplate.java:111)
    at org.springframework.data.hadoop.hbase.HbaseTemplate.execute(HbaseTemplate.java:82)
    at org.springframework.data.hadoop.samples.WordCountWorkflowTest.testWorkflowNS(WordCountWorkflowTest.java:35)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:601)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
    ....
Caused by: java.io.IOException: org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@61bb0cc0 closed
    at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:822)
    at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:810)
    at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1492)
    at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1377)
    at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:916)
    at org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:772)
    at org.apache.hadoop.hbase.client.HTable.put(HTable.java:747)
    at org.springframework.data.hadoop.samples.WordCountWorkflowTest$InnerTableCallback.doInTable(WordCountWorkflowTest.java:83)
    at org.springframework.data.hadoop.hbase.HbaseTemplate.execute(HbaseTemplate.java:72)

干杯，R

Answer 1

解决了!

我在maven pom.xml中使用了spring-data-hadoop软件包(Milestone)的旧版本和已弃用的版本。我切换到Snapshot存储库，该存储库修复了对HBase表的错误处理。

如果您使用spring-batch:我必须在新版本的* context.xml中将hadoop的<tasklet>定义更改为<job-tasklet>。

来自XML解析器的错误消息是:

The matching wildcard is strict, but no declaration can be found for element 'tasklet'

希望这可以帮助某人:-)