python-2.7 - 适用于 Python 的 H2O 机器学习平台在构建模型时出现环境错误

Question

我是 h2o 机器学习平台的新手，在尝试构建模型时遇到以下问题。

当我尝试使用不太大的数据集构建 5 GBM 模型时，出现以下错误:

gbm Model Build Progress: [##################################################] 100%

gbm Model Build Progress: [##################################################] 100%

gbm Model Build Progress: [##################################################] 100%

gbm Model Build Progress: [##################################################] 100%

gbm Model Build Progress: [#################                                 ] 34%

EnvironmentErrorTraceback (most recent call last)
<ipython-input-22-e74b34df2f1a> in <module>()
     13     params_model={'x': features_pca_all, 'y': response, 'training_frame': train_holdout_pca_hex, 'validation_frame':              validation_holdout_pca_hex, 'ntrees': ntree, 'max_depth':depth, 'min_rows': min_rows, 'learn_rate': 0.005}
     14 
---> 15     gbm_model=h2o.gbm(**params_model)
     16 
     17     #store model

C:\Anaconda2\lib\site-packages\h2o\h2o.pyc in gbm(x, y, validation_x, validation_y, training_frame, model_id, distribution, tweedie_power, ntrees, max_depth, min_rows, learn_rate, nbins, nbins_cats, validation_frame, balance_classes, max_after_balance_size, seed, build_tree_one_node, nfolds, fold_column, fold_assignment, keep_cross_validation_predictions, score_each_iteration, offset_column, weights_column, do_future, checkpoint)
   1058   parms = {k:v for k,v in locals().items() if k in ["training_frame", "validation_frame", "validation_x", "validation_y", "offset_column", "weights_column", "fold_column"] or v is not None}
   1059   parms["algo"]="gbm"
-> 1060   return h2o_model_builder.supervised(parms)
   1061 
   1062 

C:\Anaconda2\lib\site-packages\h2o\h2o_model_builder.pyc in supervised(kwargs)
     28   algo  = kwargs["algo"]
     29   parms={k:v for k,v in kwargs.items() if (k not in ["x","y","validation_x","validation_y","algo"] and v is not None) or k=="validation_frame"}
---> 30   return supervised_model_build(x,y,vx,vy,algo,offsets,weights,fold_column,parms)
     31 
     32 def unsupervised_model_build(x,validation_x,algo_url,kwargs): return _model_build(x,None,validation_x,None,algo_url,None,None,None,kwargs)

C:\Anaconda2\lib\site-packages\h2o\h2o_model_builder.pyc in supervised_model_build(x, y, vx, vy, algo, offsets, weights, fold_column, kwargs)
     16   if not is_auto_encoder and y is None: raise ValueError("Missing response")
     17   if vx is not None and vy is None:     raise ValueError("Missing response validating a supervised model")
---> 18   return _model_build(x,y,vx,vy,algo,offsets,weights,fold_column,kwargs)
     19 
     20 def supervised(kwargs):

C:\Anaconda2\lib\site-packages\h2o\h2o_model_builder.pyc in _model_build(x, y, vx, vy, algo, offsets, weights, fold_column, kwargs)
     86   do_future = kwargs.pop("do_future") if "do_future" in kwargs else False
     87   future_model = H2OModelFuture(H2OJob(H2OConnection.post_json("ModelBuilders/"+algo, **kwargs), job_type=(algo+" Model Build")), x)
---> 88   return future_model if do_future else _resolve_model(future_model, **kwargs)
     89 
     90 def _resolve_model(future_model, **kwargs):

C:\Anaconda2\lib\site-packages\h2o\h2o_model_builder.pyc in _resolve_model(future_model, **kwargs)
     89 
     90 def _resolve_model(future_model, **kwargs):
---> 91   future_model.poll()
     92   if '_rest_version' in kwargs.keys(): model_json = H2OConnection.get_json("Models/"+future_model.job.dest_key, _rest_version=kwargs['_rest_version'])["models"][0]
     93   else:                                model_json = H2OConnection.get_json("Models/"+future_model.job.dest_key)["models"][0]

C:\Anaconda2\lib\site-packages\h2o\model\model_future.pyc in poll(self)
      8 
      9     def poll(self):
---> 10         self.job.poll()
     11         self.x = None

C:\Anaconda2\lib\site-packages\h2o\job.pyc in poll(self)
     39       time.sleep(sleep)
     40       if sleep < 1.0: sleep += 0.1
---> 41       self._refresh_job_view()
     42       running = self._is_running()
     43     self._update_progress()

C:\Anaconda2\lib\site-packages\h2o\job.pyc in _refresh_job_view(self)
     52 
     53   def _refresh_job_view(self):
---> 54       jobs = H2OConnection.get_json(url_suffix="Jobs/" + self.job_key)
     55       self.job = jobs["jobs"][0] if "jobs" in jobs else jobs["job"][0]
     56       self.status = self.job["status"]

C:\Anaconda2\lib\site-packages\h2o\connection.pyc in get_json(url_suffix, **kwargs)
    410     if __H2OCONN__ is None:
    411       raise ValueError("No h2o connection. Did you run `h2o.init()` ?")
--> 412     return __H2OCONN__._rest_json(url_suffix, "GET", None, **kwargs)
    413 
    414   @staticmethod

C:\Anaconda2\lib\site-packages\h2o\connection.pyc in _rest_json(self, url_suffix, method, file_upload_info, **kwargs)
    419 
    420   def _rest_json(self, url_suffix, method, file_upload_info, **kwargs):
--> 421     raw_txt = self._do_raw_rest(url_suffix, method, file_upload_info, **kwargs)
    422     return self._process_tables(raw_txt.json())
    423 

C:\Anaconda2\lib\site-packages\h2o\connection.pyc in _do_raw_rest(self, url_suffix, method, file_upload_info, **kwargs)
    476 
    477     begin_time_seconds = time.time()
--> 478     http_result = self._attempt_rest(url, method, post_body, file_upload_info)
    479     end_time_seconds = time.time()
    480     elapsed_time_seconds = end_time_seconds - begin_time_seconds

C:\Anaconda2\lib\site-packages\h2o\connection.pyc in _attempt_rest(self, url, method, post_body, file_upload_info)
    526 
    527     except requests.ConnectionError as e:
--> 528       raise EnvironmentError("h2o-py encountered an unexpected HTTP error:\n {}".format(e))
    529 
    530     return http_result

EnvironmentError: h2o-py encountered an unexpected HTTP error:
 ('Connection aborted.', BadStatusLine("''",))

我的预感是集群内存只有大约 247.5 MB，不足以处理模型构建，因此中止了与 h2o 的连接。以下是我用来启动 h2o 的代码:

 #initialization of h2o module
import subprocess as sp
import sys
import os.path as p

# path of h2o jar file
h2o_path = p.join(sys.prefix, "h2o_jar", "h2o.jar")

# subprocess to launch h2o
# the command can be further modified to include virtual machine parameters
sp.Popen("java -jar " + h2o_path)

# h2o.init() call to verify that h2o launch is successfull
h2o.init(ip="localhost", port=54321, size=1, start_h2o=False, enable_assertions=False, \
         license=None, max_mem_size_GB=4, min_mem_size_GB=4, ice_root=None)

这是返回的状态表:

任何有关上述内容的想法将不胜感激!!

Answer 1

为了结束这个问题，我将重申上面评论中提到的解决方案。用户能够通过使用 java -jar -Xmx1g h2o.jar 从命令行启动具有 1GB 内存的 H2O 来解决该问题。 ，然后使用 Python 连接到现有的 H2O 服务器 h2o.init() 。

我不清楚为什么 h2o.init()没有使用 max_mem_size_GB 创建正确大小的集群争论。无论如何，这个参数最近已被弃用并被另一个参数取代，max_mem_size ，所以这可能不再是一个问题。