{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosted Decision Tree\n",
    "\n",
    "Implement a Gradient Boosted Decision tree (GBDT) with TensorFlow to classify\n",
    "handwritten digit images. This example is using the MNIST database of\n",
    "handwritten digits as training samples (http://yann.lecun.com/exdb/mnist/).\n",
    "\n",
    "- Author: Aymeric Damien\n",
    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier\n",
    "from tensorflow.contrib.boosted_trees.proto import learner_pb2 as gbdt_learner\n",
    "\n",
    "# Ignore all GPUs (current TF GBDT does not support GPU).\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "# Set verbosity to display errors only (Remove this line for showing warnings)\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=False,\n",
    "                                  source_url='http://yann.lecun.com/exdb/mnist/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "batch_size = 4096 # The number of samples per batch\n",
    "num_classes = 10 # The 10 digits\n",
    "num_features = 784 # Each image is 28x28 pixels\n",
    "max_steps = 10000\n",
    "\n",
    "# GBDT Parameters\n",
    "learning_rate = 0.1\n",
    "l1_regul = 0.\n",
    "l2_regul = 1.\n",
    "examples_per_layer = 1000\n",
    "num_trees = 10\n",
    "max_depth = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fill GBDT parameters into the config proto\n",
    "learner_config = gbdt_learner.LearnerConfig()\n",
    "learner_config.learning_rate_tuner.fixed.learning_rate = learning_rate\n",
    "learner_config.regularization.l1 = l1_regul\n",
    "learner_config.regularization.l2 = l2_regul / examples_per_layer\n",
    "learner_config.constraints.max_tree_depth = max_depth\n",
    "growing_mode = gbdt_learner.LearnerConfig.LAYER_BY_LAYER\n",
    "learner_config.growing_mode = growing_mode\n",
    "run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300)\n",
    "learner_config.multi_class_strategy = (\n",
    "    gbdt_learner.LearnerConfig.DIAGONAL_HESSIAN)\\\n",
    "\n",
    "# Create a TensorFlor GBDT Estimator\n",
    "gbdt_model = GradientBoostedDecisionTreeClassifier(\n",
    "    model_dir=None, # No save directory specified\n",
    "    learner_config=learner_config,\n",
    "    n_classes=num_classes,\n",
    "    examples_per_layer=examples_per_layer,\n",
    "    num_trees=num_trees,\n",
    "    center_bias=False,\n",
    "    config=run_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Active Feature Columns: ['images_0', 'images_1', 'images_2', 'images_3', 'images_4', 'images_5', 'images_6', 'images_7', 'images_8', 'images_9', 'images_10', 'images_11', 'images_12', 'images_13', 'images_14', 'images_15', 'images_16', 'images_17', 'images_18', 'images_19', 'images_20', 'images_21', 'images_22', 'images_23', 'images_24', 'images_25', 'images_26', 'images_27', 'images_28', 'images_29', 'images_30', 'images_31', 'images_32', 'images_33', 'images_34', 'images_35', 'images_36', 'images_37', 'images_38', 'images_39', 'images_40', 'images_41', 'images_42', 'images_43', 'images_44', 'images_45', 'images_46', 'images_47', 'images_48', 'images_49', 'images_50', 'images_51', 'images_52', 'images_53', 'images_54', 'images_55', 'images_56', 'images_57', 'images_58', 'images_59', 'images_60', 'images_61', 'images_62', 'images_63', 'images_64', 'images_65', 'images_66', 'images_67', 'images_68', 'images_69', 'images_70', 'images_71', 'images_72', 'images_73', 'images_74', 'images_75', 'images_76', 'images_77', 'images_78', 'images_79', 'images_80', 'images_81', 'images_82', 'images_83', 'images_84', 'images_85', 'images_86', 'images_87', 'images_88', 'images_89', 'images_90', 'images_91', 'images_92', 'images_93', 'images_94', 'images_95', 'images_96', 'images_97', 'images_98', 'images_99', 'images_100', 'images_101', 'images_102', 'images_103', 'images_104', 'images_105', 'images_106', 'images_107', 'images_108', 'images_109', 'images_110', 'images_111', 'images_112', 'images_113', 'images_114', 'images_115', 'images_116', 'images_117', 'images_118', 'images_119', 'images_120', 'images_121', 'images_122', 'images_123', 'images_124', 'images_125', 'images_126', 'images_127', 'images_128', 'images_129', 'images_130', 'images_131', 'images_132', 'images_133', 'images_134', 'images_135', 'images_136', 'images_137', 'images_138', 'images_139', 'images_140', 'images_141', 'images_142', 'images_143', 'images_144', 'images_145', 'images_146', 'images_147', 'images_148', 'images_149', 'images_150', 'images_151', 'images_152', 'images_153', 'images_154', 'images_155', 'images_156', 'images_157', 'images_158', 'images_159', 'images_160', 'images_161', 'images_162', 'images_163', 'images_164', 'images_165', 'images_166', 'images_167', 'images_168', 'images_169', 'images_170', 'images_171', 'images_172', 'images_173', 'images_174', 'images_175', 'images_176', 'images_177', 'images_178', 'images_179', 'images_180', 'images_181', 'images_182', 'images_183', 'images_184', 'images_185', 'images_186', 'images_187', 'images_188', 'images_189', 'images_190', 'images_191', 'images_192', 'images_193', 'images_194', 'images_195', 'images_196', 'images_197', 'images_198', 'images_199', 'images_200', 'images_201', 'images_202', 'images_203', 'images_204', 'images_205', 'images_206', 'images_207', 'images_208', 'images_209', 'images_210', 'images_211', 'images_212', 'images_213', 'images_214', 'images_215', 'images_216', 'images_217', 'images_218', 'images_219', 'images_220', 'images_221', 'images_222', 'images_223', 'images_224', 'images_225', 'images_226', 'images_227', 'images_228', 'images_229', 'images_230', 'images_231', 'images_232', 'images_233', 'images_234', 'images_235', 'images_236', 'images_237', 'images_238', 'images_239', 'images_240', 'images_241', 'images_242', 'images_243', 'images_244', 'images_245', 'images_246', 'images_247', 'images_248', 'images_249', 'images_250', 'images_251', 'images_252', 'images_253', 'images_254', 'images_255', 'images_256', 'images_257', 'images_258', 'images_259', 'images_260', 'images_261', 'images_262', 'images_263', 'images_264', 'images_265', 'images_266', 'images_267', 'images_268', 'images_269', 'images_270', 'images_271', 'images_272', 'images_273', 'images_274', 'images_275', 'images_276', 'images_277', 'images_278', 'images_279', 'images_280', 'images_281', 'images_282', 'images_283', 'images_284', 'images_285', 'images_286', 'images_287', 'images_288', 'images_289', 'images_290', 'images_291', 'images_292', 'images_293', 'images_294', 'images_295', 'images_296', 'images_297', 'images_298', 'images_299', 'images_300', 'images_301', 'images_302', 'images_303', 'images_304', 'images_305', 'images_306', 'images_307', 'images_308', 'images_309', 'images_310', 'images_311', 'images_312', 'images_313', 'images_314', 'images_315', 'images_316', 'images_317', 'images_318', 'images_319', 'images_320', 'images_321', 'images_322', 'images_323', 'images_324', 'images_325', 'images_326', 'images_327', 'images_328', 'images_329', 'images_330', 'images_331', 'images_332', 'images_333', 'images_334', 'images_335', 'images_336', 'images_337', 'images_338', 'images_339', 'images_340', 'images_341', 'images_342', 'images_343', 'images_344', 'images_345', 'images_346', 'images_347', 'images_348', 'images_349', 'images_350', 'images_351', 'images_352', 'images_353', 'images_354', 'images_355', 'images_356', 'images_357', 'images_358', 'images_359', 'images_360', 'images_361', 'images_362', 'images_363', 'images_364', 'images_365', 'images_366', 'images_367', 'images_368', 'images_369', 'images_370', 'images_371', 'images_372', 'images_373', 'images_374', 'images_375', 'images_376', 'images_377', 'images_378', 'images_379', 'images_380', 'images_381', 'images_382', 'images_383', 'images_384', 'images_385', 'images_386', 'images_387', 'images_388', 'images_389', 'images_390', 'images_391', 'images_392', 'images_393', 'images_394', 'images_395', 'images_396', 'images_397', 'images_398', 'images_399', 'images_400', 'images_401', 'images_402', 'images_403', 'images_404', 'images_405', 'images_406', 'images_407', 'images_408', 'images_409', 'images_410', 'images_411', 'images_412', 'images_413', 'images_414', 'images_415', 'images_416', 'images_417', 'images_418', 'images_419', 'images_420', 'images_421', 'images_422', 'images_423', 'images_424', 'images_425', 'images_426', 'images_427', 'images_428', 'images_429', 'images_430', 'images_431', 'images_432', 'images_433', 'images_434', 'images_435', 'images_436', 'images_437', 'images_438', 'images_439', 'images_440', 'images_441', 'images_442', 'images_443', 'images_444', 'images_445', 'images_446', 'images_447', 'images_448', 'images_449', 'images_450', 'images_451', 'images_452', 'images_453', 'images_454', 'images_455', 'images_456', 'images_457', 'images_458', 'images_459', 'images_460', 'images_461', 'images_462', 'images_463', 'images_464', 'images_465', 'images_466', 'images_467', 'images_468', 'images_469', 'images_470', 'images_471', 'images_472', 'images_473', 'images_474', 'images_475', 'images_476', 'images_477', 'images_478', 'images_479', 'images_480', 'images_481', 'images_482', 'images_483', 'images_484', 'images_485', 'images_486', 'images_487', 'images_488', 'images_489', 'images_490', 'images_491', 'images_492', 'images_493', 'images_494', 'images_495', 'images_496', 'images_497', 'images_498', 'images_499', 'images_500', 'images_501', 'images_502', 'images_503', 'images_504', 'images_505', 'images_506', 'images_507', 'images_508', 'images_509', 'images_510', 'images_511', 'images_512', 'images_513', 'images_514', 'images_515', 'images_516', 'images_517', 'images_518', 'images_519', 'images_520', 'images_521', 'images_522', 'images_523', 'images_524', 'images_525', 'images_526', 'images_527', 'images_528', 'images_529', 'images_530', 'images_531', 'images_532', 'images_533', 'images_534', 'images_535', 'images_536', 'images_537', 'images_538', 'images_539', 'images_540', 'images_541', 'images_542', 'images_543', 'images_544', 'images_545', 'images_546', 'images_547', 'images_548', 'images_549', 'images_550', 'images_551', 'images_552', 'images_553', 'images_554', 'images_555', 'images_556', 'images_557', 'images_558', 'images_559', 'images_560', 'images_561', 'images_562', 'images_563', 'images_564', 'images_565', 'images_566', 'images_567', 'images_568', 'images_569', 'images_570', 'images_571', 'images_572', 'images_573', 'images_574', 'images_575', 'images_576', 'images_577', 'images_578', 'images_579', 'images_580', 'images_581', 'images_582', 'images_583', 'images_584', 'images_585', 'images_586', 'images_587', 'images_588', 'images_589', 'images_590', 'images_591', 'images_592', 'images_593', 'images_594', 'images_595', 'images_596', 'images_597', 'images_598', 'images_599', 'images_600', 'images_601', 'images_602', 'images_603', 'images_604', 'images_605', 'images_606', 'images_607', 'images_608', 'images_609', 'images_610', 'images_611', 'images_612', 'images_613', 'images_614', 'images_615', 'images_616', 'images_617', 'images_618', 'images_619', 'images_620', 'images_621', 'images_622', 'images_623', 'images_624', 'images_625', 'images_626', 'images_627', 'images_628', 'images_629', 'images_630', 'images_631', 'images_632', 'images_633', 'images_634', 'images_635', 'images_636', 'images_637', 'images_638', 'images_639', 'images_640', 'images_641', 'images_642', 'images_643', 'images_644', 'images_645', 'images_646', 'images_647', 'images_648', 'images_649', 'images_650', 'images_651', 'images_652', 'images_653', 'images_654', 'images_655', 'images_656', 'images_657', 'images_658', 'images_659', 'images_660', 'images_661', 'images_662', 'images_663', 'images_664', 'images_665', 'images_666', 'images_667', 'images_668', 'images_669', 'images_670', 'images_671', 'images_672', 'images_673', 'images_674', 'images_675', 'images_676', 'images_677', 'images_678', 'images_679', 'images_680', 'images_681', 'images_682', 'images_683', 'images_684', 'images_685', 'images_686', 'images_687', 'images_688', 'images_689', 'images_690', 'images_691', 'images_692', 'images_693', 'images_694', 'images_695', 'images_696', 'images_697', 'images_698', 'images_699', 'images_700', 'images_701', 'images_702', 'images_703', 'images_704', 'images_705', 'images_706', 'images_707', 'images_708', 'images_709', 'images_710', 'images_711', 'images_712', 'images_713', 'images_714', 'images_715', 'images_716', 'images_717', 'images_718', 'images_719', 'images_720', 'images_721', 'images_722', 'images_723', 'images_724', 'images_725', 'images_726', 'images_727', 'images_728', 'images_729', 'images_730', 'images_731', 'images_732', 'images_733', 'images_734', 'images_735', 'images_736', 'images_737', 'images_738', 'images_739', 'images_740', 'images_741', 'images_742', 'images_743', 'images_744', 'images_745', 'images_746', 'images_747', 'images_748', 'images_749', 'images_750', 'images_751', 'images_752', 'images_753', 'images_754', 'images_755', 'images_756', 'images_757', 'images_758', 'images_759', 'images_760', 'images_761', 'images_762', 'images_763', 'images_764', 'images_765', 'images_766', 'images_767', 'images_768', 'images_769', 'images_770', 'images_771', 'images_772', 'images_773', 'images_774', 'images_775', 'images_776', 'images_777', 'images_778', 'images_779', 'images_780', 'images_781', 'images_782', 'images_783']\n",
      "WARNING:tensorflow:From /Users/aymeric.damien/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:678: __new__ (from tensorflow.contrib.learn.python.learn.estimators.model_fn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.EstimatorSpec. You can use the `estimator_spec` method to create an equivalent one.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /var/folders/p7/9nxn5g_s5kv636j86zdg2hrr0000gr/T/tmpE_8oiQ/model.ckpt.\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "INFO:tensorflow:loss = 2.3025992, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 2 into /var/folders/p7/9nxn5g_s5kv636j86zdg2hrr0000gr/T/tmpE_8oiQ/model.ckpt.\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "INFO:tensorflow:Saving checkpoints for 94 into /var/folders/p7/9nxn5g_s5kv636j86zdg2hrr0000gr/T/tmpE_8oiQ/model.ckpt.\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "INFO:tensorflow:global_step/sec: 0.199624\n",
      "INFO:tensorflow:loss = 0.32783023, step = 101 (500.943 sec)\n",
      "INFO:tensorflow:Requesting stop since we have reached 10 trees.\n",
      "INFO:tensorflow:Saving checkpoints for 161 into /var/folders/p7/9nxn5g_s5kv636j86zdg2hrr0000gr/T/tmpE_8oiQ/model.ckpt.\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "INFO:tensorflow:Loss for final step: 0.21336032.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostedDecisionTreeClassifier(params={'head': <tensorflow.contrib.learn.python.learn.estimators.head._MultiClassHead object at 0x127568650>, 'weight_column_name': None, 'feature_columns': None, 'center_bias': False, 'num_trees': 10, 'logits_modifier_function': None, 'use_core_libs': False, 'learner_config': num_classes: 10\n",
       "regularization {\n",
       "  l2: 0.0010000000475\n",
       "}\n",
       "constraints {\n",
       "  max_tree_depth: 16\n",
       "}\n",
       "learning_rate_tuner {\n",
       "  fixed {\n",
       "    learning_rate: 0.10000000149\n",
       "  }\n",
       "}\n",
       "pruning_mode: POST_PRUNE\n",
       "growing_mode: LAYER_BY_LAYER\n",
       "multi_class_strategy: DIAGONAL_HESSIAN\n",
       ", 'examples_per_layer': 1000})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display TF info logs\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "\n",
    "# Define the input function for training\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={'images': mnist.train.images}, y=mnist.train.labels,\n",
    "    batch_size=batch_size, num_epochs=None, shuffle=True)\n",
    "\n",
    "# Train the Model\n",
    "gbdt_model.fit(input_fn=input_fn, max_steps=max_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Active Feature Columns: ['images_0', 'images_1', 'images_2', 'images_3', 'images_4', 'images_5', 'images_6', 'images_7', 'images_8', 'images_9', 'images_10', 'images_11', 'images_12', 'images_13', 'images_14', 'images_15', 'images_16', 'images_17', 'images_18', 'images_19', 'images_20', 'images_21', 'images_22', 'images_23', 'images_24', 'images_25', 'images_26', 'images_27', 'images_28', 'images_29', 'images_30', 'images_31', 'images_32', 'images_33', 'images_34', 'images_35', 'images_36', 'images_37', 'images_38', 'images_39', 'images_40', 'images_41', 'images_42', 'images_43', 'images_44', 'images_45', 'images_46', 'images_47', 'images_48', 'images_49', 'images_50', 'images_51', 'images_52', 'images_53', 'images_54', 'images_55', 'images_56', 'images_57', 'images_58', 'images_59', 'images_60', 'images_61', 'images_62', 'images_63', 'images_64', 'images_65', 'images_66', 'images_67', 'images_68', 'images_69', 'images_70', 'images_71', 'images_72', 'images_73', 'images_74', 'images_75', 'images_76', 'images_77', 'images_78', 'images_79', 'images_80', 'images_81', 'images_82', 'images_83', 'images_84', 'images_85', 'images_86', 'images_87', 'images_88', 'images_89', 'images_90', 'images_91', 'images_92', 'images_93', 'images_94', 'images_95', 'images_96', 'images_97', 'images_98', 'images_99', 'images_100', 'images_101', 'images_102', 'images_103', 'images_104', 'images_105', 'images_106', 'images_107', 'images_108', 'images_109', 'images_110', 'images_111', 'images_112', 'images_113', 'images_114', 'images_115', 'images_116', 'images_117', 'images_118', 'images_119', 'images_120', 'images_121', 'images_122', 'images_123', 'images_124', 'images_125', 'images_126', 'images_127', 'images_128', 'images_129', 'images_130', 'images_131', 'images_132', 'images_133', 'images_134', 'images_135', 'images_136', 'images_137', 'images_138', 'images_139', 'images_140', 'images_141', 'images_142', 'images_143', 'images_144', 'images_145', 'images_146', 'images_147', 'images_148', 'images_149', 'images_150', 'images_151', 'images_152', 'images_153', 'images_154', 'images_155', 'images_156', 'images_157', 'images_158', 'images_159', 'images_160', 'images_161', 'images_162', 'images_163', 'images_164', 'images_165', 'images_166', 'images_167', 'images_168', 'images_169', 'images_170', 'images_171', 'images_172', 'images_173', 'images_174', 'images_175', 'images_176', 'images_177', 'images_178', 'images_179', 'images_180', 'images_181', 'images_182', 'images_183', 'images_184', 'images_185', 'images_186', 'images_187', 'images_188', 'images_189', 'images_190', 'images_191', 'images_192', 'images_193', 'images_194', 'images_195', 'images_196', 'images_197', 'images_198', 'images_199', 'images_200', 'images_201', 'images_202', 'images_203', 'images_204', 'images_205', 'images_206', 'images_207', 'images_208', 'images_209', 'images_210', 'images_211', 'images_212', 'images_213', 'images_214', 'images_215', 'images_216', 'images_217', 'images_218', 'images_219', 'images_220', 'images_221', 'images_222', 'images_223', 'images_224', 'images_225', 'images_226', 'images_227', 'images_228', 'images_229', 'images_230', 'images_231', 'images_232', 'images_233', 'images_234', 'images_235', 'images_236', 'images_237', 'images_238', 'images_239', 'images_240', 'images_241', 'images_242', 'images_243', 'images_244', 'images_245', 'images_246', 'images_247', 'images_248', 'images_249', 'images_250', 'images_251', 'images_252', 'images_253', 'images_254', 'images_255', 'images_256', 'images_257', 'images_258', 'images_259', 'images_260', 'images_261', 'images_262', 'images_263', 'images_264', 'images_265', 'images_266', 'images_267', 'images_268', 'images_269', 'images_270', 'images_271', 'images_272', 'images_273', 'images_274', 'images_275', 'images_276', 'images_277', 'images_278', 'images_279', 'images_280', 'images_281', 'images_282', 'images_283', 'images_284', 'images_285', 'images_286', 'images_287', 'images_288', 'images_289', 'images_290', 'images_291', 'images_292', 'images_293', 'images_294', 'images_295', 'images_296', 'images_297', 'images_298', 'images_299', 'images_300', 'images_301', 'images_302', 'images_303', 'images_304', 'images_305', 'images_306', 'images_307', 'images_308', 'images_309', 'images_310', 'images_311', 'images_312', 'images_313', 'images_314', 'images_315', 'images_316', 'images_317', 'images_318', 'images_319', 'images_320', 'images_321', 'images_322', 'images_323', 'images_324', 'images_325', 'images_326', 'images_327', 'images_328', 'images_329', 'images_330', 'images_331', 'images_332', 'images_333', 'images_334', 'images_335', 'images_336', 'images_337', 'images_338', 'images_339', 'images_340', 'images_341', 'images_342', 'images_343', 'images_344', 'images_345', 'images_346', 'images_347', 'images_348', 'images_349', 'images_350', 'images_351', 'images_352', 'images_353', 'images_354', 'images_355', 'images_356', 'images_357', 'images_358', 'images_359', 'images_360', 'images_361', 'images_362', 'images_363', 'images_364', 'images_365', 'images_366', 'images_367', 'images_368', 'images_369', 'images_370', 'images_371', 'images_372', 'images_373', 'images_374', 'images_375', 'images_376', 'images_377', 'images_378', 'images_379', 'images_380', 'images_381', 'images_382', 'images_383', 'images_384', 'images_385', 'images_386', 'images_387', 'images_388', 'images_389', 'images_390', 'images_391', 'images_392', 'images_393', 'images_394', 'images_395', 'images_396', 'images_397', 'images_398', 'images_399', 'images_400', 'images_401', 'images_402', 'images_403', 'images_404', 'images_405', 'images_406', 'images_407', 'images_408', 'images_409', 'images_410', 'images_411', 'images_412', 'images_413', 'images_414', 'images_415', 'images_416', 'images_417', 'images_418', 'images_419', 'images_420', 'images_421', 'images_422', 'images_423', 'images_424', 'images_425', 'images_426', 'images_427', 'images_428', 'images_429', 'images_430', 'images_431', 'images_432', 'images_433', 'images_434', 'images_435', 'images_436', 'images_437', 'images_438', 'images_439', 'images_440', 'images_441', 'images_442', 'images_443', 'images_444', 'images_445', 'images_446', 'images_447', 'images_448', 'images_449', 'images_450', 'images_451', 'images_452', 'images_453', 'images_454', 'images_455', 'images_456', 'images_457', 'images_458', 'images_459', 'images_460', 'images_461', 'images_462', 'images_463', 'images_464', 'images_465', 'images_466', 'images_467', 'images_468', 'images_469', 'images_470', 'images_471', 'images_472', 'images_473', 'images_474', 'images_475', 'images_476', 'images_477', 'images_478', 'images_479', 'images_480', 'images_481', 'images_482', 'images_483', 'images_484', 'images_485', 'images_486', 'images_487', 'images_488', 'images_489', 'images_490', 'images_491', 'images_492', 'images_493', 'images_494', 'images_495', 'images_496', 'images_497', 'images_498', 'images_499', 'images_500', 'images_501', 'images_502', 'images_503', 'images_504', 'images_505', 'images_506', 'images_507', 'images_508', 'images_509', 'images_510', 'images_511', 'images_512', 'images_513', 'images_514', 'images_515', 'images_516', 'images_517', 'images_518', 'images_519', 'images_520', 'images_521', 'images_522', 'images_523', 'images_524', 'images_525', 'images_526', 'images_527', 'images_528', 'images_529', 'images_530', 'images_531', 'images_532', 'images_533', 'images_534', 'images_535', 'images_536', 'images_537', 'images_538', 'images_539', 'images_540', 'images_541', 'images_542', 'images_543', 'images_544', 'images_545', 'images_546', 'images_547', 'images_548', 'images_549', 'images_550', 'images_551', 'images_552', 'images_553', 'images_554', 'images_555', 'images_556', 'images_557', 'images_558', 'images_559', 'images_560', 'images_561', 'images_562', 'images_563', 'images_564', 'images_565', 'images_566', 'images_567', 'images_568', 'images_569', 'images_570', 'images_571', 'images_572', 'images_573', 'images_574', 'images_575', 'images_576', 'images_577', 'images_578', 'images_579', 'images_580', 'images_581', 'images_582', 'images_583', 'images_584', 'images_585', 'images_586', 'images_587', 'images_588', 'images_589', 'images_590', 'images_591', 'images_592', 'images_593', 'images_594', 'images_595', 'images_596', 'images_597', 'images_598', 'images_599', 'images_600', 'images_601', 'images_602', 'images_603', 'images_604', 'images_605', 'images_606', 'images_607', 'images_608', 'images_609', 'images_610', 'images_611', 'images_612', 'images_613', 'images_614', 'images_615', 'images_616', 'images_617', 'images_618', 'images_619', 'images_620', 'images_621', 'images_622', 'images_623', 'images_624', 'images_625', 'images_626', 'images_627', 'images_628', 'images_629', 'images_630', 'images_631', 'images_632', 'images_633', 'images_634', 'images_635', 'images_636', 'images_637', 'images_638', 'images_639', 'images_640', 'images_641', 'images_642', 'images_643', 'images_644', 'images_645', 'images_646', 'images_647', 'images_648', 'images_649', 'images_650', 'images_651', 'images_652', 'images_653', 'images_654', 'images_655', 'images_656', 'images_657', 'images_658', 'images_659', 'images_660', 'images_661', 'images_662', 'images_663', 'images_664', 'images_665', 'images_666', 'images_667', 'images_668', 'images_669', 'images_670', 'images_671', 'images_672', 'images_673', 'images_674', 'images_675', 'images_676', 'images_677', 'images_678', 'images_679', 'images_680', 'images_681', 'images_682', 'images_683', 'images_684', 'images_685', 'images_686', 'images_687', 'images_688', 'images_689', 'images_690', 'images_691', 'images_692', 'images_693', 'images_694', 'images_695', 'images_696', 'images_697', 'images_698', 'images_699', 'images_700', 'images_701', 'images_702', 'images_703', 'images_704', 'images_705', 'images_706', 'images_707', 'images_708', 'images_709', 'images_710', 'images_711', 'images_712', 'images_713', 'images_714', 'images_715', 'images_716', 'images_717', 'images_718', 'images_719', 'images_720', 'images_721', 'images_722', 'images_723', 'images_724', 'images_725', 'images_726', 'images_727', 'images_728', 'images_729', 'images_730', 'images_731', 'images_732', 'images_733', 'images_734', 'images_735', 'images_736', 'images_737', 'images_738', 'images_739', 'images_740', 'images_741', 'images_742', 'images_743', 'images_744', 'images_745', 'images_746', 'images_747', 'images_748', 'images_749', 'images_750', 'images_751', 'images_752', 'images_753', 'images_754', 'images_755', 'images_756', 'images_757', 'images_758', 'images_759', 'images_760', 'images_761', 'images_762', 'images_763', 'images_764', 'images_765', 'images_766', 'images_767', 'images_768', 'images_769', 'images_770', 'images_771', 'images_772', 'images_773', 'images_774', 'images_775', 'images_776', 'images_777', 'images_778', 'images_779', 'images_780', 'images_781', 'images_782', 'images_783']\n",
      "INFO:tensorflow:Starting evaluation at 2018-07-26-01:00:06\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/p7/9nxn5g_s5kv636j86zdg2hrr0000gr/T/tmpE_8oiQ/model.ckpt-161\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-07-26-01:00:07\n",
      "INFO:tensorflow:Saving dict for global step 161: accuracy = 0.9273, global_step = 161, loss = 0.23841818\n",
      "WARNING:tensorflow:Issue encountered when serializing resources.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n",
      "'_Resource' object has no attribute 'name'\n",
      "Testing Accuracy: 0.9273\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the Model\n",
    "# Define the input function for evaluating\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={'images': mnist.test.images}, y=mnist.test.labels,\n",
    "    batch_size=batch_size, shuffle=False)\n",
    "\n",
    "# Use the Estimator 'evaluate' method\n",
    "e = gbdt_model.evaluate(input_fn=input_fn)\n",
    "print(\"Testing Accuracy:\", e['accuracy'])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}