From de734384dc2ac1d2b8ac1867090176b550217ff0 Mon Sep 17 00:00:00 2001 From: Sasha Kolpakov Date: Tue, 31 Oct 2023 21:42:22 +0100 Subject: [PATCH 1/3] Add files via upload Fine-tuning RegNet on Cifar-10 by using Cerebros. --- cifar_100_RegNet.ipynb | 3556 ++++++++++++++++++++++++++++++++++++++++ cifar_100_RegNet.py | 207 +++ 2 files changed, 3763 insertions(+) create mode 100644 cifar_100_RegNet.ipynb create mode 100644 cifar_100_RegNet.py diff --git a/cifar_100_RegNet.ipynb b/cifar_100_RegNet.ipynb new file mode 100644 index 0000000..0db37f2 --- /dev/null +++ b/cifar_100_RegNet.ipynb @@ -0,0 +1,3556 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Cifar-100 test for image classification" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(0, '../..')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [ + "imports" + ] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/sasha/jax-metal/lib/python3.8/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "from tensorflow.keras.datasets import cifar10, cifar100\n", + "from tensorflow.keras.utils import to_categorical\n", + "from tensorflow.keras.models import Model\n", + "from tensorflow.keras.layers import Input, Resizing, Lambda, Flatten, Dense\n", + "import pandas as pd\n", + "import numpy as np\n", + "from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\\\n", + " import SimpleCerebrosRandomSearch\n", + "import pendulum\n", + "from cerebros.units.units import DenseUnit\n", + "from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\\\n", + " import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid\n", + "from ast import literal_eval" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Download RegNetX016 with Imagenet weights (1000 classes)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016',\n", + " include_top=True,\n", + " include_preprocessing=True,\n", + " weights='imagenet',\n", + " input_tensor=None,\n", + " input_shape=None,\n", + " pooling=None,\n", + " classes=1000,\n", + " classifier_activation='softmax'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"regnetx016\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " input_1 (InputLayer) [(None, 224, 224, 3)] 0 [] \n", + " \n", + " regnetx016_prestem_rescali (None, 224, 224, 3) 0 ['input_1[0][0]'] \n", + " ng (Rescaling) \n", + " \n", + " regnetx016_stem_conv (Conv (None, 112, 112, 32) 864 ['regnetx016_prestem_rescaling\n", + " 2D) [0][0]'] \n", + " \n", + " regnetx016_stem_bn (BatchN (None, 112, 112, 32) 128 ['regnetx016_stem_conv[0][0]']\n", + " ormalization) \n", + " \n", + " regnetx016_stem_relu (ReLU (None, 112, 112, 32) 0 ['regnetx016_stem_bn[0][0]'] \n", + " ) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 112, 112, 72) 2304 ['regnetx016_stem_relu[0][0]']\n", + " 0_conv_1x1_1 (Conv2D) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 112, 112, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 112, 112, 72) 0 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 15552 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 5184 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 2304 ['regnetx016_stem_relu[0][0]']\n", + " 0_skip_1x1 (Conv2D) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add (TFOp (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_0_\n", + " Lambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_0_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['tf.__operators__.add[0][0]']\n", + " 0_exit_relu (ReLU) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 5184 ['regnetx016_Stage_0_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 15552 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 5184 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_1 (TF (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_1_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_0_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['tf.__operators__.add_1[0][0]\n", + " 1_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 56, 56, 168) 12096 ['regnetx016_Stage_0_XBlock_1_\n", + " 0_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 56, 56, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 56, 56, 168) 0 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 12096 ['regnetx016_Stage_0_XBlock_1_\n", + " 0_skip_1x1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_2 (TF (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_0_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_2[0][0]\n", + " 0_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_3 (TF (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_1_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_3[0][0]\n", + " 1_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_1_\n", + " 2_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_4 (TF (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_2_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_1_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_4[0][0]\n", + " 2_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_2_\n", + " 3_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_5 (TF (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_3_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_2_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_5[0][0]\n", + " 3_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 28, 28, 408) 68544 ['regnetx016_Stage_1_XBlock_3_\n", + " 0_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 28, 28, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 28, 28, 408) 0 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 68544 ['regnetx016_Stage_1_XBlock_3_\n", + " 0_skip_1x1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_6 (TF (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_0_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_6[0][0]\n", + " 0_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_7 (TF (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_1_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_7[0][0]\n", + " 1_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_1_\n", + " 2_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_8 (TF (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_2_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_1_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_8[0][0]\n", + " 2_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_2_\n", + " 3_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_9 (TF (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_3_\n", + " OpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_2_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_9[0][0]\n", + " 3_exit_relu (ReLU) '] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_3_\n", + " 4_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_10 (T (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_4_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_3_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_10[0][0\n", + " 4_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_4_\n", + " 5_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_11 (T (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_5_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_4_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_11[0][0\n", + " 5_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_5_\n", + " 6_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_12 (T (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_6_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_5_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_12[0][0\n", + " 6_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_6_\n", + " 7_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_13 (T (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_7_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_6_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_13[0][0\n", + " 7_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_7_\n", + " 8_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_14 (T (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_8_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_7_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_14[0][0\n", + " 8_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_8_\n", + " 9_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_15 (T (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_9_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_8_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_15[0][0\n", + " 9_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 14, 14, 912) 372096 ['regnetx016_Stage_2_XBlock_9_\n", + " 0_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 14, 14, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 14, 14, 912) 0 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 196992 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 831744 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 372096 ['regnetx016_Stage_2_XBlock_9_\n", + " 0_skip_1x1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_16 (T (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_0_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_3_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['tf.__operators__.add_16[0][0\n", + " 0_exit_relu (ReLU) ]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 831744 ['regnetx016_Stage_3_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 196992 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 831744 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_17 (T (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " FOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_3_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['tf.__operators__.add_17[0][0\n", + " 1_exit_relu (ReLU) ]'] \n", + " \n", + " 1_head_gap (GlobalAverageP (None, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " ooling2D) exit_relu[0][0]'] \n", + " \n", + " 1head_dense (Dense) (None, 1000) 913000 ['1_head_gap[0][0]'] \n", + " \n", + "==================================================================================================\n", + "Total params: 9233640 (35.22 MB)\n", + "Trainable params: 9190136 (35.06 MB)\n", + "Non-trainable params: 43504 (169.94 KB)\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "regnet.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Make all layers untrainable except for the very last convolutional layer" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "for layer in regnet.layers:\n", + " layer.trainable = False\n", + "regnet.layers[-6].trainable = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Cifar-100 testing" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "(X_train, y_train), (X_test, y_test) = cifar100.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "y_train_cat = to_categorical(y_train, 1000)\n", + "y_test_cat = to_categorical(y_test, 1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lambda layer for preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def resize(x):\n", + " return tf.image.resize(x,size=(224,224),method='bicubic')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Modify the model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (32,32,3)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = Input(shape=input_shape)\n", + "prep = Lambda(resize)(input_layer)\n", + "out = regnet(prep)\n", + "regnet_mod = Model(inputs=input_layer, outputs=out)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "regnet_mod.compile(optimizer='adam',\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_categorical_accuracy')])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try to fit it on Cifar-100 data and then evaluate (there is no hope this is gonna work ...)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1563/1563 [==============================] - 1840s 1s/step - loss: 5.4922 - top_1_categorical_accuracy: 4.0000e-05\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "regnet_mod.fit(X_train, y_train_cat)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "313/313 [==============================] - 366s 1s/step - loss: 5.2863 - top_1_categorical_accuracy: 0.0000e+00\n" + ] + }, + { + "data": { + "text/plain": [ + "[5.286314010620117, 0.0]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "regnet_mod.evaluate(X_test, y_test_cat)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try the same with adding a Cerebros \"add-on\" network" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "INPUT_SHAPES = [input_shape]\n", + "OUTPUT_SHAPES = [100]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use 10k-15k random samples from Cifar-100 to speed up the process" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "num_samples = 15_000\n", + "rng = np.random.default_rng()\n", + "ind = rng.permutation(X_train.shape[0])[:num_samples]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "training_x = [tf.constant(X_train[ind,:,:,:])]\n", + "y_train_cat = to_categorical(y_train[ind], 100)\n", + "train_labels = [tf.constant(y_train_cat)]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016',\n", + " include_top=True,\n", + " include_preprocessing=True,\n", + " weights='imagenet',\n", + " input_tensor=None,\n", + " input_shape=None,\n", + " pooling=None,\n", + " classes=1000,\n", + " classifier_activation='softmax'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "for layer in regnet.layers:\n", + " layer.trainable = False\n", + "regnet.layers[-6].trainable = True" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "regnet_io = Model(inputs=regnet.layers[0].input,\n", + " outputs=regnet.layers[-2].output)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = Input(shape=input_shape)\n", + "prep = Lambda(resize)(input_layer)\n", + "out = Flatten()(regnet_io(prep))\n", + "base_mod = Model(inputs=input_layer, outputs=out)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "activation = 'relu'\n", + "predecessor_level_connection_affinity_factor_first = 2.0\n", + "predecessor_level_connection_affinity_factor_main = 0.97\n", + "max_consecutive_lateral_connections = 5\n", + "p_lateral_connection = 0.97\n", + "num_lateral_connection_tries_per_unit = 2\n", + "learning_rate = 0.001\n", + "epochs = 10 # [1, 100]\n", + "batch_size = 20\n", + "maximum_levels = 4 # [3,7]\n", + "maximum_units_per_level = 7 # [2,10]\n", + "maximum_neurons_per_unit = 4 # [2,20]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "# Final training task\n", + "TIME = pendulum.now(tz='America/New_York').__str__()[:16]\\\n", + " .replace('T', '_')\\\n", + " .replace(':', '_')\\\n", + " .replace('-', '_')\n", + "#\n", + "PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test_cifar100_regnet'\n", + "#\n", + "meta_trial_number = 42\n", + "#\n", + "cerebros_automl = SimpleCerebrosRandomSearch(\n", + " unit_type=DenseUnit,\n", + " input_shapes=INPUT_SHAPES,\n", + " output_shapes=OUTPUT_SHAPES,\n", + " training_data=training_x,\n", + " labels=train_labels,\n", + " validation_split=0.2,\n", + " direction='maximize',\n", + " metric_to_rank_by=\"val_top_1_categorical_accuracy\",\n", + " minimum_levels=2,\n", + " maximum_levels=maximum_levels,\n", + " minimum_units_per_level=1,\n", + " maximum_units_per_level=maximum_units_per_level,\n", + " minimum_neurons_per_unit=1,\n", + " maximum_neurons_per_unit=maximum_neurons_per_unit,\n", + " activation=activation,\n", + " final_activation='softmax',\n", + " number_of_architecture_moities_to_try=3,\n", + " number_of_tries_per_architecture_moity=2,\n", + " minimum_skip_connection_depth=1,\n", + " maximum_skip_connection_depth=7,\n", + " predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first,\n", + " predecessor_level_connection_affinity_factor_first_rounding_rule='ceil',\n", + " predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main,\n", + " predecessor_level_connection_affinity_factor_main_rounding_rule='ceil',\n", + " predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay,\n", + " seed=8675309,\n", + " max_consecutive_lateral_connections=max_consecutive_lateral_connections,\n", + " gate_after_n_lateral_connections=3,\n", + " gate_activation_function=simple_sigmoid,\n", + " p_lateral_connection=p_lateral_connection,\n", + " p_lateral_connection_decay=zero_95_exp_decay,\n", + " num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit,\n", + " learning_rate=learning_rate,\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[tf.keras.metrics.TopKCategoricalAccuracy(\n", + " k=1, name='top_1_categorical_accuracy')\n", + " ],\n", + " epochs=epochs,\n", + " project_name=f\"{PROJECT_NAME}_meta_{meta_trial_number}\",\n", + " model_graphs='model_graphs',\n", + " batch_size=batch_size,\n", + " meta_trial_number=meta_trial_number,\n", + " base_models=[base_mod])" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'2': }, {'3': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'100': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'2': }\n", + "{'3': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'2': }\n", + "{'3': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'100': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 100), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[0][0]', \n", + " seUnit_0000000000000001_tr 'model_2[0][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[0][0]', \n", + " seUnit_0000000000000001_tr 'model_2[0][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7298 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 7376 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 184500 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8559945 (32.65 MB)\n", + "Trainable params: 1052769 (4.02 MB)\n", + "Non-trainable params: 7507176 (28.64 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 856s 1s/step - loss: 1.7874 - top_1_categorical_accuracy: 0.5453 - val_loss: 1.4817 - val_top_1_categorical_accuracy: 0.6127\n", + "Epoch 2/10\n", + "600/600 [==============================] - 1131s 2s/step - loss: 0.8883 - top_1_categorical_accuracy: 0.7395 - val_loss: 1.6004 - val_top_1_categorical_accuracy: 0.6190\n", + "Epoch 3/10\n", + "600/600 [==============================] - 1122s 2s/step - loss: 0.6110 - top_1_categorical_accuracy: 0.8154 - val_loss: 1.7295 - val_top_1_categorical_accuracy: 0.6233\n", + "Epoch 4/10\n", + "600/600 [==============================] - 1120s 2s/step - loss: 0.4026 - top_1_categorical_accuracy: 0.8727 - val_loss: 1.8371 - val_top_1_categorical_accuracy: 0.6243\n", + "Epoch 5/10\n", + "600/600 [==============================] - 931s 2s/step - loss: 0.3106 - top_1_categorical_accuracy: 0.9036 - val_loss: 1.8640 - val_top_1_categorical_accuracy: 0.6347\n", + "Epoch 6/10\n", + "600/600 [==============================] - 1046s 2s/step - loss: 0.2485 - top_1_categorical_accuracy: 0.9203 - val_loss: 2.0331 - val_top_1_categorical_accuracy: 0.6300\n", + "Epoch 7/10\n", + "600/600 [==============================] - 1095s 2s/step - loss: 0.2242 - top_1_categorical_accuracy: 0.9273 - val_loss: 2.2552 - val_top_1_categorical_accuracy: 0.6263\n", + "Epoch 8/10\n", + "600/600 [==============================] - 1091s 2s/step - loss: 0.1923 - top_1_categorical_accuracy: 0.9378 - val_loss: 2.2621 - val_top_1_categorical_accuracy: 0.6390\n", + "Epoch 9/10\n", + "600/600 [==============================] - 957s 2s/step - loss: 0.1927 - top_1_categorical_accuracy: 0.9368 - val_loss: 2.1990 - val_top_1_categorical_accuracy: 0.6363\n", + "Epoch 10/10\n", + "600/600 [==============================] - 1062s 2s/step - loss: 0.1739 - top_1_categorical_accuracy: 0.9411 - val_loss: 2.2908 - val_top_1_categorical_accuracy: 0.6417\n", + "this is neural_network_spec_file 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 1.787359 0.545333 1.481700 \n", + "1 0.888277 0.739500 1.600415 \n", + "2 0.610960 0.815417 1.729539 \n", + "3 0.402615 0.872667 1.837100 \n", + "4 0.310563 0.903583 1.864029 \n", + "5 0.248507 0.920333 2.033121 \n", + "6 0.224195 0.927250 2.255160 \n", + "7 0.192277 0.937833 2.262132 \n", + "8 0.192665 0.936833 2.199002 \n", + "9 0.173852 0.941083 2.290837 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.612667 0 0 \n", + "1 0.619000 0 0 \n", + "2 0.623333 0 0 \n", + "3 0.624333 0 0 \n", + "4 0.634667 0 0 \n", + "5 0.630000 0 0 \n", + "6 0.626333 0 0 \n", + "7 0.639000 0 0 \n", + "8 0.636333 0 0 \n", + "9 0.641667 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'2': }, {'3': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'100': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'2': }\n", + "{'3': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'2': }\n", + "{'3': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'100': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 100), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[1][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[1][0]', \n", + " seUnit_0000000000000001_tr 'model_2[1][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[1][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[1][0]', \n", + " seUnit_0000000000000001_tr 'model_2[1][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7298 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_2[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 7376 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 184500 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8559945 (32.65 MB)\n", + "Trainable params: 1052769 (4.02 MB)\n", + "Non-trainable params: 7507176 (28.64 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 1141s 2s/step - loss: 1.0151 - top_1_categorical_accuracy: 0.7005 - val_loss: 1.3601 - val_top_1_categorical_accuracy: 0.6613\n", + "Epoch 2/10\n", + "600/600 [==============================] - 905s 2s/step - loss: 0.2098 - top_1_categorical_accuracy: 0.9388 - val_loss: 1.5533 - val_top_1_categorical_accuracy: 0.6573\n", + "Epoch 3/10\n", + "600/600 [==============================] - 1025s 2s/step - loss: 0.1149 - top_1_categorical_accuracy: 0.9658 - val_loss: 1.9084 - val_top_1_categorical_accuracy: 0.6573\n", + "Epoch 4/10\n", + "600/600 [==============================] - 1095s 2s/step - loss: 0.1220 - top_1_categorical_accuracy: 0.9643 - val_loss: 1.9371 - val_top_1_categorical_accuracy: 0.6457\n", + "Epoch 5/10\n", + "600/600 [==============================] - 1111s 2s/step - loss: 0.1469 - top_1_categorical_accuracy: 0.9551 - val_loss: 2.1738 - val_top_1_categorical_accuracy: 0.6493\n", + "Epoch 6/10\n", + "600/600 [==============================] - 830s 1s/step - loss: 0.1460 - top_1_categorical_accuracy: 0.9530 - val_loss: 2.2794 - val_top_1_categorical_accuracy: 0.6453\n", + "Epoch 7/10\n", + "600/600 [==============================] - 971s 2s/step - loss: 0.1020 - top_1_categorical_accuracy: 0.9674 - val_loss: 2.3527 - val_top_1_categorical_accuracy: 0.6560\n", + "Epoch 8/10\n", + "600/600 [==============================] - 1080s 2s/step - loss: 0.1100 - top_1_categorical_accuracy: 0.9654 - val_loss: 2.6172 - val_top_1_categorical_accuracy: 0.6323\n", + "Epoch 9/10\n", + "600/600 [==============================] - 1068s 2s/step - loss: 0.1126 - top_1_categorical_accuracy: 0.9647 - val_loss: 2.6258 - val_top_1_categorical_accuracy: 0.6313\n", + "Epoch 10/10\n", + "600/600 [==============================] - 842s 1s/step - loss: 0.0914 - top_1_categorical_accuracy: 0.9723 - val_loss: 2.4642 - val_top_1_categorical_accuracy: 0.6543\n", + "this is neural_network_spec_file 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 1.015104 0.700533 1.360117 \n", + "1 0.209785 0.938833 1.553265 \n", + "2 0.114934 0.965833 1.908417 \n", + "3 0.122039 0.964250 1.937125 \n", + "4 0.146922 0.955083 2.173766 \n", + "5 0.145969 0.953000 2.279417 \n", + "6 0.101982 0.967417 2.352710 \n", + "7 0.109972 0.965417 2.617241 \n", + "8 0.112624 0.964750 2.625837 \n", + "9 0.091398 0.972333 2.464194 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.661333 0 1 \n", + "1 0.657333 0 1 \n", + "2 0.657333 0 1 \n", + "3 0.645667 0 1 \n", + "4 0.649333 0 1 \n", + "5 0.645333 0 1 \n", + "6 0.656000 0 1 \n", + "7 0.632333 0 1 \n", + "8 0.631333 0 1 \n", + "9 0.654333 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'4': }, {'4': }, {'4': }, {'2': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'2': }, {'1': }]\n", + "2\n", + "Trying to create level 2\n", + "We think level 2's predecessors are: [0, 1]\n", + "k is: 3 value is: [{'100': }]\n", + "3\n", + "Trying to create Final level 3\n", + "Trying to create level 3\n", + "We think level final level 3's predecessors are: [0, 1, 2]\n", + "levels:\n", + "[0, 1, 2, 3]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'4': }\n", + "{'4': }\n", + "{'4': }\n", + "{'2': }\n", + "{'2': }\n", + "{'1': }\n", + "Debug: I am 3 selecting 2\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2, 3]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2, 3]\n", + "Setting levels_unmaterialized[2] level_number 2 to have first successor: levels_unmaterialized[:3], having level_numbers of [3]\n", + "1\n", + "1\n", + "Debug: successor_connectivity_errors_2d []\n", + "Successor map:\n", + "{2: '0', 3: '1'}\n", + "I am: level #: 1 calling a correction to an error in:\n", + "error_0:[1 1]\n", + "I am: 3: My predecessors are [0, 1, 2]\n", + "asigning unit level 1, unit: 1 to be the input of: level: 3 unit: 0\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'4': }\n", + "{'4': }\n", + "{'4': }\n", + "{'2': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d [Array([1, 1], dtype=int32)]\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "{'2': }\n", + "{'1': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1 called\n", + "materialized network layers\n", + "[, , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , ]\n", + "{'100': }\n", + "debug: meta_level_number\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 100), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[2][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[2][0]', \n", + " seUnit_0000000000000001_tr 'model_2[2][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14596 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[2][0]', \n", + " 'model_2[2][0]', \n", + " 'model_2[2][0]', \n", + " 'model_2[2][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 14624 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14628 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[2][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[2][0]', \n", + " seUnit_0000000000000001_tr 'model_2[2][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[2][0]', \n", + " 'model_2[2][0]', \n", + " 'model_2[2][0]', \n", + " 'model_2[2][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 14624 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14596 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7314 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1848) 0 ['model_2[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_0_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_2[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1846) 0 ['model_2[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_1_cat_ (Concatenate) 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_2[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1848) 7392 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1846) 7384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 3698 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 1847 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1854) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000003_tr_ 00000000002_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000002_tr_0_0_dns_[0][0\n", + " 0000003_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1854) 7416 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000003_tr_ 0000000000000003_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000003_tr_0_0\n", + " 0000003_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 185500 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000003_tr_ 0000000000000003_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000003_tr_0_0\n", + " 0000003_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8643443 (32.97 MB)\n", + "Trainable params: 1114235 (4.25 MB)\n", + "Non-trainable params: 7529208 (28.72 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 1071s 2s/step - loss: 0.8272 - top_1_categorical_accuracy: 0.7514 - val_loss: 1.3232 - val_top_1_categorical_accuracy: 0.6680\n", + "Epoch 2/10\n", + "600/600 [==============================] - 1103s 2s/step - loss: 0.0907 - top_1_categorical_accuracy: 0.9814 - val_loss: 1.4265 - val_top_1_categorical_accuracy: 0.6900\n", + "Epoch 3/10\n", + "600/600 [==============================] - 1094s 2s/step - loss: 0.0514 - top_1_categorical_accuracy: 0.9877 - val_loss: 1.5660 - val_top_1_categorical_accuracy: 0.6747\n", + "Epoch 4/10\n", + "600/600 [==============================] - 889s 1s/step - loss: 0.0355 - top_1_categorical_accuracy: 0.9923 - val_loss: 1.7539 - val_top_1_categorical_accuracy: 0.6757\n", + "Epoch 5/10\n", + "600/600 [==============================] - 817s 1s/step - loss: 0.1014 - top_1_categorical_accuracy: 0.9702 - val_loss: 2.4363 - val_top_1_categorical_accuracy: 0.6223\n", + "Epoch 6/10\n", + "600/600 [==============================] - 1123s 2s/step - loss: 0.1437 - top_1_categorical_accuracy: 0.9556 - val_loss: 2.3636 - val_top_1_categorical_accuracy: 0.6363\n", + "Epoch 7/10\n", + "600/600 [==============================] - 1111s 2s/step - loss: 0.0868 - top_1_categorical_accuracy: 0.9736 - val_loss: 2.4318 - val_top_1_categorical_accuracy: 0.6543\n", + "Epoch 8/10\n", + "600/600 [==============================] - 680s 1s/step - loss: 0.0631 - top_1_categorical_accuracy: 0.9788 - val_loss: 2.4096 - val_top_1_categorical_accuracy: 0.6543\n", + "Epoch 9/10\n", + "600/600 [==============================] - 566s 944ms/step - loss: 0.0560 - top_1_categorical_accuracy: 0.9823 - val_loss: 2.6128 - val_top_1_categorical_accuracy: 0.6477\n", + "Epoch 10/10\n", + "600/600 [==============================] - 643s 1s/step - loss: 0.0862 - top_1_categorical_accuracy: 0.9728 - val_loss: 2.6630 - val_top_1_categorical_accuracy: 0.6473\n", + "this is neural_network_spec_file 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.827184 0.751400 1.323247 \n", + "1 0.090732 0.981417 1.426536 \n", + "2 0.051385 0.987750 1.566015 \n", + "3 0.035473 0.992333 1.753908 \n", + "4 0.101385 0.970167 2.436280 \n", + "5 0.143707 0.955583 2.363642 \n", + "6 0.086830 0.973583 2.431839 \n", + "7 0.063054 0.978833 2.409573 \n", + "8 0.056038 0.982333 2.612818 \n", + "9 0.086212 0.972833 2.663040 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.668000 0 0 \n", + "1 0.690000 0 0 \n", + "2 0.674667 0 0 \n", + "3 0.675667 0 0 \n", + "4 0.622333 0 0 \n", + "5 0.636333 0 0 \n", + "6 0.654333 0 0 \n", + "7 0.654333 0 0 \n", + "8 0.647667 0 0 \n", + "9 0.647333 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'4': }, {'4': }, {'4': }, {'2': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'2': }, {'1': }]\n", + "2\n", + "Trying to create level 2\n", + "We think level 2's predecessors are: [0, 1]\n", + "k is: 3 value is: [{'100': }]\n", + "3\n", + "Trying to create Final level 3\n", + "Trying to create level 3\n", + "We think level final level 3's predecessors are: [0, 1, 2]\n", + "levels:\n", + "[0, 1, 2, 3]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'4': }\n", + "{'4': }\n", + "{'4': }\n", + "{'2': }\n", + "{'2': }\n", + "{'1': }\n", + "Debug: I am 3 selecting 2\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2, 3]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2, 3]\n", + "Setting levels_unmaterialized[2] level_number 2 to have first successor: levels_unmaterialized[:3], having level_numbers of [3]\n", + "1\n", + "2\n", + "Debug: successor_connectivity_errors_2d []\n", + "Successor map:\n", + "{2: '0', 3: '1'}\n", + "I am: level #: 1 calling a correction to an error in:\n", + "error_0:[1 2]\n", + "I am: 3: My predecessors are [0, 1, 2]\n", + "asigning unit level 1, unit: 2 to be the input of: level: 3 unit: 0\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'4': }\n", + "{'4': }\n", + "{'4': }\n", + "{'2': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d [Array([1, 2], dtype=int32)]\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'2': }\n", + "{'1': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1 called\n", + "materialized network layers\n", + "[, , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , ]\n", + "{'100': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 100), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[3][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[3][0]', \n", + " seUnit_0000000000000001_tr 'model_2[3][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14596 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[3][0]', \n", + " 'model_2[3][0]', \n", + " 'model_2[3][0]', \n", + " 'model_2[3][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 14624 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14628 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[3][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[3][0]', \n", + " seUnit_0000000000000001_tr 'model_2[3][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[3][0]', \n", + " 'model_2[3][0]', \n", + " 'model_2[3][0]', \n", + " 'model_2[3][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3656) 14624 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14596 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7314 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 0 ['model_2[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_0_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'model_2[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1842) 0 ['model_2[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_1_cat_ (Concatenate) 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 7376 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1842) 7368 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 3690 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 1843 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1852) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000003_tr_ 00000000002_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000002_tr_0_0_dns_[0][0\n", + " 0000003_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_2[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1852) 7408 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000003_tr_ 0000000000000003_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000003_tr_0_0\n", + " 0000003_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 185300 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000003_tr_ 0000000000000003_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000003_tr_0_0\n", + " 0000003_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8643191 (32.97 MB)\n", + "Trainable params: 1114003 (4.25 MB)\n", + "Non-trainable params: 7529188 (28.72 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 666s 1s/step - loss: 0.7585 - top_1_categorical_accuracy: 0.7747 - val_loss: 1.2887 - val_top_1_categorical_accuracy: 0.6773\n", + "Epoch 2/10\n", + "600/600 [==============================] - 668s 1s/step - loss: 0.0587 - top_1_categorical_accuracy: 0.9899 - val_loss: 1.3380 - val_top_1_categorical_accuracy: 0.6950\n", + "Epoch 3/10\n", + "600/600 [==============================] - 679s 1s/step - loss: 0.0244 - top_1_categorical_accuracy: 0.9955 - val_loss: 1.4575 - val_top_1_categorical_accuracy: 0.6927\n", + "Epoch 4/10\n", + "600/600 [==============================] - 670s 1s/step - loss: 0.0279 - top_1_categorical_accuracy: 0.9950 - val_loss: 1.4996 - val_top_1_categorical_accuracy: 0.6920\n", + "Epoch 5/10\n", + "600/600 [==============================] - 657s 1s/step - loss: 0.0866 - top_1_categorical_accuracy: 0.9762 - val_loss: 2.1209 - val_top_1_categorical_accuracy: 0.6393\n", + "Epoch 6/10\n", + "600/600 [==============================] - 648s 1s/step - loss: 0.0942 - top_1_categorical_accuracy: 0.9718 - val_loss: 2.2454 - val_top_1_categorical_accuracy: 0.6493\n", + "Epoch 7/10\n", + "600/600 [==============================] - 665s 1s/step - loss: 0.0645 - top_1_categorical_accuracy: 0.9797 - val_loss: 2.4141 - val_top_1_categorical_accuracy: 0.6603\n", + "Epoch 8/10\n", + "600/600 [==============================] - 654s 1s/step - loss: 0.0683 - top_1_categorical_accuracy: 0.9793 - val_loss: 2.2461 - val_top_1_categorical_accuracy: 0.6593\n", + "Epoch 9/10\n", + "600/600 [==============================] - 651s 1s/step - loss: 0.0544 - top_1_categorical_accuracy: 0.9833 - val_loss: 2.5470 - val_top_1_categorical_accuracy: 0.6540\n", + "Epoch 10/10\n", + "600/600 [==============================] - 579s 965ms/step - loss: 0.0497 - top_1_categorical_accuracy: 0.9846 - val_loss: 2.5033 - val_top_1_categorical_accuracy: 0.6727\n", + "this is neural_network_spec_file 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.758510 0.774667 1.288737 \n", + "1 0.058713 0.989917 1.338045 \n", + "2 0.024366 0.995500 1.457486 \n", + "3 0.027870 0.995000 1.499606 \n", + "4 0.086609 0.976167 2.120878 \n", + "5 0.094161 0.971833 2.245409 \n", + "6 0.064475 0.979750 2.414118 \n", + "7 0.068332 0.979333 2.246143 \n", + "8 0.054424 0.983250 2.547007 \n", + "9 0.049730 0.984583 2.503266 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.677333 0 1 \n", + "1 0.695000 0 1 \n", + "2 0.692667 0 1 \n", + "3 0.692000 0 1 \n", + "4 0.639333 0 1 \n", + "5 0.649333 0 1 \n", + "6 0.660333 0 1 \n", + "7 0.659333 0 1 \n", + "8 0.654000 0 1 \n", + "9 0.672667 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'2': }, {'3': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'100': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'2': }\n", + "{'3': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'2': }\n", + "{'3': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'100': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 100), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[4][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[4][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[4][0]', \n", + " seUnit_0000000000000001_tr 'model_2[4][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[4][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[4][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[4][0]', \n", + " seUnit_0000000000000001_tr 'model_2[4][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7298 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_1_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_2[4][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_2[4][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1844) 7376 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 184500 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8559945 (32.65 MB)\n", + "Trainable params: 1052769 (4.02 MB)\n", + "Non-trainable params: 7507176 (28.64 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 644s 1s/step - loss: 0.7301 - top_1_categorical_accuracy: 0.7871 - val_loss: 1.2215 - val_top_1_categorical_accuracy: 0.6970\n", + "Epoch 2/10\n", + "600/600 [==============================] - 665s 1s/step - loss: 0.0464 - top_1_categorical_accuracy: 0.9937 - val_loss: 1.2956 - val_top_1_categorical_accuracy: 0.7037\n", + "Epoch 3/10\n", + "600/600 [==============================] - 650s 1s/step - loss: 0.0160 - top_1_categorical_accuracy: 0.9987 - val_loss: 1.3330 - val_top_1_categorical_accuracy: 0.7083\n", + "Epoch 4/10\n", + "600/600 [==============================] - 657s 1s/step - loss: 0.0228 - top_1_categorical_accuracy: 0.9948 - val_loss: 1.6587 - val_top_1_categorical_accuracy: 0.6790\n", + "Epoch 5/10\n", + "600/600 [==============================] - 655s 1s/step - loss: 0.0597 - top_1_categorical_accuracy: 0.9843 - val_loss: 1.9835 - val_top_1_categorical_accuracy: 0.6647\n", + "Epoch 6/10\n", + "600/600 [==============================] - 653s 1s/step - loss: 0.0678 - top_1_categorical_accuracy: 0.9775 - val_loss: 2.0322 - val_top_1_categorical_accuracy: 0.6733\n", + "Epoch 7/10\n", + "600/600 [==============================] - 651s 1s/step - loss: 0.0376 - top_1_categorical_accuracy: 0.9900 - val_loss: 2.2279 - val_top_1_categorical_accuracy: 0.6600\n", + "Epoch 8/10\n", + "600/600 [==============================] - 643s 1s/step - loss: 0.0489 - top_1_categorical_accuracy: 0.9863 - val_loss: 2.3728 - val_top_1_categorical_accuracy: 0.6657\n", + "Epoch 9/10\n", + "600/600 [==============================] - 649s 1s/step - loss: 0.0497 - top_1_categorical_accuracy: 0.9851 - val_loss: 2.3429 - val_top_1_categorical_accuracy: 0.6820\n", + "Epoch 10/10\n", + "600/600 [==============================] - 641s 1s/step - loss: 0.0505 - top_1_categorical_accuracy: 0.9839 - val_loss: 2.6628 - val_top_1_categorical_accuracy: 0.6513\n", + "this is neural_network_spec_file 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.730148 0.787067 1.221453 \n", + "1 0.046418 0.993667 1.295566 \n", + "2 0.016036 0.998667 1.332982 \n", + "3 0.022846 0.994833 1.658705 \n", + "4 0.059721 0.984250 1.983496 \n", + "5 0.067825 0.977500 2.032192 \n", + "6 0.037567 0.990000 2.227859 \n", + "7 0.048914 0.986333 2.372792 \n", + "8 0.049732 0.985083 2.342876 \n", + "9 0.050452 0.983917 2.662754 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.697000 0 0 \n", + "1 0.703667 0 0 \n", + "2 0.708333 0 0 \n", + "3 0.679000 0 0 \n", + "4 0.664667 0 0 \n", + "5 0.673333 0 0 \n", + "6 0.660000 0 0 \n", + "7 0.665667 0 0 \n", + "8 0.682000 0 0 \n", + "9 0.651333 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'2': }, {'3': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'100': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'2': }\n", + "{'3': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'2': }\n", + "{'3': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'100': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 100), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[5][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[5][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[5][0]', \n", + " seUnit_0000000000000001_tr 'model_2[5][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[5][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[5][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[5][0]', \n", + " seUnit_0000000000000001_tr 'model_2[5][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7298 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1843) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_2[5][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_2[5][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1843) 7372 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 184400 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8559841 (32.65 MB)\n", + "Trainable params: 1052667 (4.02 MB)\n", + "Non-trainable params: 7507174 (28.64 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 543s 892ms/step - loss: 0.6854 - top_1_categorical_accuracy: 0.7959 - val_loss: 1.1961 - val_top_1_categorical_accuracy: 0.6943\n", + "Epoch 2/10\n", + "600/600 [==============================] - 643s 1s/step - loss: 0.0339 - top_1_categorical_accuracy: 0.9976 - val_loss: 1.2492 - val_top_1_categorical_accuracy: 0.7097\n", + "Epoch 3/10\n", + "600/600 [==============================] - 665s 1s/step - loss: 0.0137 - top_1_categorical_accuracy: 0.9992 - val_loss: 1.2785 - val_top_1_categorical_accuracy: 0.7203\n", + "Epoch 4/10\n", + "600/600 [==============================] - 674s 1s/step - loss: 0.0138 - top_1_categorical_accuracy: 0.9977 - val_loss: 1.4829 - val_top_1_categorical_accuracy: 0.7017\n", + "Epoch 5/10\n", + "600/600 [==============================] - 659s 1s/step - loss: 0.0375 - top_1_categorical_accuracy: 0.9902 - val_loss: 1.9024 - val_top_1_categorical_accuracy: 0.6473\n", + "Epoch 6/10\n", + "600/600 [==============================] - 666s 1s/step - loss: 0.0613 - top_1_categorical_accuracy: 0.9817 - val_loss: 2.0893 - val_top_1_categorical_accuracy: 0.6603\n", + "Epoch 7/10\n", + "600/600 [==============================] - 668s 1s/step - loss: 0.0582 - top_1_categorical_accuracy: 0.9840 - val_loss: 2.2556 - val_top_1_categorical_accuracy: 0.6613\n", + "Epoch 8/10\n", + "600/600 [==============================] - 685s 1s/step - loss: 0.0406 - top_1_categorical_accuracy: 0.9877 - val_loss: 2.3167 - val_top_1_categorical_accuracy: 0.6727\n", + "Epoch 9/10\n", + "600/600 [==============================] - 659s 1s/step - loss: 0.0390 - top_1_categorical_accuracy: 0.9879 - val_loss: 2.5246 - val_top_1_categorical_accuracy: 0.6480\n", + "Epoch 10/10\n", + "600/600 [==============================] - 669s 1s/step - loss: 0.0339 - top_1_categorical_accuracy: 0.9898 - val_loss: 2.5872 - val_top_1_categorical_accuracy: 0.6597\n", + "this is neural_network_spec_file 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.685352 0.795867 1.196145 \n", + "1 0.033869 0.997583 1.249168 \n", + "2 0.013692 0.999167 1.278499 \n", + "3 0.013782 0.997667 1.482887 \n", + "4 0.037496 0.990167 1.902371 \n", + "5 0.061280 0.981667 2.089307 \n", + "6 0.058171 0.984000 2.255635 \n", + "7 0.040618 0.987667 2.316699 \n", + "8 0.038963 0.987917 2.524557 \n", + "9 0.033940 0.989750 2.587205 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.694333 0 1 \n", + "1 0.709667 0 1 \n", + "2 0.720333 0 1 \n", + "3 0.701667 0 1 \n", + "4 0.647333 0 1 \n", + "5 0.660333 0 1 \n", + "6 0.661333 0 1 \n", + "7 0.672667 0 1 \n", + "8 0.648000 0 1 \n", + "9 0.659667 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_22_12_36_cerebros_auto_ml_test_cifar10... \n", + "Index(['loss', 'top_1_categorical_accuracy', 'val_loss',\n", + " 'val_top_1_categorical_accuracy', 'trial_number', 'subtrial_number',\n", + " 'model_name'],\n", + " dtype='object')\n", + "metric_to_rank_by is: 'val_top_1_categorical_accuracy'\n", + "Type of metric_to_rank_by is: \n", + "Best result this trial was: 0.7203333377838135\n", + "Type of best result: \n", + "Best model name: 2023_10_22_12_36_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001\n", + "CPU times: user 1d 3h 29min 31s, sys: 3h 41min 44s, total: 1d 7h 11min 16s\n", + "Wall time: 13h 47min 30s\n" + ] + } + ], + "source": [ + "%%time\n", + "result = cerebros_automl.run_random_search()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best accuracy achieved is 0.7203333377838135\n", + "top-1 categorical accuracy\n" + ] + } + ], + "source": [ + "print(f'Best accuracy achieved is {result}')\n", + "print(f'top-1 categorical accuracy')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evaluating the best model found" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "best_model_found = cerebros_automl.get_best_model()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "#\n", + "eval_loss = tf.keras.losses.CategoricalCrossentropy()\n", + "#\n", + "eval_metrics =\\\n", + "[tf.keras.metrics.TopKCategoricalAccuracy(k=1,\\\n", + " name='eval_top_1_categorical_accuracy'),\n", + " tf.keras.metrics.TopKCategoricalAccuracy(k=5,\\\n", + " name='eval_top_5_categorical_accuracy')\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_2 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[0][0]', \n", + " seUnit_0000000000000001_tr 'model_2[0][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_2[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_2[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_2[0][0]', \n", + " seUnit_0000000000000001_tr 'model_2[0][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7298 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1843) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_2[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_2[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1843) 7372 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 100) 184400 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8559841 (32.65 MB)\n", + "Trainable params: 1052667 (4.02 MB)\n", + "Non-trainable params: 7507174 (28.64 MB)\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "best_model_found.compile(loss=eval_loss, metrics=eval_metrics)\n", + "best_model_found.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluating best model found ...\n", + "Loss | Top-1 accuracy | Top-5 accuracy\n", + "313/313 [==============================] - 366s 1s/step - loss: 2.6348 - eval_top_1_categorical_accuracy: 0.6492 - eval_top_5_categorical_accuracy: 0.8770\n" + ] + }, + { + "data": { + "text/plain": [ + "[2.63478422164917, 0.6492000222206116, 0.8769999742507935]" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Evaluating best model found ...\")\n", + "print(\"Loss | Top-1 accuracy | Top-5 accuracy\")\n", + "y_test_cat = to_categorical(y_test, 100)\n", + "best_model_found.evaluate(X_test, y_test_cat)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "kubeflow_notebook": { + "autosnapshot": true, + "deploy_config": {}, + "docker_image": "", + "experiment_name": "image-classification-b", + "katib_metadata": { + "algorithm": { + "algorithmName": "bayesianoptimization", + "algorithmSettings": [ + { + "name": "random_state", + "value": "10" + }, + { + "name": "acq_optimizer", + "value": "auto" + }, + { + "name": "acq_func", + "value": "gp_hedge" + }, + { + "name": "base_estimator", + "value": "GP" + } + ] + }, + "maxFailedTrialCount": 10, + "maxTrialCount": 40, + "objective": { + "additionalMetricNames": [], + "goal": 1, + "objectiveMetricName": "val-top-1-categorical-accuracy", + "type": "maximize" + }, + "parallelTrialCount": 3, + "parameters": [ + { + "feasibleSpace": { + "list": [ + "relu", + "elu", + "gelu" + ] + }, + "name": "activation", + "parameterType": "categorical" + }, + { + "feasibleSpace": { + "max": "40", + "min": "1", + "step": "0.1" + }, + "name": "predecessor_level_connection_affinity_factor_first", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "65", + "min": "0.1", + "step": "0.1" + }, + "name": "predecessor_level_connection_affinity_factor_main", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "40", + "min": "2", + "step": "1" + }, + "name": "max_consecutive_lateral_connections", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "50", + "min": "0.1", + "step": "0.1" + }, + "name": "p_lateral_connection", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "40", + "min": "1", + "step": "1" + }, + "name": "num_lateral_connection_tries_per_unit", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "0.3", + "min": "0.00001", + "step": "0.00001" + }, + "name": "learning_rate", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "7", + "min": "1", + "step": "1" + }, + "name": "epochs", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "40", + "min": "5", + "step": "1" + }, + "name": "batch_size", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "8", + "min": "2" + }, + "name": "maximum_levels", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "10", + "min": "2", + "step": "1" + }, + "name": "maximum_units_per_level", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "10", + "min": "2", + "step": "1" + }, + "name": "maximum_neurons_per_unit", + "parameterType": "int" + } + ] + }, + "katib_run": true, + "pipeline_description": "Cifar 10 example", + "pipeline_name": "image-classification-b", + "snapshot_volumes": true, + "volume_access_mode": "rwm", + "volumes": [ + { + "annotations": [], + "mount_point": "/home/jovyan", + "name": "test-images-workspace-pmbn9", + "size": 5, + "size_type": "Gi", + "snapshot": false, + "type": "clone" + } + ] + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cifar_100_RegNet.py b/cifar_100_RegNet.py new file mode 100644 index 0000000..1c47f3c --- /dev/null +++ b/cifar_100_RegNet.py @@ -0,0 +1,207 @@ +# Initializing + +import sys +sys.path.insert(0, '../..') + +import tensorflow as tf +import tensorflow_datasets as tfds +from tensorflow.keras.datasets import cifar10, cifar100 +from tensorflow.keras.utils import to_categorical +from tensorflow.keras.models import Model +from tensorflow.keras.layers import Input, Resizing, Lambda, Flatten, Dense +import pandas as pd +import numpy as np +from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\ + import SimpleCerebrosRandomSearch +import pendulum +from cerebros.units.units import DenseUnit +from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\ + import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid +from ast import literal_eval + +# Download RegNetX016 with Imagenet weights (1000 classes) + +regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016', + include_top=True, + include_preprocessing=True, + weights='imagenet', + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000, + classifier_activation='softmax' + ) + +regnet.summary() + +# Make all layers untrainable except for the very last convolutional layer + +for layer in regnet.layers: + layer.trainable = False +regnet.layers[-6].trainable = True + +# Cifar-100 testing + +(X_train, y_train), (X_test, y_test) = cifar100.load_data() + +y_train_cat = to_categorical(y_train, 1000) +y_test_cat = to_categorical(y_test, 1000) + +# Lambda layer for preprocessing + +def resize(x): + return tf.image.resize(x,size=(224,224),method='bicubic') + +# Modify the model + +input_shape = (32,32,3) + +input_layer = Input(shape=input_shape) +prep = Lambda(resize)(input_layer) +out = regnet(prep) +regnet_mod = Model(inputs=input_layer, outputs=out) + +regnet_mod.compile(optimizer='adam', + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=[tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_categorical_accuracy')]) + +# Try to fit it on Cifar-100 data and then evaluate (there is no hope this is gonna work ...) + +regnet_mod.fit(X_train, y_train_cat) + +regnet_mod.evaluate(X_test, y_test_cat) + +# Try the same with adding a Cerebros "add-on" network + +INPUT_SHAPES = [input_shape] +OUTPUT_SHAPES = [100] + +# Use 10k-15k random samples from Cifar-100 to speed up the process + +num_samples = 15_000 +rng = np.random.default_rng() +ind = rng.permutation(X_train.shape[0])[:num_samples] + +training_x = [tf.constant(X_train[ind,:,:,:])] +y_train_cat = to_categorical(y_train[ind], 100) +train_labels = [tf.constant(y_train_cat)] + +regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016', + include_top=True, + include_preprocessing=True, + weights='imagenet', + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000, + classifier_activation='softmax' + ) + +for layer in regnet.layers: + layer.trainable = False +regnet.layers[-6].trainable = True + +regnet_io = Model(inputs=regnet.layers[0].input, + outputs=regnet.layers[-2].output) + +input_layer = Input(shape=input_shape) +prep = Lambda(resize)(input_layer) +out = Flatten()(regnet_io(prep)) +base_mod = Model(inputs=input_layer, outputs=out) + +activation = 'relu' +predecessor_level_connection_affinity_factor_first = 2.0 +predecessor_level_connection_affinity_factor_main = 0.97 +max_consecutive_lateral_connections = 5 +p_lateral_connection = 0.97 +num_lateral_connection_tries_per_unit = 2 +learning_rate = 0.001 +epochs = 10 # [1, 100] +batch_size = 20 +maximum_levels = 4 # [3,7] +maximum_units_per_level = 7 # [2,10] +maximum_neurons_per_unit = 4 # [2,20] + +# Final training task +TIME = pendulum.now(tz='America/New_York').__str__()[:16]\ + .replace('T', '_')\ + .replace(':', '_')\ + .replace('-', '_') +# +PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test_cifar100_regnet' +# +meta_trial_number = 42 +# +cerebros_automl = SimpleCerebrosRandomSearch( + unit_type=DenseUnit, + input_shapes=INPUT_SHAPES, + output_shapes=OUTPUT_SHAPES, + training_data=training_x, + labels=train_labels, + validation_split=0.2, + direction='maximize', + metric_to_rank_by="val_top_1_categorical_accuracy", + minimum_levels=2, + maximum_levels=maximum_levels, + minimum_units_per_level=1, + maximum_units_per_level=maximum_units_per_level, + minimum_neurons_per_unit=1, + maximum_neurons_per_unit=maximum_neurons_per_unit, + activation=activation, + final_activation='softmax', + number_of_architecture_moities_to_try=3, + number_of_tries_per_architecture_moity=2, + minimum_skip_connection_depth=1, + maximum_skip_connection_depth=7, + predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first, + predecessor_level_connection_affinity_factor_first_rounding_rule='ceil', + predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main, + predecessor_level_connection_affinity_factor_main_rounding_rule='ceil', + predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay, + seed=8675309, + max_consecutive_lateral_connections=max_consecutive_lateral_connections, + gate_after_n_lateral_connections=3, + gate_activation_function=simple_sigmoid, + p_lateral_connection=p_lateral_connection, + p_lateral_connection_decay=zero_95_exp_decay, + num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit, + learning_rate=learning_rate, + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=[tf.keras.metrics.TopKCategoricalAccuracy( + k=1, name='top_1_categorical_accuracy') + ], + epochs=epochs, + project_name=f"{PROJECT_NAME}_meta_{meta_trial_number}", + model_graphs='model_graphs', + batch_size=batch_size, + meta_trial_number=meta_trial_number, + base_models=[base_mod]) + +# Commented out IPython magic to ensure Python compatibility. +#%%time +result = cerebros_automl.run_random_search() + +print(f'Best accuracy achieved is {result}') +print(f'top-1 categorical accuracy') + +# Evaluating the best model found + +best_model_found = cerebros_automl.get_best_model() + +# +eval_loss = tf.keras.losses.CategoricalCrossentropy() +# +eval_metrics =\ +[tf.keras.metrics.TopKCategoricalAccuracy(k=1,\ + name='eval_top_1_categorical_accuracy'), + tf.keras.metrics.TopKCategoricalAccuracy(k=5,\ + name='eval_top_5_categorical_accuracy') +] + +best_model_found.compile(loss=eval_loss, metrics=eval_metrics) +best_model_found.summary() + +print("Evaluating best model found ...") +print("Loss | Top-1 accuracy | Top-5 accuracy") +y_test_cat = to_categorical(y_test, 100) +best_model_found.evaluate(X_test, y_test_cat) \ No newline at end of file From 17bb6a5eff07cab0d22091156cb84f5d506915c5 Mon Sep 17 00:00:00 2001 From: Sasha Kolpakov Date: Tue, 31 Oct 2023 21:42:46 +0100 Subject: [PATCH 2/3] Update cifar_100_RegNet.py --- cifar_100_RegNet.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cifar_100_RegNet.py b/cifar_100_RegNet.py index 1c47f3c..572a4e8 100644 --- a/cifar_100_RegNet.py +++ b/cifar_100_RegNet.py @@ -1,8 +1,5 @@ # Initializing -import sys -sys.path.insert(0, '../..') - import tensorflow as tf import tensorflow_datasets as tfds from tensorflow.keras.datasets import cifar10, cifar100 @@ -204,4 +201,4 @@ def resize(x): print("Evaluating best model found ...") print("Loss | Top-1 accuracy | Top-5 accuracy") y_test_cat = to_categorical(y_test, 100) -best_model_found.evaluate(X_test, y_test_cat) \ No newline at end of file +best_model_found.evaluate(X_test, y_test_cat) From 3d95fbfff816e8fd0a1f6df977719e9e3af50bce Mon Sep 17 00:00:00 2001 From: Sasha Kolpakov Date: Tue, 31 Oct 2023 21:43:11 +0100 Subject: [PATCH 3/3] Update cifar_100_RegNet.ipynb --- cifar_100_RegNet.ipynb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cifar_100_RegNet.ipynb b/cifar_100_RegNet.ipynb index 0db37f2..bf715f6 100644 --- a/cifar_100_RegNet.ipynb +++ b/cifar_100_RegNet.ipynb @@ -9,16 +9,6 @@ "# Cifar-100 test for image classification" ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.insert(0, '../..')" - ] - }, { "cell_type": "code", "execution_count": 2,