init commit
This commit is contained in:
commit
d5d633b6c7
517 changed files with 18824 additions and 0 deletions
54
README.md
Normal file
54
README.md
Normal file
|
@ -0,0 +1,54 @@
|
|||
# VisRecall: Quantifying Information Visualisation Recallability via Question Answering
|
||||
|
||||
*Yao Wang, Chuhan Jiao(Aalto University), Mihai Bâce and Andreas Bulling*
|
||||
|
||||
submitted to The IEEE Transactions on Visualization and Computer Graphics (TVCG2022)
|
||||
|
||||
This repository contains the dataset and models for predicting visualisation recallability.
|
||||
|
||||
```
|
||||
$Root Directory
|
||||
│
|
||||
│─ README.md —— this file
|
||||
│
|
||||
|─ RecallNet —— Source code of the network to predict infovis recallability
|
||||
│ │
|
||||
│ │─ environment.yaml —— conda environments
|
||||
│ │
|
||||
│ │─ notebooks
|
||||
│ │ │
|
||||
│ │ │─ train_RecallNet.ipynb —— main notebook for training and validation
|
||||
│ │ │
|
||||
│ │ └─ massvis_recall.json —— saved recallability scores for MASSVIS dataset
|
||||
│ │
|
||||
│ └─ src
|
||||
│ │
|
||||
│ │─ singleduration_models.py —— RecallNet model
|
||||
│ │
|
||||
│ │─ sal_imp_utilities.py —— image processing utilities
|
||||
│ │
|
||||
│ │─ losses_keras2.py —— loss functions
|
||||
│ │
|
||||
│ ...
|
||||
│
|
||||
└─ VisRecall —— The dataset
|
||||
│
|
||||
│─ answer_raw —— raw answers from AMT workers
|
||||
│
|
||||
│─ merged
|
||||
│ │
|
||||
│ │─ src —— original images
|
||||
│ │
|
||||
│ │─ qa —— question annotations
|
||||
│ │
|
||||
│ └─ image_annotation —— other metadata annotations
|
||||
│
|
||||
└─ training_data
|
||||
│
|
||||
│─ all —— all averaged questions
|
||||
│
|
||||
└─ X-question —— a specific type of question (T-, FE-, F-, RV-, U-)
|
||||
```
|
||||
|
||||
|
||||
contact: yao.wang@vis.uni-stuttgart.de
|
160
RecallNet/environment.yaml
Normal file
160
RecallNet/environment.yaml
Normal file
|
@ -0,0 +1,160 @@
|
|||
name: tf-cuda9
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- _libgcc_mutex=0.1=main
|
||||
- _tflow_select=2.1.0=gpu
|
||||
- absl-py=0.11.0=py37h06a4308_0
|
||||
- argon2-cffi=20.1.0=py37h8f50634_2
|
||||
- astor=0.8.1=py37_0
|
||||
- async_generator=1.10=py_0
|
||||
- attrs=20.2.0=pyh9f0ad1d_0
|
||||
- backports=1.0=py_2
|
||||
- backports.functools_lru_cache=1.6.1=py_0
|
||||
- blas=1.0=mkl
|
||||
- bleach=3.2.1=pyh9f0ad1d_0
|
||||
- c-ares=1.16.1=h7b6447c_0
|
||||
- ca-certificates=2021.4.13=h06a4308_1
|
||||
- certifi=2020.12.5=py37h06a4308_0
|
||||
- cffi=1.14.3=py37he30daa8_0
|
||||
- cudatoolkit=9.0=h13b8566_0
|
||||
- cudnn=7.3.1=cuda9.0_0
|
||||
- cupti=9.0.176=0
|
||||
- decorator=4.4.2=py_0
|
||||
- defusedxml=0.6.0=py_0
|
||||
- entrypoints=0.3=py37hc8dfbb8_1002
|
||||
- gast=0.4.0=py_0
|
||||
- google-pasta=0.2.0=py_0
|
||||
- grpcio=1.31.0=py37hf8bcb03_0
|
||||
- h5py=2.10.0=py37hd6299e0_1
|
||||
- hdf5=1.10.6=hb1b8bf9_0
|
||||
- importlib-metadata=2.0.0=py_1
|
||||
- importlib_metadata=2.0.0=1
|
||||
- intel-openmp=2020.2=254
|
||||
- ipykernel=5.3.4=py37hc6149b9_1
|
||||
- ipython=5.8.0=py37_1
|
||||
- ipython_genutils=0.2.0=py_1
|
||||
- ipywidgets=7.5.1=pyh9f0ad1d_1
|
||||
- jinja2=2.11.2=pyh9f0ad1d_0
|
||||
- jsonschema=3.2.0=py_2
|
||||
- jupyter_client=6.1.7=py_0
|
||||
- jupyter_core=4.6.3=py37hc8dfbb8_2
|
||||
- jupyterlab_pygments=0.1.2=pyh9f0ad1d_0
|
||||
- keras=2.3.1=0
|
||||
- keras-applications=1.0.8=py_1
|
||||
- keras-base=2.3.1=py37_0
|
||||
- keras-preprocessing=1.1.0=py_1
|
||||
- ld_impl_linux-64=2.33.1=h53a641e_7
|
||||
- libedit=3.1.20191231=h14c3975_1
|
||||
- libffi=3.3=he6710b0_2
|
||||
- libgcc-ng=9.1.0=hdf63c60_0
|
||||
- libgfortran-ng=7.3.0=hdf63c60_0
|
||||
- libprotobuf=3.13.0.1=hd408876_0
|
||||
- libsodium=1.0.18=h516909a_1
|
||||
- libstdcxx-ng=9.1.0=hdf63c60_0
|
||||
- markdown=3.3.2=py37_0
|
||||
- markupsafe=1.1.1=py37hb5d75c8_2
|
||||
- mistune=0.8.4=py37h8f50634_1002
|
||||
- mkl=2020.2=256
|
||||
- mkl-service=2.3.0=py37he904b0f_0
|
||||
- mkl_fft=1.2.0=py37h23d657b_0
|
||||
- mkl_random=1.1.1=py37h0573a6f_0
|
||||
- nbclient=0.5.1=py_0
|
||||
- nbconvert=6.0.7=py37hc8dfbb8_2
|
||||
- nbformat=5.0.8=py_0
|
||||
- ncurses=6.2=he6710b0_1
|
||||
- nest-asyncio=1.4.1=py_0
|
||||
- notebook=6.1.4=py37hc8dfbb8_1
|
||||
- numpy=1.19.2=py37h54aff64_0
|
||||
- numpy-base=1.19.2=py37hfa32c7d_0
|
||||
- openssl=1.1.1k=h27cfd23_0
|
||||
- packaging=20.4=pyh9f0ad1d_0
|
||||
- pandoc=2.11.0.4=hd18ef5c_0
|
||||
- pandocfilters=1.4.2=py_1
|
||||
- pexpect=4.8.0=pyh9f0ad1d_2
|
||||
- pickleshare=0.7.5=py_1003
|
||||
- pip=20.2.4=py37_0
|
||||
- prometheus_client=0.8.0=pyh9f0ad1d_0
|
||||
- prompt_toolkit=1.0.15=py_1
|
||||
- protobuf=3.13.0.1=py37he6710b0_1
|
||||
- ptyprocess=0.6.0=py_1001
|
||||
- pycparser=2.20=pyh9f0ad1d_2
|
||||
- pygments=2.7.2=py_0
|
||||
- pyparsing=2.4.7=pyh9f0ad1d_0
|
||||
- pyrsistent=0.17.3=py37h8f50634_1
|
||||
- python=3.7.9=h7579374_0
|
||||
- python-dateutil=2.8.1=py_0
|
||||
- python_abi=3.7=1_cp37m
|
||||
- pyyaml=5.3.1=py37h7b6447c_1
|
||||
- pyzmq=19.0.2=py37hac76be4_2
|
||||
- readline=8.0=h7b6447c_0
|
||||
- send2trash=1.5.0=py_0
|
||||
- setuptools=50.3.0=py37hb0f4dca_1
|
||||
- simplegeneric=0.8.1=py_1
|
||||
- six=1.15.0=py_0
|
||||
- sqlite=3.33.0=h62c20be_0
|
||||
- tensorboard=1.14.0=py37hf484d3e_0
|
||||
- tensorflow=1.14.0=gpu_py37hae64822_0
|
||||
- tensorflow-base=1.14.0=gpu_py37h8f37b9b_0
|
||||
- tensorflow-estimator=1.14.0=py_0
|
||||
- tensorflow-gpu=1.14.0=h0d30ee6_0
|
||||
- termcolor=1.1.0=py37_1
|
||||
- terminado=0.9.1=py37hc8dfbb8_1
|
||||
- testpath=0.4.4=py_0
|
||||
- tk=8.6.10=hbc83047_0
|
||||
- tornado=6.0.4=py37h8f50634_2
|
||||
- traitlets=5.0.5=py_0
|
||||
- wcwidth=0.2.5=pyh9f0ad1d_2
|
||||
- webencodings=0.5.1=py_1
|
||||
- werkzeug=1.0.1=py_0
|
||||
- wheel=0.35.1=py_0
|
||||
- widgetsnbextension=3.5.1=py37hc8dfbb8_4
|
||||
- wrapt=1.12.1=py37h7b6447c_1
|
||||
- xz=5.2.5=h7b6447c_0
|
||||
- yaml=0.2.5=h7b6447c_0
|
||||
- zeromq=4.3.3=he1b5a44_2
|
||||
- zipp=3.4.0=pyhd3eb1b0_0
|
||||
- zlib=1.2.11=h7b6447c_3
|
||||
- pip:
|
||||
- adjusttext==0.7.3
|
||||
- alignment==0.0.1
|
||||
- chardet==4.0.0
|
||||
- cmasher==1.6.2
|
||||
- colorspacious==1.1.2
|
||||
- cycler==0.10.0
|
||||
- e13tools==0.9.6
|
||||
- editdistance==0.5.3
|
||||
- fastdtw==0.3.4
|
||||
- idna==2.10
|
||||
- imageio==2.9.0
|
||||
- joblib==1.0.0
|
||||
- kiwisolver==1.2.0
|
||||
- mat4py==0.5.0
|
||||
- matplotlib==3.4.3
|
||||
- natsort==7.1.1
|
||||
- networkx==2.5
|
||||
- opencv-python==4.4.0.44
|
||||
- pandas==1.2.1
|
||||
- patsy==0.5.1
|
||||
- pillow==8.0.1
|
||||
- pytz==2021.1
|
||||
- pywavelets==1.1.1
|
||||
- requests==2.25.1
|
||||
- researchpy==0.3.2
|
||||
- salicon==1.0
|
||||
- scikit-image==0.18.1
|
||||
- scikit-learn==0.24.1
|
||||
- scipy==1.2.0
|
||||
- seaborn==0.11.1
|
||||
- simplejson==3.17.5
|
||||
- sklearn==0.0
|
||||
- spotlight==2.3.1
|
||||
- statsmodels==0.12.2
|
||||
- threadpoolctl==2.1.0
|
||||
- tifffile==2021.3.17
|
||||
- tqdm==4.51.0
|
||||
- urllib3==1.26.4
|
||||
- wget==3.2
|
||||
prefix: /netpool/homes/wangyo/.conda/envs/tf-cuda9
|
||||
|
File diff suppressed because one or more lines are too long
1747
RecallNet/notebooks/massvis_recall.json
Normal file
1747
RecallNet/notebooks/massvis_recall.json
Normal file
File diff suppressed because it is too large
Load diff
984
RecallNet/notebooks/train_RecallNet.ipynb
Normal file
984
RecallNet/notebooks/train_RecallNet.ipynb
Normal file
File diff suppressed because one or more lines are too long
183
RecallNet/src/.ipynb_checkpoints/Untitled-checkpoint.ipynb
Normal file
183
RecallNet/src/.ipynb_checkpoints/Untitled-checkpoint.ipynb
Normal file
|
@ -0,0 +1,183 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Using TensorFlow backend.\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
|
||||
"/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
|
||||
" np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from xception_custom import Xception_wrapper"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from keras.layers import Input"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[[[155 225 83]\n",
|
||||
" [174 33 86]\n",
|
||||
" [ 24 223 10]\n",
|
||||
" ...\n",
|
||||
" [147 233 79]\n",
|
||||
" [232 187 173]\n",
|
||||
" [ 69 126 85]]\n",
|
||||
"\n",
|
||||
" [[166 203 47]\n",
|
||||
" [111 65 37]\n",
|
||||
" [210 182 244]\n",
|
||||
" ...\n",
|
||||
" [154 62 70]\n",
|
||||
" [ 62 93 101]\n",
|
||||
" [132 231 126]]\n",
|
||||
"\n",
|
||||
" [[ 30 110 125]\n",
|
||||
" [242 45 71]\n",
|
||||
" [150 10 217]\n",
|
||||
" ...\n",
|
||||
" [ 38 165 128]\n",
|
||||
" [ 64 58 127]\n",
|
||||
" [179 174 72]]\n",
|
||||
"\n",
|
||||
" ...\n",
|
||||
"\n",
|
||||
" [[159 2 99]\n",
|
||||
" [201 220 158]\n",
|
||||
" [170 172 13]\n",
|
||||
" ...\n",
|
||||
" [ 79 72 65]\n",
|
||||
" [ 10 228 7]\n",
|
||||
" [ 99 60 129]]\n",
|
||||
"\n",
|
||||
" [[187 249 6]\n",
|
||||
" [ 57 166 83]\n",
|
||||
" [187 243 66]\n",
|
||||
" ...\n",
|
||||
" [109 184 147]\n",
|
||||
" [142 158 83]\n",
|
||||
" [190 61 30]]\n",
|
||||
"\n",
|
||||
" [[146 238 74]\n",
|
||||
" [156 20 43]\n",
|
||||
" [ 55 217 43]\n",
|
||||
" ...\n",
|
||||
" [208 181 141]\n",
|
||||
" [196 88 15]\n",
|
||||
" [132 225 63]]]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "TypeError",
|
||||
"evalue": "Error converting shape to a TensorShape: only size-1 arrays can be converted to Python scalars.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mmake_shape\u001b[0;34m(v, arg_name)\u001b[0m\n\u001b[1;32m 145\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensor_shape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 147\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/tensor_shape.py\u001b[0m in \u001b[0;36mas_shape\u001b[0;34m(shape)\u001b[0m\n\u001b[1;32m 1203\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1204\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mTensorShape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/tensor_shape.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dims)\u001b[0m\n\u001b[1;32m 773\u001b[0m \u001b[0;31m# Got a list of dimensions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 774\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dims\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mas_dimension\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdims_iter\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 775\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/tensor_shape.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 773\u001b[0m \u001b[0;31m# Got a list of dimensions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 774\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dims\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mas_dimension\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdims_iter\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 775\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/tensor_shape.py\u001b[0m in \u001b[0;36mas_dimension\u001b[0;34m(value)\u001b[0m\n\u001b[1;32m 715\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 716\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mDimension\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 717\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/framework/tensor_shape.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 186\u001b[0m if (not isinstance(value, compat.bytes_or_text_types) and\n",
|
||||
"\u001b[0;31mTypeError\u001b[0m: only size-1 arrays can be converted to Python scalars",
|
||||
"\nDuring handling of the above exception, another exception occurred:\n",
|
||||
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m<ipython-input-8-8f06c869009e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mxception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mXception_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minclude_top\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'imagenet'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_tensor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpooling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'xception:'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mtest_xception_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
|
||||
"\u001b[0;32m<ipython-input-8-8f06c869009e>\u001b[0m in \u001b[0;36mtest_xception_shape\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0minput_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m240\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m320\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0minp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0mxception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mXception_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minclude_top\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'imagenet'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_tensor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpooling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'xception:'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/keras/engine/input_layer.py\u001b[0m in \u001b[0;36mInput\u001b[0;34m(shape, batch_shape, name, dtype, sparse, tensor)\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[0msparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m input_tensor=tensor)\n\u001b[0m\u001b[1;32m 179\u001b[0m \u001b[0;31m# Return tensor including _keras_shape and _keras_history.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[0;31m# Note that in this case train_output and test_output are the same pointer.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m 90\u001b[0m 'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/keras/engine/input_layer.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, input_shape, batch_size, batch_input_shape, dtype, input_tensor, sparse, name)\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0msparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m name=self.name)\n\u001b[0m\u001b[1;32m 88\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_placeholder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mplaceholder\u001b[0;34m(shape, ndim, dtype, sparse, name)\u001b[0m\n\u001b[1;32m 734\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfloatx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 735\u001b[0m x = tf_keras_backend.placeholder(\n\u001b[0;32m--> 736\u001b[0;31m shape=shape, ndim=ndim, dtype=dtype, sparse=sparse, name=name)\n\u001b[0m\u001b[1;32m 737\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 738\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mndim\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36mplaceholder\u001b[0;34m(shape, ndim, dtype, sparse, name)\u001b[0m\n\u001b[1;32m 996\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparse_placeholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 997\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 998\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplaceholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 999\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1000\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py\u001b[0m in \u001b[0;36mplaceholder\u001b[0;34m(dtype, shape, name)\u001b[0m\n\u001b[1;32m 2141\u001b[0m \"eager execution.\")\n\u001b[1;32m 2142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2143\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplaceholder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/ops/gen_array_ops.py\u001b[0m in \u001b[0;36mplaceholder\u001b[0;34m(dtype, shape, name)\u001b[0m\n\u001b[1;32m 6258\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6259\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6260\u001b[0;31m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_execute\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"shape\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6261\u001b[0m _, _, _op = _op_def_lib._apply_op_helper(\n\u001b[1;32m 6262\u001b[0m \"Placeholder\", dtype=dtype, shape=shape, name=name)\n",
|
||||
"\u001b[0;32m/netpool/homes/wangyo/.conda/envs/tf-cuda9/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mmake_shape\u001b[0;34m(v, arg_name)\u001b[0m\n\u001b[1;32m 146\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensor_shape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error converting %s to a TensorShape: %s.\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0marg_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 149\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 150\u001b[0m raise ValueError(\"Error converting %s to a TensorShape: %s.\" % (arg_name,\n",
|
||||
"\u001b[0;31mTypeError\u001b[0m: Error converting shape to a TensorShape: only size-1 arrays can be converted to Python scalars."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def test_xception_shape():\n",
|
||||
" input_ = np.random.randint(0,256, (240,320,3))\n",
|
||||
" print(input_)\n",
|
||||
" inp = Input(input_)\n",
|
||||
" xception = Xception_wrapper(include_top=False, weights='imagenet', input_tensor=inp, pooling=None)\n",
|
||||
" print('xception:',xception.output.shape)\n",
|
||||
"test_xception_shape()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
BIN
RecallNet/src/__pycache__/attentive_convlstm_new.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/attentive_convlstm_new.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/attentive_convlstm_new.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/attentive_convlstm_new.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/cb.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/cb.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/cb.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/cb.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/data_loading.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/data_loading.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/data_loading.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/data_loading.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/dcn_resnet_new.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/dcn_resnet_new.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/dcn_resnet_new.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/dcn_resnet_new.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/eval.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/eval.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/eval_saliconeval.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/eval_saliconeval.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/gaussian_prior_new.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/gaussian_prior_new.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/gaussian_prior_new.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/gaussian_prior_new.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/losses_keras2.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/losses_keras2.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/losses_keras2.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/losses_keras2.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/multiduration_models.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/multiduration_models.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/multiduration_models.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/multiduration_models.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/sal_imp_utilities.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/sal_imp_utilities.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/sal_imp_utilities.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/sal_imp_utilities.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/singleduration_models.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/singleduration_models.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/singleduration_models.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/singleduration_models.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/util.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/util.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/util.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/util.cpython-37.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/xception_custom.cpython-36.pyc
Normal file
BIN
RecallNet/src/__pycache__/xception_custom.cpython-36.pyc
Normal file
Binary file not shown.
BIN
RecallNet/src/__pycache__/xception_custom.cpython-37.pyc
Normal file
BIN
RecallNet/src/__pycache__/xception_custom.cpython-37.pyc
Normal file
Binary file not shown.
717
RecallNet/src/attentive_convlstm_new.py
Normal file
717
RecallNet/src/attentive_convlstm_new.py
Normal file
|
@ -0,0 +1,717 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Convolutional-recurrent layers.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import tensorflow as tf
|
||||
from keras import backend as K
|
||||
from keras import activations
|
||||
from keras import initializers
|
||||
from keras import regularizers
|
||||
from keras import constraints
|
||||
#from keras.layers.recurrent import _generate_dropout_mask
|
||||
#from keras.layers.recurrent import _standardize_args
|
||||
|
||||
import numpy as np
|
||||
import warnings
|
||||
from keras.engine.base_layer import InputSpec, Layer
|
||||
from keras.utils import conv_utils
|
||||
#from keras.legacy import interfaces
|
||||
#from keras.legacy.layers import Recurrent, ConvRecurrent2D
|
||||
from keras.layers.recurrent import RNN
|
||||
from keras.utils.generic_utils import has_arg
|
||||
from keras.utils.generic_utils import to_list
|
||||
from keras.utils.generic_utils import transpose_shape
|
||||
from tensorflow.python.keras.layers.convolutional_recurrent import ConvRNN2D
|
||||
|
||||
|
||||
def _generate_dropout_mask(ones, rate, training=None, count=1):
|
||||
def dropped_inputs():
|
||||
return K.dropout(ones, rate)
|
||||
|
||||
class AttentiveConvLSTM2DCell(Layer):
|
||||
|
||||
def __init__(self,
|
||||
filters,
|
||||
attentive_filters,
|
||||
kernel_size,
|
||||
attentive_kernel_size,
|
||||
strides=(1, 1),
|
||||
padding='valid',
|
||||
data_format=None,
|
||||
dilation_rate=(1, 1),
|
||||
activation='tanh',
|
||||
recurrent_activation='hard_sigmoid',
|
||||
attentive_activation='tanh',
|
||||
use_bias=True,
|
||||
kernel_initializer='glorot_uniform',
|
||||
recurrent_initializer='orthogonal',
|
||||
attentive_initializer='zeros',
|
||||
bias_initializer='zeros',
|
||||
unit_forget_bias=True,
|
||||
kernel_regularizer=None,
|
||||
recurrent_regularizer=None,
|
||||
attentive_regularizer=None,
|
||||
bias_regularizer=None,
|
||||
kernel_constraint=None,
|
||||
recurrent_constraint=None,
|
||||
attentive_constraint=None,
|
||||
bias_constraint=None,
|
||||
dropout=0.,
|
||||
recurrent_dropout=0.,
|
||||
attentive_dropout=0.,
|
||||
**kwargs):
|
||||
super(AttentiveConvLSTM2DCell, self).__init__(**kwargs)
|
||||
self.filters = filters
|
||||
self.attentive_filters = attentive_filters
|
||||
self.kernel_size = conv_utils.normalize_tuple(kernel_size, 2, 'kernel_size')
|
||||
self.attentive_kernel_size = conv_utils.normalize_tuple(attentive_kernel_size, 2, 'attentive_kernel_size')
|
||||
self.strides = conv_utils.normalize_tuple(strides, 2, 'strides')
|
||||
self.padding = conv_utils.normalize_padding(padding)
|
||||
self.data_format = K.normalize_data_format(data_format)
|
||||
self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, 2,
|
||||
'dilation_rate')
|
||||
self.activation = activations.get(activation)
|
||||
self.recurrent_activation = activations.get(recurrent_activation)
|
||||
self.attentive_activation = activations.get(attentive_activation)
|
||||
self.use_bias = use_bias
|
||||
|
||||
self.kernel_initializer = initializers.get(kernel_initializer)
|
||||
self.recurrent_initializer = initializers.get(recurrent_initializer)
|
||||
self.attentive_initializer = initializers.get(attentive_initializer)
|
||||
self.bias_initializer = initializers.get(bias_initializer)
|
||||
self.unit_forget_bias = unit_forget_bias
|
||||
|
||||
self.kernel_regularizer = regularizers.get(kernel_regularizer)
|
||||
self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
|
||||
self.attentive_regularizer = regularizers.get(attentive_regularizer)
|
||||
self.bias_regularizer = regularizers.get(bias_regularizer)
|
||||
|
||||
self.kernel_constraint = constraints.get(kernel_constraint)
|
||||
self.recurrent_constraint = constraints.get(recurrent_constraint)
|
||||
self.attentive_constraint = constraints.get(attentive_constraint)
|
||||
self.bias_constraint = constraints.get(bias_constraint)
|
||||
|
||||
|
||||
if K.backend() == 'theano' and (dropout or recurrent_dropout):
|
||||
warnings.warn(
|
||||
'RNN dropout is no longer supported with the Theano backend '
|
||||
'due to technical limitations. '
|
||||
'You can either set `dropout` and `recurrent_dropout` to 0, '
|
||||
'or use the TensorFlow backend.')
|
||||
dropout = 0.
|
||||
recurrent_dropout = 0.
|
||||
self.dropout = min(1., max(0., dropout))
|
||||
self.recurrent_dropout = min(1., max(0., recurrent_dropout))
|
||||
self.attentive_dropout = min(1., max(0., attentive_dropout))
|
||||
self.state_size = (self.filters, self.filters)
|
||||
self._dropout_mask = None
|
||||
self._recurrent_dropout_mask = None
|
||||
self._attentive_dropout_mask = None
|
||||
|
||||
def build(self, input_shape):
|
||||
|
||||
if self.data_format == 'channels_first':
|
||||
channel_axis = 1
|
||||
else:
|
||||
channel_axis = -1
|
||||
if input_shape[channel_axis] is None:
|
||||
raise ValueError('The channel dimension of the inputs '
|
||||
'should be defined. Found `None`.')
|
||||
input_dim = input_shape[channel_axis]
|
||||
kernel_shape = self.kernel_size + (input_dim, self.filters * 4)
|
||||
|
||||
self.kernel_shape = kernel_shape
|
||||
print('kernel_shape', kernel_shape)
|
||||
recurrent_kernel_shape = self.kernel_size + (self.filters, self.filters * 4)
|
||||
input_attentive_kernel_shape = self.attentive_kernel_size + (input_dim, self.attentive_filters)
|
||||
hidden_attentive_kernel_shape = self.attentive_kernel_size + (self.filters, self.attentive_filters)
|
||||
squeeze_attentive_kernel_shape = self.attentive_kernel_size + (self.attentive_filters, 1)
|
||||
|
||||
self.kernel = self.add_weight(shape=kernel_shape,
|
||||
initializer=self.kernel_initializer,
|
||||
name='kernel',
|
||||
regularizer=self.kernel_regularizer,
|
||||
constraint=self.kernel_constraint)
|
||||
|
||||
self.recurrent_kernel = self.add_weight(
|
||||
shape=recurrent_kernel_shape,
|
||||
initializer=self.recurrent_initializer,
|
||||
name='recurrent_kernel',
|
||||
regularizer=self.recurrent_regularizer,
|
||||
constraint=self.recurrent_constraint)
|
||||
|
||||
self.input_attentive_kernel = self.add_weight(
|
||||
shape=input_attentive_kernel_shape,
|
||||
initializer=self.attentive_initializer,
|
||||
name='input_attentive_kernel',
|
||||
regularizer=self.attentive_regularizer,
|
||||
constraint=self.attentive_constraint)
|
||||
|
||||
self.hidden_attentive_kernel = self.add_weight(
|
||||
shape=hidden_attentive_kernel_shape,
|
||||
initializer=self.attentive_initializer,
|
||||
name='hidden_attentive_kernel',
|
||||
regularizer=self.attentive_regularizer,
|
||||
constraint=self.attentive_constraint)
|
||||
|
||||
self.squeeze_attentive_kernel = self.add_weight(
|
||||
shape=squeeze_attentive_kernel_shape,
|
||||
initializer=self.attentive_initializer,
|
||||
name='squeeze_attentive_kernel',
|
||||
regularizer=self.attentive_regularizer,
|
||||
constraint=self.attentive_constraint)
|
||||
|
||||
|
||||
if self.use_bias:
|
||||
if self.unit_forget_bias:
|
||||
|
||||
def bias_initializer(_, *args, **kwargs):
|
||||
return K.concatenate([
|
||||
self.bias_initializer((self.filters,), *args, **kwargs),
|
||||
initializers.Ones()((self.filters,), *args, **kwargs),
|
||||
self.bias_initializer((self.filters * 2,), *args, **kwargs),
|
||||
])
|
||||
else:
|
||||
bias_initializer = self.bias_initializer
|
||||
self.bias = self.add_weight(
|
||||
shape=(self.filters * 4,),
|
||||
name='bias',
|
||||
initializer=bias_initializer,
|
||||
regularizer=self.bias_regularizer,
|
||||
constraint=self.bias_constraint)
|
||||
|
||||
self.attentive_bias = self.add_weight(
|
||||
shape=(self.attentive_filters * 2,),
|
||||
name='attentive_bias',
|
||||
initializer=bias_initializer,
|
||||
regularizer=self.bias_regularizer,
|
||||
constraint=self.bias_constraint)
|
||||
|
||||
else:
|
||||
self.bias = None
|
||||
|
||||
self.kernel_i = self.kernel[:, :, :, :self.filters]
|
||||
self.recurrent_kernel_i = self.recurrent_kernel[:, :, :, :self.filters]
|
||||
self.kernel_f = self.kernel[:, :, :, self.filters: self.filters * 2]
|
||||
self.recurrent_kernel_f = (self.recurrent_kernel[:, :, :, self.filters:
|
||||
self.filters * 2])
|
||||
self.kernel_c = self.kernel[:, :, :, self.filters * 2: self.filters * 3]
|
||||
self.recurrent_kernel_c = (self.recurrent_kernel[:, :, :, self.filters * 2:
|
||||
self.filters * 3])
|
||||
self.kernel_o = self.kernel[:, :, :, self.filters * 3:]
|
||||
self.recurrent_kernel_o = self.recurrent_kernel[:, :, :, self.filters * 3:]
|
||||
|
||||
if self.use_bias:
|
||||
self.bias_i = self.bias[:self.filters]
|
||||
self.bias_f = self.bias[self.filters: self.filters * 2]
|
||||
self.bias_c = self.bias[self.filters * 2: self.filters * 3]
|
||||
self.bias_o = self.bias[self.filters * 3:]
|
||||
self.bias_wa = self.attentive_bias[:self.attentive_filters ]
|
||||
self.bias_ua = self.attentive_bias[self.attentive_filters : self.attentive_filters * 2]
|
||||
else:
|
||||
self.bias_i = None
|
||||
self.bias_f = None
|
||||
self.bias_c = None
|
||||
self.bias_o = None
|
||||
|
||||
self.built = True
|
||||
|
||||
def call(self, inputs, states, training=None):
|
||||
if 0 < self.dropout < 1 and self._dropout_mask is None:
|
||||
self._dropout_mask = _generate_dropout_mask(
|
||||
K.ones_like(inputs),
|
||||
self.dropout,
|
||||
training=training,
|
||||
count=4)
|
||||
if (0 < self.recurrent_dropout < 1 and
|
||||
self._recurrent_dropout_mask is None):
|
||||
self._recurrent_dropout_mask = _generate_dropout_mask(
|
||||
K.ones_like(states[1]),
|
||||
self.recurrent_dropout,
|
||||
training=training,
|
||||
count=4)
|
||||
# if (0 < self.attentive_dropout < 1 and self._attentive_dropout_mask is None):
|
||||
# self._attentive_dropout_mask = _generate_dropout_mask(
|
||||
# K.ones_like(inputs),
|
||||
# self.attentive_dropout,
|
||||
# training=training,
|
||||
# count=4)
|
||||
|
||||
# dropout matrices for input units
|
||||
dp_mask = self._dropout_mask
|
||||
# dropout matrices for recurrent units
|
||||
rec_dp_mask = self._recurrent_dropout_mask
|
||||
# dropout matrices for attentive units
|
||||
# att_dp_mask = self._attentive_dropout_mask
|
||||
|
||||
h_tm1 = states[0] # previous memory state
|
||||
c_tm1 = states[1] # previous carry state
|
||||
|
||||
|
||||
##### ATTENTION MECHANISM
|
||||
|
||||
h_and_x = self.input_conv(h_tm1, self.hidden_attentive_kernel, self.bias_ua, padding='same') + self.input_conv(inputs, self.input_attentive_kernel, self.bias_wa, padding='same')
|
||||
|
||||
e = self.recurrent_conv(self.attentive_activation(h_and_x), self.squeeze_attentive_kernel)
|
||||
|
||||
a = K.reshape(K.softmax(K.batch_flatten(e)), K.shape(e))
|
||||
inputs = inputs * K.repeat_elements(a, inputs.shape[-1], -1)
|
||||
|
||||
|
||||
##### END OF ATTENTION MECHANISM
|
||||
|
||||
|
||||
if 0 < self.dropout < 1.:
|
||||
inputs_i = inputs * dp_mask[0]
|
||||
inputs_f = inputs * dp_mask[1]
|
||||
inputs_c = inputs * dp_mask[2]
|
||||
inputs_o = inputs * dp_mask[3]
|
||||
else:
|
||||
inputs_i = inputs
|
||||
inputs_f = inputs
|
||||
inputs_c = inputs
|
||||
inputs_o = inputs
|
||||
|
||||
if 0 < self.recurrent_dropout < 1.:
|
||||
h_tm1_i = h_tm1 * rec_dp_mask[0]
|
||||
h_tm1_f = h_tm1 * rec_dp_mask[1]
|
||||
h_tm1_c = h_tm1 * rec_dp_mask[2]
|
||||
h_tm1_o = h_tm1 * rec_dp_mask[3]
|
||||
else:
|
||||
h_tm1_i = h_tm1
|
||||
h_tm1_f = h_tm1
|
||||
h_tm1_c = h_tm1
|
||||
h_tm1_o = h_tm1
|
||||
|
||||
|
||||
x_i = self.input_conv(inputs_i, self.kernel_i, self.bias_i,
|
||||
padding=self.padding)
|
||||
x_f = self.input_conv(inputs_f, self.kernel_f, self.bias_f,
|
||||
padding=self.padding)
|
||||
x_c = self.input_conv(inputs_c, self.kernel_c, self.bias_c,
|
||||
padding=self.padding)
|
||||
x_o = self.input_conv(inputs_o, self.kernel_o, self.bias_o,
|
||||
padding=self.padding)
|
||||
h_i = self.recurrent_conv(h_tm1_i,
|
||||
self.recurrent_kernel_i)
|
||||
h_f = self.recurrent_conv(h_tm1_f,
|
||||
self.recurrent_kernel_f)
|
||||
h_c = self.recurrent_conv(h_tm1_c,
|
||||
self.recurrent_kernel_c)
|
||||
h_o = self.recurrent_conv(h_tm1_o,
|
||||
self.recurrent_kernel_o)
|
||||
|
||||
i = self.recurrent_activation(x_i + h_i)
|
||||
f = self.recurrent_activation(x_f + h_f)
|
||||
c = f * c_tm1 + i * self.activation(x_c + h_c)
|
||||
o = self.recurrent_activation(x_o + h_o)
|
||||
h = o * self.activation(c)
|
||||
|
||||
if 0 < self.dropout + self.recurrent_dropout:
|
||||
if training is None:
|
||||
h._uses_learning_phase = True
|
||||
|
||||
return h, [h, c]
|
||||
|
||||
def input_conv(self, x, w, b=None, padding='valid'):
|
||||
conv_out = K.conv2d(x, w, strides=self.strides,
|
||||
padding=padding,
|
||||
data_format=self.data_format,
|
||||
dilation_rate=self.dilation_rate)
|
||||
if b is not None:
|
||||
conv_out = K.bias_add(conv_out, b,
|
||||
data_format=self.data_format)
|
||||
return conv_out
|
||||
|
||||
def recurrent_conv(self, x, w):
|
||||
conv_out = K.conv2d(x, w, strides=(1, 1),
|
||||
padding='same',
|
||||
data_format=self.data_format)
|
||||
return conv_out
|
||||
|
||||
def get_config(self):
|
||||
config = {'filters': self.filters,
|
||||
'attentive_filters': self.attentive_filters,
|
||||
'kernel_size': self.kernel_size,
|
||||
'attentive_kernel_size': self.attentive_kernel_size,
|
||||
'strides': self.strides,
|
||||
'padding': self.padding,
|
||||
'data_format': self.data_format,
|
||||
'dilation_rate': self.dilation_rate,
|
||||
'activation': activations.serialize(self.activation),
|
||||
'recurrent_activation': activations.serialize(
|
||||
self.recurrent_activation),
|
||||
'attentive_activation': activations.serialize(
|
||||
self.attentive_activation),
|
||||
'use_bias': self.use_bias,
|
||||
'kernel_initializer': initializers.serialize(
|
||||
self.kernel_initializer),
|
||||
'recurrent_initializer': initializers.serialize(
|
||||
self.recurrent_initializer),
|
||||
'attentive_initializer': initializers.serialize(
|
||||
self.attentive_initializer),
|
||||
'bias_initializer': initializers.serialize(self.bias_initializer),
|
||||
'unit_forget_bias': self.unit_forget_bias,
|
||||
'kernel_regularizer': regularizers.serialize(
|
||||
self.kernel_regularizer),
|
||||
'recurrent_regularizer': regularizers.serialize(
|
||||
self.recurrent_regularizer),
|
||||
'attentive_regularizer': regularizers.serialize(
|
||||
self.attentive_regularizer),
|
||||
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
|
||||
'kernel_constraint': constraints.serialize(
|
||||
self.kernel_constraint),
|
||||
'recurrent_constraint': constraints.serialize(
|
||||
self.recurrent_constraint),
|
||||
'attentive_constraint': constraints.serialize(
|
||||
self.attentive_constraint),
|
||||
'bias_constraint': constraints.serialize(self.bias_constraint),
|
||||
'dropout': self.dropout,
|
||||
'recurrent_dropout': self.recurrent_dropout,
|
||||
'attentive_dropout': self.attentive_dropout}
|
||||
base_config = super(AttentiveConvLSTM2DCell, self).get_config()
|
||||
return dict(list(base_config.items()) + list(config.items()))
|
||||
|
||||
|
||||
class AttentiveConvLSTM2D(ConvRNN2D):
|
||||
"""Convolutional LSTM.
|
||||
It is similar to an LSTM layer, but the input transformations
|
||||
and recurrent transformations are both convolutional.
|
||||
Arguments:
|
||||
filters: Integer, the dimensionality of the output space
|
||||
(i.e. the number of output filters in the convolution).
|
||||
kernel_size: An integer or tuple/list of n integers, specifying the
|
||||
dimensions of the convolution window.
|
||||
strides: An integer or tuple/list of n integers,
|
||||
specifying the strides of the convolution.
|
||||
Specifying any stride value != 1 is incompatible with specifying
|
||||
any `dilation_rate` value != 1.
|
||||
padding: One of `"valid"` or `"same"` (case-insensitive).
|
||||
data_format: A string,
|
||||
one of `channels_last` (default) or `channels_first`.
|
||||
The ordering of the dimensions in the inputs.
|
||||
`channels_last` corresponds to inputs with shape
|
||||
`(batch, time, ..., channels)`
|
||||
while `channels_first` corresponds to
|
||||
inputs with shape `(batch, time, channels, ...)`.
|
||||
It defaults to the `image_data_format` value found in your
|
||||
Keras config file at `~/.keras/keras.json`.
|
||||
If you never set it, then it will be "channels_last".
|
||||
dilation_rate: An integer or tuple/list of n integers, specifying
|
||||
the dilation rate to use for dilated convolution.
|
||||
Currently, specifying any `dilation_rate` value != 1 is
|
||||
incompatible with specifying any `strides` value != 1.
|
||||
activation: Activation function to use.
|
||||
If you don't specify anything, no activation is applied
|
||||
(ie. "linear" activation: `a(x) = x`).
|
||||
recurrent_activation: Activation function to use
|
||||
for the recurrent step.
|
||||
use_bias: Boolean, whether the layer uses a bias vector.
|
||||
kernel_initializer: Initializer for the `kernel` weights matrix,
|
||||
used for the linear transformation of the inputs.
|
||||
recurrent_initializer: Initializer for the `recurrent_kernel`
|
||||
weights matrix,
|
||||
used for the linear transformation of the recurrent state.
|
||||
bias_initializer: Initializer for the bias vector.
|
||||
unit_forget_bias: Boolean.
|
||||
If True, add 1 to the bias of the forget gate at initialization.
|
||||
Use in combination with `bias_initializer="zeros"`.
|
||||
This is recommended in [Jozefowicz et al.]
|
||||
(http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
|
||||
kernel_regularizer: Regularizer function applied to
|
||||
the `kernel` weights matrix.
|
||||
recurrent_regularizer: Regularizer function applied to
|
||||
the `recurrent_kernel` weights matrix.
|
||||
bias_regularizer: Regularizer function applied to the bias vector.
|
||||
activity_regularizer: Regularizer function applied to.
|
||||
kernel_constraint: Constraint function applied to
|
||||
the `kernel` weights matrix.
|
||||
recurrent_constraint: Constraint function applied to
|
||||
the `recurrent_kernel` weights matrix.
|
||||
bias_constraint: Constraint function applied to the bias vector.
|
||||
return_sequences: Boolean. Whether to return the last output
|
||||
in the output sequence, or the full sequence.
|
||||
go_backwards: Boolean (default False).
|
||||
If True, process the input sequence backwards.
|
||||
stateful: Boolean (default False). If True, the last state
|
||||
for each sample at index i in a batch will be used as initial
|
||||
state for the sample of index i in the following batch.
|
||||
dropout: Float between 0 and 1.
|
||||
Fraction of the units to drop for
|
||||
the linear transformation of the inputs.
|
||||
recurrent_dropout: Float between 0 and 1.
|
||||
Fraction of the units to drop for
|
||||
the linear transformation of the recurrent state.
|
||||
Input shape:
|
||||
- if data_format='channels_first'
|
||||
5D tensor with shape:
|
||||
`(samples, time, channels, rows, cols)`
|
||||
- if data_format='channels_last'
|
||||
5D tensor with shape:
|
||||
`(samples, time, rows, cols, channels)`
|
||||
Output shape:
|
||||
- if `return_sequences`
|
||||
- if data_format='channels_first'
|
||||
5D tensor with shape:
|
||||
`(samples, time, filters, output_row, output_col)`
|
||||
- if data_format='channels_last'
|
||||
5D tensor with shape:
|
||||
`(samples, time, output_row, output_col, filters)`
|
||||
- else
|
||||
- if data_format ='channels_first'
|
||||
4D tensor with shape:
|
||||
`(samples, filters, output_row, output_col)`
|
||||
- if data_format='channels_last'
|
||||
4D tensor with shape:
|
||||
`(samples, output_row, output_col, filters)`
|
||||
where o_row and o_col depend on the shape of the filter and
|
||||
the padding
|
||||
Raises:
|
||||
ValueError: in case of invalid constructor arguments.
|
||||
References:
|
||||
- [Convolutional LSTM Network: A Machine Learning Approach for
|
||||
Precipitation Nowcasting](http://arxiv.org/abs/1506.04214v1)
|
||||
The current implementation does not include the feedback loop on the
|
||||
cells output.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
filters,
|
||||
attentive_filters,
|
||||
kernel_size,
|
||||
attentive_kernel_size,
|
||||
strides=(1, 1),
|
||||
padding='valid',
|
||||
data_format=None,
|
||||
dilation_rate=(1, 1),
|
||||
activation='tanh',
|
||||
recurrent_activation='hard_sigmoid',
|
||||
attentive_activation='tanh',
|
||||
use_bias=True,
|
||||
kernel_initializer='glorot_uniform',
|
||||
recurrent_initializer='orthogonal',
|
||||
attentive_initializer='zeros',
|
||||
bias_initializer='zeros',
|
||||
unit_forget_bias=True,
|
||||
kernel_regularizer=None,
|
||||
recurrent_regularizer=None,
|
||||
attentive_regularizer=None,
|
||||
bias_regularizer=None,
|
||||
activity_regularizer=None,
|
||||
kernel_constraint=None,
|
||||
recurrent_constraint=None,
|
||||
attentive_constraint=None,
|
||||
bias_constraint=None,
|
||||
return_sequences=False,
|
||||
go_backwards=False,
|
||||
stateful=False,
|
||||
dropout=0.,
|
||||
recurrent_dropout=0.,
|
||||
attentive_dropout=0.,
|
||||
**kwargs):
|
||||
cell = AttentiveConvLSTM2DCell(filters=filters,
|
||||
attentive_filters=attentive_filters,
|
||||
kernel_size=kernel_size,
|
||||
attentive_kernel_size=attentive_kernel_size,
|
||||
strides=strides,
|
||||
padding=padding,
|
||||
data_format=data_format,
|
||||
dilation_rate=dilation_rate,
|
||||
activation=activation,
|
||||
recurrent_activation=recurrent_activation,
|
||||
attentive_activation=attentive_activation,
|
||||
use_bias=use_bias,
|
||||
kernel_initializer=kernel_initializer,
|
||||
recurrent_initializer=recurrent_initializer,
|
||||
attentive_initializer=attentive_initializer,
|
||||
bias_initializer=bias_initializer,
|
||||
unit_forget_bias=unit_forget_bias,
|
||||
kernel_regularizer=kernel_regularizer,
|
||||
recurrent_regularizer=recurrent_regularizer,
|
||||
attentive_regularizer=attentive_regularizer,
|
||||
bias_regularizer=bias_regularizer,
|
||||
kernel_constraint=kernel_constraint,
|
||||
recurrent_constraint=recurrent_constraint,
|
||||
attentive_constraint=attentive_constraint,
|
||||
bias_constraint=bias_constraint,
|
||||
dropout=dropout,
|
||||
recurrent_dropout=recurrent_dropout,
|
||||
attentive_dropout=attentive_dropout)
|
||||
|
||||
super(AttentiveConvLSTM2D, self).__init__(cell,
|
||||
return_sequences=return_sequences,
|
||||
go_backwards=go_backwards,
|
||||
stateful=stateful,
|
||||
**kwargs)
|
||||
self.activity_regularizer = regularizers.get(activity_regularizer)
|
||||
|
||||
def call(self, inputs, mask=None, training=None, initial_state=None):
|
||||
return super(AttentiveConvLSTM2D, self).call(inputs,
|
||||
mask=mask,
|
||||
training=training,
|
||||
initial_state=initial_state)
|
||||
|
||||
@property
|
||||
def filters(self):
|
||||
return self.cell.filters
|
||||
|
||||
@property
|
||||
def attentive_filters(self):
|
||||
return self.cell.attentive_filters
|
||||
|
||||
@property
|
||||
def kernel_size(self):
|
||||
return self.cell.kernel_size
|
||||
|
||||
@property
|
||||
def attentive_kernel_size(self):
|
||||
return self.cell.attentive_kernel_size
|
||||
|
||||
@property
|
||||
def strides(self):
|
||||
return self.cell.strides
|
||||
|
||||
@property
|
||||
def padding(self):
|
||||
return self.cell.padding
|
||||
|
||||
@property
|
||||
def data_format(self):
|
||||
return self.cell.data_format
|
||||
|
||||
@property
|
||||
def dilation_rate(self):
|
||||
return self.cell.dilation_rate
|
||||
|
||||
@property
|
||||
def activation(self):
|
||||
return self.cell.activation
|
||||
|
||||
@property
|
||||
def recurrent_activation(self):
|
||||
return self.cell.recurrent_activation
|
||||
|
||||
@property
|
||||
def attentive_activation(self):
|
||||
return self.cell.attentive_activation
|
||||
|
||||
@property
|
||||
def use_bias(self):
|
||||
return self.cell.use_bias
|
||||
|
||||
@property
|
||||
def kernel_initializer(self):
|
||||
return self.cell.kernel_initializer
|
||||
|
||||
@property
|
||||
def recurrent_initializer(self):
|
||||
return self.cell.recurrent_initializer
|
||||
|
||||
@property
|
||||
def attentive_initializer(self):
|
||||
return self.cell.attentive_initializer
|
||||
|
||||
@property
|
||||
def bias_initializer(self):
|
||||
return self.cell.bias_initializer
|
||||
|
||||
@property
|
||||
def unit_forget_bias(self):
|
||||
return self.cell.unit_forget_bias
|
||||
|
||||
@property
|
||||
def kernel_regularizer(self):
|
||||
return self.cell.kernel_regularizer
|
||||
|
||||
@property
|
||||
def recurrent_regularizer(self):
|
||||
return self.cell.recurrent_regularizer
|
||||
|
||||
@property
|
||||
def attentive_regularizer(self):
|
||||
return self.cell.attentive_regularizer
|
||||
|
||||
@property
|
||||
def bias_regularizer(self):
|
||||
return self.cell.bias_regularizer
|
||||
|
||||
@property
|
||||
def kernel_constraint(self):
|
||||
return self.cell.kernel_constraint
|
||||
|
||||
@property
|
||||
def recurrent_constraint(self):
|
||||
return self.cell.recurrent_constraint
|
||||
|
||||
@property
|
||||
def attentive_constraint(self):
|
||||
return self.cell.attentive_constraint
|
||||
|
||||
@property
|
||||
def bias_constraint(self):
|
||||
return self.cell.bias_constraint
|
||||
|
||||
@property
|
||||
def dropout(self):
|
||||
return self.cell.dropout
|
||||
|
||||
@property
|
||||
def recurrent_dropout(self):
|
||||
return self.cell.recurrent_dropout
|
||||
|
||||
@property
|
||||
def attentive_dropout(self):
|
||||
return self.cell.attentive_dropout
|
||||
|
||||
def get_config(self):
|
||||
config = {'filters': self.filters,
|
||||
'attentive_filters': self.attentive_filters,
|
||||
'kernel_size': self.kernel_size,
|
||||
'attentive_kernel_size': self.attentive_kernel_size,
|
||||
'strides': self.strides,
|
||||
'padding': self.padding,
|
||||
'data_format': self.data_format,
|
||||
'dilation_rate': self.dilation_rate,
|
||||
'activation': activations.serialize(self.activation),
|
||||
'recurrent_activation': activations.serialize(
|
||||
self.recurrent_activation),
|
||||
'attentive_activation': activations.serialize(
|
||||
self.attentive_activation),
|
||||
'use_bias': self.use_bias,
|
||||
'kernel_initializer': initializers.serialize(
|
||||
self.kernel_initializer),
|
||||
'recurrent_initializer': initializers.serialize(
|
||||
self.recurrent_initializer),
|
||||
'attentive_initializer': initializers.serialize(
|
||||
self.attentive_initializer),
|
||||
'bias_initializer': initializers.serialize(self.bias_initializer),
|
||||
'unit_forget_bias': self.unit_forget_bias,
|
||||
'kernel_regularizer': regularizers.serialize(
|
||||
self.kernel_regularizer),
|
||||
'recurrent_regularizer': regularizers.serialize(
|
||||
self.recurrent_regularizer),
|
||||
'attentive_regularizer': regularizers.serialize(
|
||||
self.attentive_regularizer),
|
||||
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
|
||||
'activity_regularizer': regularizers.serialize(
|
||||
self.activity_regularizer),
|
||||
'kernel_constraint': constraints.serialize(
|
||||
self.kernel_constraint),
|
||||
'recurrent_constraint': constraints.serialize(
|
||||
self.recurrent_constraint),
|
||||
'attentive_constraint': constraints.serialize(
|
||||
self.attentive_constraint),
|
||||
'bias_constraint': constraints.serialize(self.bias_constraint),
|
||||
'dropout': self.dropout,
|
||||
'recurrent_dropout': self.recurrent_dropout,
|
||||
'attentive_dropout': self.attentive_dropout}
|
||||
base_config = super(AttentiveConvLSTM2D, self).get_config()
|
||||
del base_config['cell']
|
||||
return dict(list(base_config.items()) + list(config.items()))
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
return cls(**config)
|
80
RecallNet/src/cb.py
Normal file
80
RecallNet/src/cb.py
Normal file
|
@ -0,0 +1,80 @@
|
|||
import keras
|
||||
import matplotlib.pyplot as plt
|
||||
from IPython.display import clear_output
|
||||
import os
|
||||
import keras
|
||||
from keras.callbacks import ModelCheckpoint
|
||||
import numpy as np
|
||||
import math
|
||||
|
||||
|
||||
class Unfreeze(keras.callbacks.Callback):
|
||||
def __init__(self,it_to_unfreeze):
|
||||
self.it_to_unfreeze = it_to_unfreeze
|
||||
self.c=0
|
||||
self.frozen=True
|
||||
|
||||
def on_batch_end(self, batch, logs=None):
|
||||
self.c+=1
|
||||
if not self.c > self.it_to_unfreeze and self.frozen:
|
||||
print('Iteration %d reached: UNFREEZING ENCODER' % self.c)
|
||||
self.frozen=False
|
||||
for layer in self.model.layers:
|
||||
layer.trainable=True
|
||||
|
||||
class InteractivePlot(keras.callbacks.Callback):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def on_train_begin(self, logs={}):
|
||||
self.losses = []
|
||||
self.logs = []
|
||||
self.batchnr = 0
|
||||
self.icount = 0
|
||||
|
||||
def on_train_end(self, logs={}):
|
||||
pass
|
||||
|
||||
def on_epoch_end(self, epoch, logs={}):
|
||||
self.batchnr = 0
|
||||
loss_train = logs.get('loss')
|
||||
|
||||
self.losses.append(loss_train)
|
||||
|
||||
self.icount+=1
|
||||
clear_output(wait=True)
|
||||
plt.figure(figsize=(14,10))
|
||||
train_vals = [self.losses]
|
||||
desc = ['loss']
|
||||
for i in range(len(train_vals)):
|
||||
#plt.subplot(2, 3, i+1)
|
||||
plt.plot(range(self.icount), train_vals[i], label=desc[i])
|
||||
plt.legend()
|
||||
#plt.savefig(self.logfile.replace('.txt', '.png'), bbox_inches='tight', format='png')
|
||||
plt.show()
|
||||
|
||||
def on_batch_end(self, batch, logs=None):
|
||||
self.batchnr+=1
|
||||
if self.batchnr % 10 == 0:
|
||||
self.on_epoch_end(epoch=0, logs=logs)
|
||||
|
||||
|
||||
|
||||
def ckpt_callback(model_name, dataset, l_str, bs, extra_str='',
|
||||
period=1, save_weights_only=True,
|
||||
ckpt_folder_path = '../../predimportance_shared/models/ckpt/'):
|
||||
path = os.path.join(ckpt_folder_path, model_name)
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
filepath = os.path.join(path, model_name+'_'+dataset+'_'+l_str+'_bs'+str(bs)+extra_str+'_ep{epoch:02d}_valloss{val_loss:.4f}.hdf5')
|
||||
cb_chk = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_weights_only=True, period=1)
|
||||
return cb_chk
|
||||
|
||||
|
||||
def step_decay(init_lr = 0.0001, drop = 0.1, epochs_drop = 3.0):
|
||||
def inner(epoch):
|
||||
lrate = init_lr * math.pow(drop, math.floor((1+epoch)/epochs_drop))
|
||||
if not (epoch+1)%epochs_drop:
|
||||
print('Reducing lr. New lr is:', lrate)
|
||||
return lrate
|
||||
return inner
|
566
RecallNet/src/classif_capable_models.py
Normal file
566
RecallNet/src/classif_capable_models.py
Normal file
|
@ -0,0 +1,566 @@
|
|||
import numpy as np
|
||||
import keras
|
||||
import sys
|
||||
import os
|
||||
from keras.layers import Layer, Input, Multiply, Dropout,DepthwiseConv2D, TimeDistributed, LSTM, Activation, Lambda, Conv2D, Dense, GlobalAveragePooling2D, MaxPooling2D, ZeroPadding2D, UpSampling2D, BatchNormalization, Concatenate
|
||||
import keras.backend as K
|
||||
from keras.models import Model
|
||||
import tensorflow as tf
|
||||
from keras.utils import Sequence
|
||||
import cv2
|
||||
import scipy.io
|
||||
import math
|
||||
from attentive_convlstm_new import AttentiveConvLSTM2D
|
||||
from dcn_resnet_new import dcn_resnet
|
||||
from gaussian_prior_new import LearningPrior
|
||||
from sal_imp_utilities import *
|
||||
from multiduration_models import decoder_block_timedist
|
||||
from xception_custom import Xception_wrapper
|
||||
from keras.applications import keras_modules_injection
|
||||
|
||||
|
||||
|
||||
def xception_cl(input_shape = (None, None, 3),
|
||||
verbose=True,
|
||||
print_shapes=True,
|
||||
n_outs=1,
|
||||
ups=8,
|
||||
freeze_enc=False,
|
||||
dil_rate = (2,2),
|
||||
freeze_cl=True,
|
||||
append_classif=True,
|
||||
num_classes=5):
|
||||
"""Xception with classification capabilities"""
|
||||
inp = Input(shape=input_shape)
|
||||
|
||||
### ENCODER ###
|
||||
xception = Xception_wrapper(include_top=False, weights='imagenet', input_tensor=inp, pooling=None)
|
||||
if print_shapes: print('xception output shapes:',xception.output.shape)
|
||||
if freeze_enc:
|
||||
for layer in xception.layers:
|
||||
layer.trainable = False
|
||||
|
||||
### CLASSIFIER ###
|
||||
cl = GlobalAveragePooling2D(name='gap_cl')(xception.output)
|
||||
cl = Dense(512,name='dense_cl')(cl)
|
||||
cl = Dropout(0.3, name='dropout_cl')(cl)
|
||||
cl = Dense(num_classes, activation='softmax', name='dense_cl_out')(cl)
|
||||
|
||||
## DECODER ##
|
||||
outs_dec = decoder_block(xception.output, dil_rate=dil_rate, print_shapes=print_shapes, dec_filt=512, prefix='decoder')
|
||||
|
||||
outs_final = [outs_dec]*n_outs
|
||||
|
||||
if append_classif:
|
||||
outs_final.append(cl)
|
||||
|
||||
# Building model
|
||||
m = Model(inp, outs_final) # Last element of outs_final is classification vector
|
||||
if verbose:
|
||||
m.summary()
|
||||
|
||||
if freeze_cl:
|
||||
print('Freezing classification dense layers')
|
||||
m.get_layer('dense_cl').trainable = False
|
||||
m.get_layer('dense_cl_out').trainable = False
|
||||
|
||||
return m
|
||||
|
||||
def xception_cl_fus(input_shape=(None, None, 3),
|
||||
verbose=True,
|
||||
print_shapes=True,
|
||||
n_outs=1,
|
||||
ups=8,
|
||||
dil_rate=(2,2),
|
||||
freeze_enc=False,
|
||||
freeze_cl=True,
|
||||
internal_filts=256,
|
||||
num_classes=5,
|
||||
dp=0.3):
|
||||
"""Xception with classification capabilities that fuses representations from both tasks"""
|
||||
inp = Input(shape=input_shape)
|
||||
|
||||
### ENCODER ###
|
||||
xception = Xception_wrapper(include_top=False, weights='imagenet', input_tensor=inp, pooling=None)
|
||||
if print_shapes: print('xception output shapes:',xception.output.shape)
|
||||
if freeze_enc:
|
||||
for layer in xception.layers:
|
||||
layer.trainable = False
|
||||
|
||||
### GLOBAL FEATURES ###
|
||||
g_n = global_net(xception.output, nfilts=internal_filts, dp=dp)
|
||||
if print_shapes: print('g_n shapes:', g_n.shape)
|
||||
|
||||
### CLASSIFIER ###
|
||||
# We potentially need another layer here
|
||||
out_classif = Dense(num_classes, activation='softmax', name='out_classif')(g_n)
|
||||
|
||||
### ASPP (MID LEVEL FEATURES) ###
|
||||
aspp_out = app(xception.output, internal_filts)
|
||||
if print_shapes: print('aspp out shapes:', aspp_out.shape)
|
||||
|
||||
### FUSION ###
|
||||
dense_f = Dense(internal_filts, name = 'dense_fusion')(g_n)
|
||||
if print_shapes: print('dense_f shapes:', dense_f.shape)
|
||||
reshap = Lambda(lambda x: K.repeat_elements(K.expand_dims(K.repeat_elements(K.expand_dims(x, axis=1), K.int_shape(aspp_out)[2], axis=1), axis=1), K.int_shape(aspp_out)[1], axis=1),
|
||||
lambda s: (s[0], K.int_shape(aspp_out)[1], K.int_shape(aspp_out)[2], s[1]))(dense_f)
|
||||
if print_shapes: print('after lambda shapes:', reshap.shape)
|
||||
|
||||
conc = Concatenate()([aspp_out,reshap])
|
||||
|
||||
### Projection ###
|
||||
x = Conv2D(internal_filts, (1, 1), padding='same', use_bias=False, name='concat_projection')(conc)
|
||||
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
|
||||
x = Activation('relu')(x)
|
||||
x = Dropout(dp)(x)
|
||||
|
||||
|
||||
### DECODER ###
|
||||
outs_dec = decoder_block(x, dil_rate=dil_rate, print_shapes=print_shapes, dec_filt=internal_filts, dp=dp)
|
||||
|
||||
outs_final = [outs_dec]*n_outs
|
||||
outs_final.append(out_classif)
|
||||
|
||||
# Building model
|
||||
m = Model(inp, outs_final) # Last element of outs_final is classification vector
|
||||
|
||||
if freeze_cl:
|
||||
m.get_layer('out_classif').trainable = False
|
||||
# for l in g_n.layers:
|
||||
# l.trainable=False
|
||||
|
||||
if verbose:
|
||||
m.summary()
|
||||
|
||||
return m
|
||||
|
||||
|
||||
|
||||
def xception_cl_fus_aspp(input_shape=(None |