Skip to content

Commit

Permalink
Add files via upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@sayakpaul
sayakpaul authored Apr 13, 2021
1 parent 245e73d commit 4382481
Showing 1 changed file with 46 additions and 95 deletions.
141 changes: 46 additions & 95 deletions Evaluation_OOD.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,14 @@
"id": "rhDWmAR9OERr",
"outputId": "f8603efc-b244-4a75-8e42-034b1387e3de"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/bin/bash: gdown: command not found\n"
]
}
],
"outputs": [],
"source": [
"# Retrieve the pre-trained ResNet20 model.\n",
"# Refer to this notebook for details: https://git.io/JOJuL.\n",
"!gdown --id 1-6Xp7__VMJeMBtGMByI7_GYVeIIv-1Ay -O odin_rn_model.tar.gz\n",
"!tar xf odin_rn_model.tar.gz"
"!/home/jupyter/.local/bin/gdown --id 1-6Xp7__VMJeMBtGMByI7_GYVeIIv-1Ay \n",
"!/home/jupyter/.local/bin/gdown --id 1y5WfyzQ0uxvEfkjdJb4mr5nYT0ItdoIL \n",
"!tar xf odin_rn_model.tar.gz\n",
"!tar xf rn_baseline_model.tar.gz"
]
},
{
Expand Down Expand Up @@ -151,25 +145,6 @@
"print(f\"Total examples in the test set of SVHN: {tf.data.experimental.cardinality(svhn_test).numpy()}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Prepare Masks"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"cifar10_id = np.arange(0, 10)\n",
"svhn_id = np.arange(1, 11)\n",
"\n",
"svhn_mask = [id in set(svhn_id) for id in cifar10_id]"
]
},
{
"cell_type": "markdown",
"metadata": {
Expand All @@ -181,7 +156,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 5,
"metadata": {
"id": "6tzvQDiJSf8Z"
},
Expand All @@ -193,7 +168,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 6,
"metadata": {
"id": "njjGGYBwQE8d"
},
Expand All @@ -209,57 +184,28 @@
"x_test = x_test.astype(\"float32\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now we need to select a sub-portion of the CIFAR-10 dataset that has overlapping labels with SVHN. "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"((9000, 32, 32, 3), (9000, 1))"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_y_test = np.array(list(filter(lambda x: x > 0 and x < 10, y_test)))\n",
"new_x_test = x_test[new_y_test].squeeze()\n",
"new_x_test.shape, new_y_test.shape"
]
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"ds = tf.data.Dataset.from_tensor_slices((new_x_test, new_y_test))\n",
"cifar10_test = ds.skip(1000).take(len(new_x_test) - 1000)"
"ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
"cifar10_test = ds.skip(1000).take(len(x_test) - 1000)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'svhn': 26032, 'cifar10_test': 8000}"
"{'svhn': 26032, 'cifar10_test': 9000}"
]
},
"execution_count": 10,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -290,7 +236,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 9,
"metadata": {
"id": "WzUie-5-8jAZ"
},
Expand All @@ -313,29 +259,34 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 10,
"metadata": {
"id": "R1u32-NbTHx1"
},
"outputs": [],
"source": [
"def perturb_images(images, model, epsilon=0.08):\n",
"\n",
" test_ds_var = tf.Variable(images, trainable=True)\n",
" \n",
"\n",
" with tf.GradientTape() as tape:\n",
" # Calculate the scores.\n",
" tape.watch(test_ds_var)\n",
" logits = model(test_ds_var, training=False)\n",
" scores = tf.reduce_max(logits, axis=1)\n",
" loss = tf.reduce_max(logits, axis=1)\n",
" loss = -tf.reduce_mean(loss)\n",
"\n",
" # Calculate the gradients of the scores with respect to the inputs.\n",
" # (Notice the sign.)\n",
" gradients = -tape.gradient(scores, test_ds_var)\n",
" gradients = tape.gradient(loss, test_ds_var)\n",
" gradients = tf.math.greater_equal(gradients, 0)\n",
" gradients = tf.cast(gradients, tf.float32)\n",
"\n",
" # Perturb the inputs and derive new mean score.\n",
" test_ds_var.assign_sub(epsilon * tf.sign(gradients))\n",
" # test_ds_var.assign_add(epsilon * gradients)\n",
" static_tensor = tf.convert_to_tensor(test_ds_var)\n",
"\n",
" static_tensor = static_tensor + epsilon + gradients\n",
" static_tensor = tf.clip_by_value(static_tensor, 0., 255.)\n",
" \n",
" return static_tensor"
]
},
Expand All @@ -361,7 +312,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -383,13 +334,13 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9b90b86bdb53466ca6b3adcad6389387",
"model_id": "75a10f80024c4a8f8a18e9e5e0202a72",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -403,12 +354,12 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d2b4a3ecc5a84ba2838093227827457d",
"model_id": "72874f0b7e7142d7ab6e08c25f6ebb1d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/8000 [00:00<?, ?it/s]"
" 0%| | 0/9000 [00:00<?, ?it/s]"
]
},
"metadata": {},
Expand All @@ -422,43 +373,43 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"# We create new data loaders.\n",
"noe_loader = perturbed_svhn.batch(BATCH_SIZE)\n",
"cifar10_test = pertubed_cifar10.batch(BATCH_SIZE)"
"perturbed_svhn = perturbed_svhn.batch(BATCH_SIZE)\n",
"pertubed_cifar10 = pertubed_cifar10.batch(BATCH_SIZE)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.8340361910341118\n"
"0.792900278716793\n"
]
}
],
"source": [
"# Calculate AUROC.\n",
"print(metrics.calculate_auroc(cifar10_test, noe_loader, rn_odin_model, svhn_mask, type=\"odin\"))"
"print(metrics.calculate_auroc(pertubed_cifar10, perturbed_svhn, rn_odin_model, type=\"odin\"))"
]
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.884750228084665\n"
"0.9154817660315508\n"
]
}
],
Expand All @@ -468,30 +419,30 @@
"# for our baseline model.\n",
"svhn_test = svhn_test.batch(BATCH_SIZE)\n",
"\n",
"ds = tf.data.Dataset.from_tensor_slices((new_x_test, new_y_test))\n",
"cifar10_test = ds.skip(1000).take(len(new_x_test) - 1000)\n",
"ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
"cifar10_test = ds.skip(1000).take(len(x_test) - 1000)\n",
"cifar10_test = cifar10_test.batch(BATCH_SIZE)\n",
"\n",
"print(metrics.calculate_auroc(cifar10_test, svhn_test, rn_baseline_model, svhn_mask, type=\"baseline\"))"
"print(metrics.calculate_auroc(cifar10_test, svhn_test, rn_baseline_model, type=\"baseline\"))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.883523756818531\n"
"0.8830217040565458\n"
]
}
],
"source": [
"# Baseline AUROC (without perturbation).\n",
"# Baseline AUROC (with perturbation).\n",
"cifar10_test = pertubed_cifar10.batch(BATCH_SIZE)\n",
"print(metrics.calculate_auroc(cifar10_test, noe_loader, rn_baseline_model, svhn_mask, type=\"baseline\"))"
"print(metrics.calculate_auroc(pertubed_cifar10, perturbed_svhn, rn_baseline_model, type=\"baseline\"))"
]
}
],
Expand Down

0 comments on commit 4382481

Please sign in to comment.