Skip to content

Commit

Permalink
Merge pull request #2 from m3dev/dev_kawai
Browse files Browse the repository at this point in the history 
update examples
  • Loading branch information
@vaaaaanquish
vaaaaanquish authored Aug 18, 2019
2 parents 15f7423 + 4b9919d commit 756c9f0
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 109 deletions.
221 changes: 114 additions & 107 deletions examples/example.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,14 @@
"\n",
"class SampleTask(gokart.TaskOnKart):\n",
" task_namespace = 'sample'\n",
" model_name = luigi.Parameter()\n",
" name = luigi.Parameter()\n",
" number = luigi.IntParameter()\n",
" \n",
" def require(self):\n",
" return\n",
"\n",
" def output(self):\n",
" return self.make_target(f'{self.model_name}/sample.pkl')\n",
" return self.make_target(f'{self.name}/sample.pkl')\n",
"\n",
" def run(self):\n",
" self.dump(f'this is sample output. model number: {self.number}')\n",
Expand All @@ -70,7 +70,7 @@
}
],
"source": [
"!cat ./example_task.py"
"!cat ./task.py"
]
},
{
Expand All @@ -80,12 +80,12 @@
"outputs": [],
"source": [
"# sample task run\n",
"!python example_task.py sample.SampleTask --model-name='EXAMPLE' --number=1 --local-scheduler 2> /dev/null\n",
"!python example_task.py sample.SampleTask --model-name='EXAMPLE' --number=2 --local-scheduler 2> /dev/null\n",
"!python example_task.py sample.SampleTask --model-name='EXAMPLE' --number=3 --local-scheduler 2> /dev/null\n",
"!python example_task.py sample.SampleTask --model-name='TEMP' --number=1 --local-scheduler 2> /dev/null\n",
"!python example_task.py sample.SampleTask --model-name='TEMP' --number=2 --local-scheduler 2> /dev/null\n",
"!python example_task.py sample.SecondTask --model-name='TEMP' --number=2 --param='RUN' --local-scheduler --local-temporary-directory='./resource' 2> /dev/null"
"!python task.py sample.SampleTask --name='EXAMPLE' --number=1 --local-scheduler 2> /dev/null\n",
"!python task.py sample.SampleTask --name='EXAMPLE' --number=2 --local-scheduler 2> /dev/null\n",
"!python task.py sample.SampleTask --name='EXAMPLE' --number=3 --local-scheduler 2> /dev/null\n",
"!python task.py sample.SampleTask --name='TEMP' --number=1 --local-scheduler 2> /dev/null\n",
"!python task.py sample.SampleTask --name='TEMP' --number=2 --local-scheduler 2> /dev/null\n",
"!python task.py sample.SecondTask --name='TEMP' --number=2 --param='RUN' --local-scheduler --local-temporary-directory='./resource' 2> /dev/null"
]
},
{
Expand All @@ -99,36 +99,36 @@
"text": [
"\u001b[0;34m./resources/\u001b[00m\n",
"├── \u001b[0;34mEXAMPLE\u001b[00m\n",
"│   ├── sample_222e2155cfed25bcda5234056cf1fa5c.pkl\n",
"│   ├── sample_9a30d39315e2db5a8db587544a3bd5c2.pkl\n",
"│   └── sample_af3d19290570cca7e6aa0a5a5dc534ea.pkl\n",
"│   ├── sample_84b0b9c5a39bce072271599c9f730660.pkl\n",
"│   ├── sample_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
"│   └── sample_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
"├── \u001b[0;34mSECOND_TASK\u001b[00m\n",
"│   └── task_6cf070abfbefc917a198521a26818c7a.pkl\n",
"│   └── task_ea1806322904199b2455d6e115c525ea.pkl\n",
"├── \u001b[0;34mTEMP\u001b[00m\n",
"│   ├── sample_c4cfb28a2370db69219de0acc802b721.pkl\n",
"│   └── sample_dc93d9e52a67bed92f72193bb7dd10a9.pkl\n",
"│   ├── sample_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
"│   └── sample_d57cff8074e2560896974850e5d3174d.pkl\n",
"└── \u001b[0;34mlog\u001b[00m\n",
" ├── \u001b[0;34mprocessing_time\u001b[00m\n",
" │   ├── SampleTask_222e2155cfed25bcda5234056cf1fa5c.pkl\n",
" │   ├── SampleTask_9a30d39315e2db5a8db587544a3bd5c2.pkl\n",
" │   ├── SampleTask_af3d19290570cca7e6aa0a5a5dc534ea.pkl\n",
" │   ├── SampleTask_c4cfb28a2370db69219de0acc802b721.pkl\n",
" │   ├── SampleTask_dc93d9e52a67bed92f72193bb7dd10a9.pkl\n",
" │   └── SecondTask_6cf070abfbefc917a198521a26818c7a.pkl\n",
" │   ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
" │   ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
" │   ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
" │   ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
" │   ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
" │   └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
" ├── \u001b[0;34mtask_log\u001b[00m\n",
" │   ├── SampleTask_222e2155cfed25bcda5234056cf1fa5c.pkl\n",
" │   ├── SampleTask_9a30d39315e2db5a8db587544a3bd5c2.pkl\n",
" │   ├── SampleTask_af3d19290570cca7e6aa0a5a5dc534ea.pkl\n",
" │   ├── SampleTask_c4cfb28a2370db69219de0acc802b721.pkl\n",
" │   ├── SampleTask_dc93d9e52a67bed92f72193bb7dd10a9.pkl\n",
" │   └── SecondTask_6cf070abfbefc917a198521a26818c7a.pkl\n",
" │   ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
" │   ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
" │   ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
" │   ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
" │   ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
" │   └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
" └── \u001b[0;34mtask_params\u001b[00m\n",
" ├── SampleTask_222e2155cfed25bcda5234056cf1fa5c.pkl\n",
" ├── SampleTask_9a30d39315e2db5a8db587544a3bd5c2.pkl\n",
" ├── SampleTask_af3d19290570cca7e6aa0a5a5dc534ea.pkl\n",
" ├── SampleTask_c4cfb28a2370db69219de0acc802b721.pkl\n",
" ├── SampleTask_dc93d9e52a67bed92f72193bb7dd10a9.pkl\n",
" └── SecondTask_6cf070abfbefc917a198521a26818c7a.pkl\n",
" ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
" ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
" ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
" ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
" ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
" └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
"\n",
"7 directories, 24 files\n"
]
Expand Down Expand Up @@ -166,7 +166,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 6/6 [00:00<00:00, 5551.69it/s]\n"
"100%|██████████| 6/6 [00:00<00:00, 5310.37it/s]\n"
]
}
],
Expand Down Expand Up @@ -230,65 +230,65 @@
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>SampleTask</td>\n",
" <td>2019-08-18 01:57:03.552976</td>\n",
" <td>{'model_name': 'EXAMPLE', 'number': '2'}</td>\n",
" <td>SecondTask</td>\n",
" <td>2019-08-18 10:15:48.289674</td>\n",
" <td>{'name': 'TEMP', 'number': '2', 'param': 'RUN'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>SampleTask</td>\n",
" <td>2019-08-18 01:57:06.890364</td>\n",
" <td>{'model_name': 'TEMP', 'number': '2'}</td>\n",
" <td>2019-08-18 10:15:46.210884</td>\n",
" <td>{'name': 'TEMP', 'number': '1'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>SampleTask</td>\n",
" <td>2019-08-18 01:57:04.874456</td>\n",
" <td>{'model_name': 'EXAMPLE', 'number': '3'}</td>\n",
" <td>2019-08-18 10:15:45.179609</td>\n",
" <td>{'name': 'EXAMPLE', 'number': '3'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>SampleTask</td>\n",
" <td>2019-08-18 01:57:05.879843</td>\n",
" <td>{'model_name': 'TEMP', 'number': '1'}</td>\n",
" <td>2019-08-18 10:15:47.255227</td>\n",
" <td>{'name': 'TEMP', 'number': '2'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>SampleTask</td>\n",
" <td>2019-08-18 01:57:02.499170</td>\n",
" <td>{'model_name': 'EXAMPLE', 'number': '1'}</td>\n",
" <td>2019-08-18 10:15:43.040046</td>\n",
" <td>{'name': 'EXAMPLE', 'number': '1'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>5</td>\n",
" <td>SecondTask</td>\n",
" <td>2019-08-18 01:57:07.918226</td>\n",
" <td>{'model_name': 'TEMP', 'number': '2', 'param': 'RUN'}</td>\n",
" <td>SampleTask</td>\n",
" <td>2019-08-18 10:15:44.158041</td>\n",
" <td>{'name': 'EXAMPLE', 'number': '2'}</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" task_id task_name last_modified \\\n",
"0 0 SampleTask 2019-08-18 01:57:03.552976 \n",
"1 1 SampleTask 2019-08-18 01:57:06.890364 \n",
"2 2 SampleTask 2019-08-18 01:57:04.874456 \n",
"3 3 SampleTask 2019-08-18 01:57:05.879843 \n",
"4 4 SampleTask 2019-08-18 01:57:02.499170 \n",
"5 5 SecondTask 2019-08-18 01:57:07.918226 \n",
"0 0 SecondTask 2019-08-18 10:15:48.289674 \n",
"1 1 SampleTask 2019-08-18 10:15:46.210884 \n",
"2 2 SampleTask 2019-08-18 10:15:45.179609 \n",
"3 3 SampleTask 2019-08-18 10:15:47.255227 \n",
"4 4 SampleTask 2019-08-18 10:15:43.040046 \n",
"5 5 SampleTask 2019-08-18 10:15:44.158041 \n",
"\n",
" task_params \n",
"0 {'model_name': 'EXAMPLE', 'number': '2'} \n",
"1 {'model_name': 'TEMP', 'number': '2'} \n",
"2 {'model_name': 'EXAMPLE', 'number': '3'} \n",
"3 {'model_name': 'TEMP', 'number': '1'} \n",
"4 {'model_name': 'EXAMPLE', 'number': '1'} \n",
"5 {'model_name': 'TEMP', 'number': '2', 'param': 'RUN'} "
" task_params \n",
"0 {'name': 'TEMP', 'number': '2', 'param': 'RUN'} \n",
"1 {'name': 'TEMP', 'number': '1'} \n",
"2 {'name': 'EXAMPLE', 'number': '3'} \n",
"3 {'name': 'TEMP', 'number': '2'} \n",
"4 {'name': 'EXAMPLE', 'number': '1'} \n",
"5 {'name': 'EXAMPLE', 'number': '2'} "
]
},
"execution_count": 8,
Expand All @@ -309,8 +309,8 @@
"## thunderbolt filter\n",
"2nd arg 'task_filters' is str or list. \n",
"task_filters is partial match word for example: \n",
" - 'Tag' -> GaussTag, NormalizeGaussTag, MaxwellTag, TagSingle,...\n",
" - ['Train', 'Tag'] -> TrainModel, TrainData,GaussTag, NormalizeGaussTag, MaxwellTag, TagSingle,..."
" - 'Tag' -> HogeTag, NormalizeHogeTag, TagTask, ...\n",
" - ['Train', 'Tag'] -> TrainModel, TrainData, HogeTag, NormalizeHogeTag, TagTask, ..."
]
},
{
Expand All @@ -322,7 +322,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 6/6 [00:00<00:00, 3927.25it/s]\n"
"100%|██████████| 6/6 [00:00<00:00, 3103.83it/s]\n"
]
}
],
Expand Down Expand Up @@ -367,74 +367,74 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2019-08-18 01:57:03.552976</td>\n",
" <td>9a30d39315e2db5a8db587544a3bd5c2</td>\n",
" <td>0</td>\n",
" <td>{'file_path': ['./resources/EXAMPLE/sample_9a30d39315e2db5a8db587544a3bd5c2.pkl']}</td>\n",
" <td>2019-08-18 10:15:46.210884</td>\n",
" <td>d57cff8074e2560896974850e5d3174d</td>\n",
" <td>1</td>\n",
" <td>{'file_path': ['./resources/TEMP/sample_d57cff8074e2560896974850e5d3174d.pkl']}</td>\n",
" <td>SampleTask</td>\n",
" <td>{'model_name': 'EXAMPLE', 'number': '2'}</td>\n",
" <td>{'name': 'TEMP', 'number': '1'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2019-08-18 01:57:06.890364</td>\n",
" <td>c4cfb28a2370db69219de0acc802b721</td>\n",
" <td>1</td>\n",
" <td>{'file_path': ['./resources/TEMP/sample_c4cfb28a2370db69219de0acc802b721.pkl']}</td>\n",
" <td>2019-08-18 10:15:45.179609</td>\n",
" <td>944fc52ef5011b71b5839f035f4d7e48</td>\n",
" <td>2</td>\n",
" <td>{'file_path': ['./resources/EXAMPLE/sample_944fc52ef5011b71b5839f035f4d7e48.pkl']}</td>\n",
" <td>SampleTask</td>\n",
" <td>{'model_name': 'TEMP', 'number': '2'}</td>\n",
" <td>{'name': 'EXAMPLE', 'number': '3'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2019-08-18 01:57:04.874456</td>\n",
" <td>af3d19290570cca7e6aa0a5a5dc534ea</td>\n",
" <td>2</td>\n",
" <td>{'file_path': ['./resources/EXAMPLE/sample_af3d19290570cca7e6aa0a5a5dc534ea.pkl']}</td>\n",
" <td>2019-08-18 10:15:47.255227</td>\n",
" <td>d05a2ab961781d3d8eca3e2e5f0d608b</td>\n",
" <td>3</td>\n",
" <td>{'file_path': ['./resources/TEMP/sample_d05a2ab961781d3d8eca3e2e5f0d608b.pkl']}</td>\n",
" <td>SampleTask</td>\n",
" <td>{'model_name': 'EXAMPLE', 'number': '3'}</td>\n",
" <td>{'name': 'TEMP', 'number': '2'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2019-08-18 01:57:05.879843</td>\n",
" <td>dc93d9e52a67bed92f72193bb7dd10a9</td>\n",
" <td>3</td>\n",
" <td>{'file_path': ['./resources/TEMP/sample_dc93d9e52a67bed92f72193bb7dd10a9.pkl']}</td>\n",
" <td>2019-08-18 10:15:43.040046</td>\n",
" <td>e883bcfad65f5fb68259d1cd4691f384</td>\n",
" <td>4</td>\n",
" <td>{'file_path': ['./resources/EXAMPLE/sample_e883bcfad65f5fb68259d1cd4691f384.pkl']}</td>\n",
" <td>SampleTask</td>\n",
" <td>{'model_name': 'TEMP', 'number': '1'}</td>\n",
" <td>{'name': 'EXAMPLE', 'number': '1'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2019-08-18 01:57:02.499170</td>\n",
" <td>222e2155cfed25bcda5234056cf1fa5c</td>\n",
" <td>4</td>\n",
" <td>{'file_path': ['./resources/EXAMPLE/sample_222e2155cfed25bcda5234056cf1fa5c.pkl']}</td>\n",
" <td>2019-08-18 10:15:44.158041</td>\n",
" <td>84b0b9c5a39bce072271599c9f730660</td>\n",
" <td>5</td>\n",
" <td>{'file_path': ['./resources/EXAMPLE/sample_84b0b9c5a39bce072271599c9f730660.pkl']}</td>\n",
" <td>SampleTask</td>\n",
" <td>{'model_name': 'EXAMPLE', 'number': '1'}</td>\n",
" <td>{'name': 'EXAMPLE', 'number': '2'}</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" last_modified task_hash task_id \\\n",
"0 2019-08-18 01:57:03.552976 9a30d39315e2db5a8db587544a3bd5c2 0 \n",
"1 2019-08-18 01:57:06.890364 c4cfb28a2370db69219de0acc802b721 1 \n",
"2 2019-08-18 01:57:04.874456 af3d19290570cca7e6aa0a5a5dc534ea 2 \n",
"3 2019-08-18 01:57:05.879843 dc93d9e52a67bed92f72193bb7dd10a9 3 \n",
"4 2019-08-18 01:57:02.499170 222e2155cfed25bcda5234056cf1fa5c 4 \n",
"0 2019-08-18 10:15:46.210884 d57cff8074e2560896974850e5d3174d 1 \n",
"1 2019-08-18 10:15:45.179609 944fc52ef5011b71b5839f035f4d7e48 2 \n",
"2 2019-08-18 10:15:47.255227 d05a2ab961781d3d8eca3e2e5f0d608b 3 \n",
"3 2019-08-18 10:15:43.040046 e883bcfad65f5fb68259d1cd4691f384 4 \n",
"4 2019-08-18 10:15:44.158041 84b0b9c5a39bce072271599c9f730660 5 \n",
"\n",
" task_log \\\n",
"0 {'file_path': ['./resources/EXAMPLE/sample_9a30d39315e2db5a8db587544a3bd5c2.pkl']} \n",
"1 {'file_path': ['./resources/TEMP/sample_c4cfb28a2370db69219de0acc802b721.pkl']} \n",
"2 {'file_path': ['./resources/EXAMPLE/sample_af3d19290570cca7e6aa0a5a5dc534ea.pkl']} \n",
"3 {'file_path': ['./resources/TEMP/sample_dc93d9e52a67bed92f72193bb7dd10a9.pkl']} \n",
"4 {'file_path': ['./resources/EXAMPLE/sample_222e2155cfed25bcda5234056cf1fa5c.pkl']} \n",
"0 {'file_path': ['./resources/TEMP/sample_d57cff8074e2560896974850e5d3174d.pkl']} \n",
"1 {'file_path': ['./resources/EXAMPLE/sample_944fc52ef5011b71b5839f035f4d7e48.pkl']} \n",
"2 {'file_path': ['./resources/TEMP/sample_d05a2ab961781d3d8eca3e2e5f0d608b.pkl']} \n",
"3 {'file_path': ['./resources/EXAMPLE/sample_e883bcfad65f5fb68259d1cd4691f384.pkl']} \n",
"4 {'file_path': ['./resources/EXAMPLE/sample_84b0b9c5a39bce072271599c9f730660.pkl']} \n",
"\n",
" task_name task_params \n",
"0 SampleTask {'model_name': 'EXAMPLE', 'number': '2'} \n",
"1 SampleTask {'model_name': 'TEMP', 'number': '2'} \n",
"2 SampleTask {'model_name': 'EXAMPLE', 'number': '3'} \n",
"3 SampleTask {'model_name': 'TEMP', 'number': '1'} \n",
"4 SampleTask {'model_name': 'EXAMPLE', 'number': '1'} "
" task_name task_params \n",
"0 SampleTask {'name': 'TEMP', 'number': '1'} \n",
"1 SampleTask {'name': 'EXAMPLE', 'number': '3'} \n",
"2 SampleTask {'name': 'TEMP', 'number': '2'} \n",
"3 SampleTask {'name': 'EXAMPLE', 'number': '1'} \n",
"4 SampleTask {'name': 'EXAMPLE', 'number': '2'} "
]
},
"execution_count": 10,
Expand Down Expand Up @@ -468,7 +468,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"this is sample output. model number: 1\n"
"this is sample output. model number: 2\n"
]
}
],
Expand All @@ -493,7 +493,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 6/6 [00:00<00:00, 6403.52it/s]\n"
"100%|██████████| 6/6 [00:00<00:00, 8200.01it/s]\n"
]
},
{
Expand All @@ -511,6 +511,13 @@
"tb = Thunderbolt(task_dir, 'Second')\n",
"tb.load(task_id=tb.get_task_df().sort_values('last_modified').task_id.iloc[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
4 changes: 2 additions & 2 deletions examples/example_task.py → examples/task.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@

class SampleTask(gokart.TaskOnKart):
task_namespace = 'sample'
model_name = luigi.Parameter()
name = luigi.Parameter()
number = luigi.IntParameter()

def require(self):
return

def output(self):
return self.make_target(f'{self.model_name}/sample.pkl')
return self.make_target(f'{self.name}/sample.pkl')

def run(self):
self.dump(f'this is sample output. model number: {self.number}')
Expand Down

0 comments on commit 756c9f0

Please sign in to comment.