Tags (#649)

* initial tags

* add tags endpoint for listeners with tests

* update tag

* expand to agents

* fixing some things

* fix agent tags, fix listener api tests

* abstract tag tests

* abstract tags endpoints

* add plugin task tags

* add tags to credentials

* add tags to downloads

* add events for tags

* refactor. working on list endpoint

* more cleanup, basic list endpoint

* add validation

* add ordering to tags

* remove starkiller submodule

* remove todo

* more tags tests. add tags to download filters

* fix stager test cleanup

* add label to tag dto

* fix user filters

* fix reset test

empire/server/api/app.py

@@ -66,6 +66,7 @@ def initialize(secure: bool = False, ip: str = "0.0.0.0", port: int = 1337):
     from empire.server.api.v2.plugin import plugin_api, plugin_task_api
     from empire.server.api.v2.profile import profile_api
     from empire.server.api.v2.stager import stager_api, stager_template_api
+    from empire.server.api.v2.tag import tag_api
     from empire.server.api.v2.user import user_api
     from empire.server.server import main
 
@@ -97,6 +98,7 @@ def shutdown_event():
     v2App.include_router(meta_api.router)
     v2App.include_router(plugin_task_api.router)
     v2App.include_router(plugin_api.router)
+    v2App.include_router(tag_api.router)
 
     v2App.add_middleware(
         EmpireCORSMiddleware,

empire/server/api/v2/agent/agent_api.py

@@ -24,6 +24,7 @@
     NotFoundResponse,
     OrderDirection,
 )
+from empire.server.api.v2.tag import tag_api
 from empire.server.core.config import empire_config
 from empire.server.core.db import models
 from empire.server.server import main
@@ -50,6 +51,9 @@ async def get_agent(uid: str, db: Session = Depends(get_db)):
     raise HTTPException(404, f"Agent not found for id {uid}")
 
 
+tag_api.add_endpoints_to_taggable(router, "/{uid}/tags", get_agent)
+
+
 @router.get("/checkins", response_model=AgentCheckIns)
 def read_agent_checkins_all(
     db: Session = Depends(get_db),

empire/server/api/v2/agent/agent_dto.py

@@ -5,6 +5,7 @@
 from pydantic import BaseModel
 
 from empire.server.api.v2.shared_dto import PROXY_ID
+from empire.server.api.v2.tag.tag_dto import Tag, domain_to_dto_tag
 from empire.server.core.db import models
 
 
@@ -48,6 +49,7 @@ def domain_to_dto_agent(agent: models.Agent):
         archived=agent.archived,
         # Could make this a typed class later to match the schema
         proxies=to_proxy_dto(agent.proxies),
+        tags=list(map(lambda x: domain_to_dto_tag(x), agent.tags)),
     )
 
 
@@ -111,6 +113,7 @@ class Agent(BaseModel):
     archived: bool
     stale: bool
     proxies: Optional[Dict]
+    tags: List[Tag]
 
 
 class Agents(BaseModel):

empire/server/api/v2/agent/agent_task_api.py

@@ -38,6 +38,8 @@
     NotFoundResponse,
     OrderDirection,
 )
+from empire.server.api.v2.tag import tag_api
+from empire.server.api.v2.tag.tag_dto import TagStr
 from empire.server.core.agent_service import AgentService
 from empire.server.core.agent_task_service import AgentTaskService
 from empire.server.core.db import models
@@ -83,6 +85,9 @@ async def get_task(
     )
 
 
+tag_api.add_endpoints_to_taggable(router, "/{agent_id}/tasks/{uid}/tags", get_task)
+
+
 @router.get("/tasks", response_model=AgentTasks)
 async def read_tasks_all_agents(
     limit: int = -1,
@@ -96,13 +101,15 @@ async def read_tasks_all_agents(
     status: Optional[AgentTaskStatus] = None,
     agents: Optional[List[str]] = Query(None),
     users: Optional[List[int]] = Query(None),
+    tags: Optional[List[TagStr]] = Query(None),
     query: Optional[str] = None,
     db: Session = Depends(get_db),
 ):
     tasks, total = agent_task_service.get_tasks(
         db,
         agents=agents,
         users=users,
+        tags=tags,
         limit=limit,
         offset=(page - 1) * limit,
         include_full_input=include_full_input,
@@ -145,6 +152,7 @@ async def read_tasks(
     order_direction: OrderDirection = OrderDirection.desc,
     status: Optional[AgentTaskStatus] = None,
     users: Optional[List[int]] = Query(None),
+    tags: Optional[List[TagStr]] = Query(None),
     db: Session = Depends(get_db),
     db_agent: models.Agent = Depends(get_agent),
     query: Optional[str] = None,
@@ -153,6 +161,7 @@ async def read_tasks(
         db,
         agents=[db_agent.session_id],
         users=users,
+        tags=tags,
         limit=limit,
         offset=(page - 1) * limit,
         include_full_input=include_full_input,

empire/server/api/v2/agent/agent_task_dto.py

@@ -8,6 +8,7 @@
     DownloadDescription,
     domain_to_dto_download_description,
 )
+from empire.server.api.v2.tag.tag_dto import Tag, domain_to_dto_tag
 from empire.server.core.db import models
 
 
@@ -41,6 +42,7 @@ def domain_to_dto_task(
         status=task.status,
         created_at=task.created_at,
         updated_at=task.updated_at,
+        tags=list(map(lambda x: domain_to_dto_tag(x), task.tags)),
     )
 
 
@@ -59,6 +61,7 @@ class AgentTask(BaseModel):
     status: models.AgentTaskStatus
     created_at: datetime
     updated_at: datetime
+    tags: List[Tag]
 
 
 class AgentTasks(BaseModel):

empire/server/api/v2/credential/credential_api.py

@@ -16,6 +16,7 @@
 )
 from empire.server.api.v2.shared_dependencies import get_db
 from empire.server.api.v2.shared_dto import BadRequestResponse, NotFoundResponse
+from empire.server.api.v2.tag import tag_api
 from empire.server.core.db import models
 from empire.server.server import main
 
@@ -41,6 +42,9 @@ async def get_credential(uid: int, db: Session = Depends(get_db)):
     raise HTTPException(404, f"Credential not found for id {uid}")
 
 
+tag_api.add_endpoints_to_taggable(router, "/{uid}/tags", get_credential)
+
+
 @router.get("/{uid}", response_model=Credential)
 async def read_credential(
     uid: int, db_credential: models.Credential = Depends(get_credential)

empire/server/api/v2/credential/credential_dto.py

@@ -3,6 +3,8 @@
 
 from pydantic import BaseModel
 
+from empire.server.api.v2.tag.tag_dto import Tag, domain_to_dto_tag
+
 
 def domain_to_dto_credential(credential):
     return Credential(
@@ -17,6 +19,7 @@ def domain_to_dto_credential(credential):
         notes=credential.notes,
         created_at=credential.created_at,
         updated_at=credential.updated_at,
+        tags=list(map(lambda x: domain_to_dto_tag(x), credential.tags)),
     )
 
 
@@ -32,6 +35,7 @@ class Credential(BaseModel):
     notes: Optional[str]
     created_at: datetime
     updated_at: datetime
+    tags: List[Tag]
 
 
 class Credentials(BaseModel):

empire/server/api/v2/download/download_api.py

@@ -20,6 +20,8 @@
     NotFoundResponse,
     OrderDirection,
 )
+from empire.server.api.v2.tag import tag_api
+from empire.server.api.v2.tag.tag_dto import TagStr
 from empire.server.core.db import models
 from empire.server.server import main
 
@@ -59,8 +61,12 @@ async def download_download(
     return FileResponse(db_download.location, filename=filename)
 
 
+tag_api.add_endpoints_to_taggable(router, "/{uid}/tags", get_download)
+
+
 @router.get(
     "/{uid}",
+    response_model=Download,
 )
 async def read_download(
     uid: int,
@@ -79,10 +85,12 @@ async def read_downloads(
     order_by: DownloadOrderOptions = DownloadOrderOptions.updated_at,
     query: Optional[str] = None,
     sources: Optional[List[DownloadSourceFilter]] = Query(None),
+    tags: Optional[List[TagStr]] = Query(None),
 ):
     downloads, total = download_service.get_all(
         db=db,
         download_types=sources,
+        tags=tags,
         q=query,
         limit=limit,
         offset=(page - 1) * limit,

empire/server/api/v2/download/download_dto.py

@@ -4,6 +4,8 @@
 
 from pydantic import BaseModel
 
+from empire.server.api.v2.tag.tag_dto import Tag, domain_to_dto_tag
+
 
 def removeprefix(value: str, prefix: str) -> str:
     if value.startswith(prefix):
@@ -21,6 +23,7 @@ def domain_to_dto_download(download):
         size=download.size,
         created_at=download.created_at,
         updated_at=download.updated_at,
+        tags=list(map(lambda x: domain_to_dto_tag(x), download.tags)),
     )
 
 
@@ -46,6 +49,7 @@ class Download(BaseModel):
     size: int
     created_at: datetime
     updated_at: datetime
+    tags: List[Tag]
 
 
 class Downloads(BaseModel):

empire/server/api/v2/listener/listener_api.py

@@ -14,6 +14,7 @@
 )
 from empire.server.api.v2.shared_dependencies import get_db
 from empire.server.api.v2.shared_dto import BadRequestResponse, NotFoundResponse
+from empire.server.api.v2.tag import tag_api
 from empire.server.core.db import models
 from empire.server.server import main
 
@@ -39,6 +40,9 @@ async def get_listener(uid: int, db: Session = Depends(get_db)):
     raise HTTPException(404, f"Listener not found for id {uid}")
 
 
+tag_api.add_endpoints_to_taggable(router, "/{uid}/tags", get_listener)
+
+
 @router.get("/{uid}", response_model=Listener)
 async def read_listener(uid: int, db_listener: models.Listener = Depends(get_listener)):
     return domain_to_dto_listener(db_listener)

empire/server/api/v2/listener/listener_dto.py

@@ -4,6 +4,7 @@
 from pydantic import BaseModel
 
 from empire.server.api.v2.shared_dto import Author, CustomOptionSchema, to_value_type
+from empire.server.api.v2.tag.tag_dto import Tag, domain_to_dto_tag
 
 
 def domain_to_dto_template(listener, uid: str):
@@ -59,6 +60,7 @@ def domain_to_dto_listener(listener):
         enabled=listener.enabled,
         options=options,
         created_at=listener.created_at,
+        tags=list(map(lambda x: domain_to_dto_tag(x), listener.tags)),
     )
 
 
@@ -249,6 +251,7 @@ class Listener(BaseModel):
     template: str
     options: Dict[str, str]
     created_at: datetime
+    tags: List[Tag]
 
 
 class Listeners(BaseModel):

empire/server/api/v2/plugin/plugin_task_api.py

@@ -19,6 +19,8 @@
     NotFoundResponse,
     OrderDirection,
 )
+from empire.server.api.v2.tag import tag_api
+from empire.server.api.v2.tag.tag_dto import TagStr
 from empire.server.core.db import models
 from empire.server.core.db.models import PluginTaskStatus
 from empire.server.core.download_service import DownloadService
@@ -59,6 +61,9 @@ async def get_task(uid: int, db: Session = Depends(get_db), plugin=Depends(get_p
     )
 
 
+tag_api.add_endpoints_to_taggable(router, "/{plugin_id}/tasks/{uid}/tags", get_task)
+
+
 @router.get("/tasks", response_model=PluginTasks)
 async def read_tasks_all_plugins(
     limit: int = -1,
@@ -71,13 +76,15 @@ async def read_tasks_all_plugins(
     status: Optional[PluginTaskStatus] = None,
     plugins: Optional[List[str]] = Query(None),
     users: Optional[List[int]] = Query(None),
+    tags: Optional[List[TagStr]] = Query(None),
     query: Optional[str] = None,
     db: Session = Depends(get_db),
 ):
     tasks, total = plugin_service.get_tasks(
         db,
         plugins=plugins,
         users=users,
+        tags=tags,
         limit=limit,
         offset=(page - 1) * limit,
         include_full_input=include_full_input,
@@ -116,6 +123,7 @@ async def read_tasks(
     order_direction: OrderDirection = OrderDirection.desc,
     status: Optional[PluginTaskStatus] = None,
     users: Optional[List[int]] = Query(None),
+    tags: Optional[List[TagStr]] = Query(None),
     db: Session = Depends(get_db),
     plugin=Depends(get_plugin),
     query: Optional[str] = None,
@@ -124,6 +132,7 @@ async def read_tasks(
         db,
         plugins=[plugin.info["Name"]],
         users=users,
+        tags=tags,
         limit=limit,
         offset=(page - 1) * limit,
         include_full_input=include_full_input,

empire/server/api/v2/plugin/plugin_task_dto.py

@@ -8,6 +8,7 @@
     DownloadDescription,
     domain_to_dto_download_description,
 )
+from empire.server.api.v2.tag.tag_dto import Tag, domain_to_dto_tag
 from empire.server.core.db import models
 
 
@@ -37,6 +38,7 @@ def domain_to_dto_plugin_task(
         status=task.status,
         created_at=task.created_at,
         updated_at=task.updated_at,
+        tags=list(map(lambda x: domain_to_dto_tag(x), task.tags)),
     )
 
 
@@ -52,6 +54,7 @@ class PluginTask(BaseModel):
     status: Optional[models.PluginTaskStatus]
     created_at: datetime
     updated_at: datetime
+    tags: List[Tag]
 
 
 class PluginTasks(BaseModel):