Skip to content

Commit

Permalink
feat: Events: Git operations
Browse files Browse the repository at this point in the history
Does branch and commit management.

Signed-off-by: Samuel Giffard <[email protected]>
  • Loading branch information
Mulugruntz committed Feb 20, 2024
1 parent df5b078 commit 40289be
Show file tree
Hide file tree
Showing 6 changed files with 208 additions and 9 deletions.
4 changes: 4 additions & 0 deletions dvalin-tools/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,7 @@ DVALIN_S3_ACCESS_KEY=your_dvalin_s3_access_key
DVALIN_S3_SECRET_KEY=your_dvalin_s3_secret_key
DVALIN_CELERY_BROKER_URL=redis://redis:6379/0
DVALIN_CELERY_RESULT_BACKEND=redis://redis:6379/0
DVALIN_GIT_USER_EMAIL=your_git_user_email
DVALIN_GIT_USER_NAME=your_git_user_name
DVALIN_GIT_GITHUB_USERNAME=your_github_username
DVALIN_GIT_PRIVATE_ACCESS_TOKEN=your_git_private_access_token
21 changes: 12 additions & 9 deletions dvalin-tools/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -47,20 +47,23 @@ COPY --from=builder /usr/local/bin /usr/local/bin
# Copy application code from the builder stage
COPY --from=builder --chown=celeryuser:celeryuser /usr/src/app/dvalin_tools ./dvalin_tools

# Dependencies for the production stage
FROM python:3.11-alpine as prod-deps
# Runtime Stage (production)
FROM runtime-localdev as runtime-prod

ARG DVALIN_REPO_ROOT_DIR=${DVALIN_REPO_ROOT_DIR:-/usr/src/repo/dvalin-data}

USER root
# Install build dependencies
RUN apk fix && \
apk --no-cache --update add git git-lfs gpg less openssh patch && \
git lfs install

WORKDIR /usr/src/data
RUN git clone https://github.com/dval-in/dvalin-data.git
USER celeryuser

# Runtime Stage (production)
FROM runtime-localdev as runtime-prod
WORKDIR ${DVALIN_REPO_ROOT_DIR}

ARG DVALIN_REPO_ROOT_DIR=${DVALIN_REPO_ROOT_DIR:-/usr/src/repo/dvalin-data}
RUN git clone https://github.com/dval-in/dvalin-data.git ${DVALIN_REPO_ROOT_DIR}

WORKDIR /usr/src/app

COPY --from=prod-deps --chown=celeryuser:celeryuser /usr/src/data/dvalin-data ${DVALIN_REPO_ROOT_DIR}
COPY --chown=celeryuser:celeryuser __scraper_cache__ /usr/src/__scraper_cache__
COPY --chown=celeryuser:celeryuser __scraper_cache__ /usr/src/__scraper_cache__
4 changes: 4 additions & 0 deletions dvalin-tools/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ services:
- DVALIN_S3_ENDPOINT=${DVALIN_S3_ENDPOINT}
- DVALIN_S3_ACCESS_KEY=${DVALIN_S3_ACCESS_KEY}
- DVALIN_S3_SECRET_KEY=${DVALIN_S3_SECRET_KEY}
- DVALIN_GIT_USER_EMAIL=${DVALIN_GIT_USER_EMAIL}
- DVALIN_GIT_USER_NAME=${DVALIN_GIT_USER_NAME}
- DVALIN_GIT_GITHUB_USERNAME=${DVALIN_GIT_GITHUB_USERNAME}
- DVALIN_GIT_PRIVATE_ACCESS_TOKEN=${DVALIN_GIT_PRIVATE_ACCESS_TOKEN}
- DVALIN_REPO_ROOT_DIR=/usr/src/repo/dvalin-data
depends_on:
- redis
Expand Down
5 changes: 5 additions & 0 deletions dvalin-tools/dvalin_tools/agents/event_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from dvalin_tools.lib.fs_lock import fs_lock
from dvalin_tools.lib.languages import LANGUAGE_CODE_TO_DIR, LanguageCode
from dvalin_tools.lib.repository import loop_end_with_changes, loop_start, prog_init
from dvalin_tools.lib.settings import DvalinSettings
from dvalin_tools.models.common import Game
from dvalin_tools.models.events import EventFile, MessageType
Expand All @@ -27,6 +28,7 @@

@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs) -> None:
prog_init(settings.repo_root_dir)
sender.add_periodic_task(crontab(minute="*/1"), event_flow.s())


Expand Down Expand Up @@ -79,6 +81,7 @@ def process_new_events(there_are_new_events: bool) -> None:

async def process_new_events_async() -> None:
print("Processing new events async")
loop_start(settings.repo_root_dir)
data_dir = settings.data_path
latest_post_id = get_last_event_post_id(data_dir)
events = await get_all_events(
Expand All @@ -88,6 +91,8 @@ async def process_new_events_async() -> None:
modified_event_files = write_events(events, data_dir)
await update_event_files(modified_event_files)
print("New events processed")
if modified_event_files:
loop_end_with_changes(settings.repo_root_dir, modified_event_files)


if __name__ == "__main__":
Expand Down
169 changes: 169 additions & 0 deletions dvalin-tools/dvalin_tools/lib/repository.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
import re
from datetime import datetime
from pathlib import Path
from shlex import split
from subprocess import PIPE, Popen

from httpx import URL

from dvalin_tools.lib.settings import DvalinSettings, GitSettings

AUTO_DATA_EVENT_BRANCH_PREFIX = "auto-data-event"

# Regex for branch that are handled by the bot
RE_AUTO_DATA_EVENT_BRANCH = re.compile(
rf"^(origin/)?{AUTO_DATA_EVENT_BRANCH_PREFIX}/(.*)$"
)

git_settings: GitSettings = DvalinSettings().git


class Repository:
def __init__(self, path: Path, master_name: str = "main") -> None:
self.path = path
self.temp_branches = []
self.master_name = master_name

def _execute(self, command: str | list[str]) -> str:
cmd_as_list = split(command) if isinstance(command, str) else command
process = Popen(cmd_as_list, cwd=self.path, stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
raise ValueError(f"Command failed: {command}\n{stderr.decode()}")
return stdout.decode()

def initialize_git_config(self) -> None:
self._execute(f"git config user.email {git_settings.user_email}")
self._execute(f"git config user.name {git_settings.user_name}")
self._execute(f"git remote set-url origin {self.get_remote_url_with_auth()}")
print(
f"Initialized git config with `{git_settings.user_name} <{git_settings.user_email}>`"
)

def get_current_branch(self) -> str:
return self._execute("git rev-parse --abbrev-ref HEAD").strip()

def create_temporary_branch(self) -> str:
name = f"tmp-{datetime.now():%Y%m%d_%H%M%S}"
self._execute(f"git checkout -b {name}")
self.temp_branches.append(name)
return name

def checkout_remote_branch(self, branch_name: str) -> None:
self._execute(f"git checkout -b {branch_name} origin/{branch_name}")

def rename_current_branch(self, new_name: str) -> None:
self._execute(f"git branch -m {new_name}")

def reset_to_master(self) -> None:
"""Go to master, and make sure it's up-to-date with origin/master."""
self._execute(f"git checkout {self.master_name}")
self._execute(f"git pull origin {self.master_name}")

def get_all_branches(self) -> list[str]:
return self._execute("git branch -a --format='%(refname:short)'").splitlines()

def get_remote_branches(self) -> list[str]:
return self._execute("git branch -r --format='%(refname:short)'").splitlines()

@staticmethod
def filter_auto_branches(branches: list[str]) -> list[str]:
return [
branch for branch in branches if RE_AUTO_DATA_EVENT_BRANCH.match(branch)
]

def get_all_auto_branches(self) -> list[str]:
return self.filter_auto_branches(self.get_all_branches())

def get_remote_auto_branches(self) -> list[str]:
return self.filter_auto_branches(self.get_remote_branches())

def destroy_all_local_branches(self) -> None:
current_branch = self.get_current_branch()
local_branches = self._execute(
"git branch --format='%(refname:short)'"
).splitlines()
for branch in local_branches:
if branch not in (self.master_name, current_branch):
self._execute(f"git branch -D {branch}")

def commit_and_push(self, file_list: list[Path] | None = None) -> None:
if file_list is None:
self._execute("git add .")
else:
self._execute(
[
"git",
"add",
*[str(file.relative_to(self.path)) for file in file_list],
]
)
self._execute(
f"git commit -m 'feat: Auto-data: Event: {datetime.now():%Y-%m-%d %H:%M:%S}'"
)
self._execute(f"git push origin {self.get_current_branch()}")

def get_remote_url(self) -> URL:
return URL(self._execute("git remote get-url origin").strip())

def get_remote_url_with_auth(self) -> URL:
url_origin = self.get_remote_url()
username = git_settings.github_username
token = git_settings.private_access_token
return url_origin.copy_with(username=username, password=token)

@staticmethod
def generate_auto_branch_name() -> str:
return f"{AUTO_DATA_EVENT_BRANCH_PREFIX}/{datetime.now():%Y%m%d_%H%M%S}"

def go_to_latest_auto_branch(self) -> None:
branches = self.get_all_auto_branches()
if not branches:
self.create_temporary_branch()
return
remote_branches = [
branch for branch in branches if branch.startswith("origin/")
]
if remote_branches:
local_name = remote_branches[0].removeprefix("origin/")
self.checkout_remote_branch(local_name)


def prog_init(path: Path) -> None:
repo = Repository(path)
repo.initialize_git_config()
repo.reset_to_master()
repo.destroy_all_local_branches()
remote_auto_branches = repo.get_remote_auto_branches()
if remote_auto_branches:
# we use the latest remote branch, pull it and check it out
latest_remote = remote_auto_branches[-1].removeprefix("origin/")
repo.checkout_remote_branch(latest_remote)
print(f"Checked out remote branch {latest_remote}")


def loop_start(path: Path) -> None:
repo = Repository(path)
# if auto remote exists AND it's different from current local branch, pull, check it out
remote_auto_branches = repo.get_remote_auto_branches()
if remote_auto_branches:
latest_remote = remote_auto_branches[-1].removeprefix("origin/")
current_branch = repo.get_current_branch()
if current_branch != latest_remote:
repo.checkout_remote_branch(latest_remote)
print(f"Checked out remote branch {latest_remote}")
return

# we create a new branch
temp_b_name = repo.create_temporary_branch()
print(f"Created a new temporary branch {temp_b_name}")


def loop_end_with_changes(path: Path, files: list[Path]) -> None:
repo = Repository(path)
current_branch = repo.get_current_branch()
if current_branch.startswith("tmp-"):
repo.rename_current_branch(repo.generate_auto_branch_name())
print(f"Renamed branch to {repo.get_current_branch()}")

repo.commit_and_push(files)
14 changes: 14 additions & 0 deletions dvalin-tools/dvalin_tools/lib/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,19 @@ class CelerySettings(BaseSettings):
result_backend: str


class GitSettings(BaseSettings):
model_config = SettingsConfigDict(
env_file=ROOT_DIR_DVALIN_TOOLS / ".env",
env_file_encoding="utf-8",
env_prefix=f"{PROJECT_PREFIX}_GIT_",
)

user_email: str
user_name: str
github_username: str
private_access_token: str


class DvalinSettings(BaseSettings):
model_config = SettingsConfigDict(
env_file=ROOT_DIR_DVALIN_TOOLS / ".env",
Expand All @@ -40,6 +53,7 @@ class DvalinSettings(BaseSettings):

s3: S3Settings = Field(default_factory=S3Settings)
celery: CelerySettings = Field(default_factory=CelerySettings)
git: GitSettings = Field(default_factory=GitSettings)
repo_root_dir: Path = Field(
default=ROOT_DIR_DVALIN_DATA,
description="The root directory of the dvalin-data repository",
Expand Down

0 comments on commit 40289be

Please sign in to comment.