From 22ebac30eafa502f584f0c9122953a6148927f21 Mon Sep 17 00:00:00 2001 From: Arash Date: Thu, 31 Oct 2024 11:47:41 +0100 Subject: [PATCH 1/4] Update GitHub Actions workflows to disable credential persistence and validate PR number input --- .github/workflows/preview.yml | 7 ++++++- .github/workflows/publish_content.yml | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 6c0061f..966bdaf 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -25,13 +25,18 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set environment variables run: | if [[ "${{ github.event.number }}" ]]; then echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_ENV - else + elif [[ "${{ github.event.inputs.pr_number }}" =~ ^[0-9]+$ ]]; then echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV + else + echo "Invalid PR number!" >&2 + exit 1 fi echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV echo "MASTODON_ACCESS_TOKEN=${{ secrets.MASTODON_ACCESS_TOKEN }}" >> $GITHUB_ENV diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml index c43215e..e4c930f 100644 --- a/.github/workflows/publish_content.yml +++ b/.github/workflows/publish_content.yml @@ -19,6 +19,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set environment variables run: | From ce11f7dcaedb8c3733d6c3464ea98b5960bd34a1 Mon Sep 17 00:00:00 2001 From: Arash Date: Thu, 31 Oct 2024 12:03:17 +0100 Subject: [PATCH 2/4] move ${{ github.event.inputs.pr_number }} out of inline code --- .github/workflows/preview.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 966bdaf..da7336a 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -29,11 +29,13 @@ jobs: persist-credentials: false - name: Set environment variables + env: + pr_number: ${{ github.event.inputs.pr_number }} run: | if [[ "${{ github.event.number }}" ]]; then echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_ENV - elif [[ "${{ github.event.inputs.pr_number }}" =~ ^[0-9]+$ ]]; then - echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV + elif [[ "$pr_number" =~ ^[0-9]+$ ]]; then + echo "PR_NUMBER=$pr_number" >> $GITHUB_ENV else echo "Invalid PR number!" >&2 exit 1 From 3d8ba7339d1b4354d98ed1320240dd03a457889a Mon Sep 17 00:00:00 2001 From: Arash Date: Wed, 6 Nov 2024 18:51:01 +0100 Subject: [PATCH 3/4] move workflow logic into the python file and create one unit workflow to do all --- .github/actions/galaxy-social/action.yml | 77 -------------- .github/workflows/galaxy_social.yml | 52 +++++++++ .github/workflows/preview.yml | 57 ---------- .github/workflows/publish_content.yml | 40 ------- README.md | 6 +- github_run.py | 128 ++++++++++++++--------- lib/galaxy_social.py | 10 +- 7 files changed, 138 insertions(+), 232 deletions(-) delete mode 100644 .github/actions/galaxy-social/action.yml create mode 100644 .github/workflows/galaxy_social.yml delete mode 100644 .github/workflows/preview.yml delete mode 100644 .github/workflows/publish_content.yml diff --git a/.github/actions/galaxy-social/action.yml b/.github/actions/galaxy-social/action.yml deleted file mode 100644 index 457312c..0000000 --- a/.github/actions/galaxy-social/action.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Galaxy Social composite action -description: Perform common steps for getting changed files, setting up the environment, and running the main logic. -inputs: - files: - description: "File patterns to check for changes" - default: "posts/**" - - preview: - description: "Whether to create a preview or not" - default: "false" - -runs: - using: "composite" - steps: - - name: Get changed files in posts folder - id: get_changed_files - uses: tj-actions/changed-files@v44 - with: - files: ${{ inputs.files }} - json: "true" - - - name: get published files cache - if: steps.get_changed_files.outputs.any_changed == 'true' - shell: bash - run: | - if ! git ls-remote --heads origin | grep -q "refs/heads/processed_files"; then - git checkout --orphan processed_files - git rm -rf . - echo "{}" > processed_files.json - git add processed_files.json - git config user.name 'github-actions[bot]' - git config user.email 'github-actions[bot]@users.noreply.github.com' - git commit -m "Initialize processed_files branch with processed_files.json" - git push origin processed_files - git checkout main - fi - git fetch origin processed_files:processed_files - git checkout processed_files -- processed_files.json - - - name: Set up Python - if: steps.get_changed_files.outputs.any_changed == 'true' - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install dependencies - if: steps.get_changed_files.outputs.any_changed == 'true' - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - - name: Create env variables for changed files - if: steps.get_changed_files.outputs.any_changed == 'true' - shell: bash - run: | - echo "CHANGED_FILES=${{ steps.get_changed_files.outputs.all_changed_files }}" >> $GITHUB_ENV - - - name: Run script to create preview - if: steps.get_changed_files.outputs.any_changed == 'true' && inputs.preview == 'true' - shell: bash - run: python -u github_run.py --preview - - - name: Run script to publish contents - if: steps.get_changed_files.outputs.any_changed == 'true' && inputs.preview == 'false' - shell: bash - run: python -u github_run.py --json-out processed_files.json - - - name: Commit changes - if: steps.get_changed_files.outputs.any_changed == 'true' && inputs.preview == 'false' - uses: stefanzweifel/git-auto-commit-action@v5 - with: - file_pattern: "processed_files.json" - branch: "processed_files" - - - name: Checkout again to prevent errors on post processing - uses: actions/checkout@v4 diff --git a/.github/workflows/galaxy_social.yml b/.github/workflows/galaxy_social.yml new file mode 100644 index 0000000..2f3c23f --- /dev/null +++ b/.github/workflows/galaxy_social.yml @@ -0,0 +1,52 @@ +name: Create Preview + +on: + pull_request_target: + branches: [main] + types: [opened, synchronize, reopened, closed] + paths: + - "posts/**" + + workflow_dispatch: + inputs: + pr_number: + description: "The number of the pull request to create preview for" + required: true + +jobs: + manage_content: + name: Manage Content + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + actions: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Run script to create preview or publish contents + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MASTODON_ACCESS_TOKEN: ${{ secrets.MASTODON_ACCESS_TOKEN }} + BLUESKY_PASSWORD: ${{ secrets.BLUESKY_PASSWORD }} + MATRIX_ACCESS_TOKEN: ${{ secrets.MATRIX_ACCESS_TOKEN }} + SLACK_ACCESS_TOKEN: ${{ secrets.SLACK_ACCESS_TOKEN }} + MASTODON_EU_FR_TOKEN: ${{ secrets.MASTODON_EU_FR_TOKEN }} + LINKEDIN_ACCESS_TOKEN: ${{ secrets.LINKEDIN_ACCESS_TOKEN }} + BRC_MASTODON_ACCESS_TOKEN: ${{ secrets.BRC_MASTODON_ACCESS_TOKEN }} + BRC_BLUESKY_PASSWORD: ${{ secrets.BRC_BLUESKY_PASSWORD }} + BRC_LINKEDIN_ACCESS_TOKEN: ${{ secrets.BRC_LINKEDIN_ACCESS_TOKEN }} + run: python -u github_run.py diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml deleted file mode 100644 index da7336a..0000000 --- a/.github/workflows/preview.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Create Preview - -on: - pull_request_target: - branches: [main] - types: [opened, synchronize, reopened] - paths: - - "posts/**" - - workflow_dispatch: - inputs: - pr_number: - description: "The number of the pull request to handle" - required: true - type: number - -jobs: - preview: - name: Preview - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set environment variables - env: - pr_number: ${{ github.event.inputs.pr_number }} - run: | - if [[ "${{ github.event.number }}" ]]; then - echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_ENV - elif [[ "$pr_number" =~ ^[0-9]+$ ]]; then - echo "PR_NUMBER=$pr_number" >> $GITHUB_ENV - else - echo "Invalid PR number!" >&2 - exit 1 - fi - echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV - echo "MASTODON_ACCESS_TOKEN=${{ secrets.MASTODON_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "BLUESKY_PASSWORD=${{ secrets.BLUESKY_PASSWORD }}" >> $GITHUB_ENV - echo "MATRIX_ACCESS_TOKEN=${{ secrets.MATRIX_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "SLACK_ACCESS_TOKEN=${{ secrets.SLACK_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "MASTODON_EU_FR_TOKEN=${{ secrets.MASTODON_EU_FR_TOKEN }}" >> $GITHUB_ENV - echo "LINKEDIN_ACCESS_TOKEN=${{ secrets.LINKEDIN_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "BRC_MASTODON_ACCESS_TOKEN=${{ secrets.BRC_MASTODON_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "BRC_BLUESKY_PASSWORD=${{ secrets.BRC_BLUESKY_PASSWORD }}" >> $GITHUB_ENV - echo "BRC_LINKEDIN_ACCESS_TOKEN=${{ secrets.BRC_LINKEDIN_ACCESS_TOKEN }}" >> $GITHUB_ENV - - - name: Run composite action - uses: ./.github/actions/galaxy-social - with: - preview: "true" diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml deleted file mode 100644 index e4c930f..0000000 --- a/.github/workflows/publish_content.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Publish Content - -on: - pull_request_target: - branches: [main] - types: [closed] - paths: - - "posts/**" - -jobs: - publish: - if: github.event.pull_request.merged == true - name: Publish content - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - actions: write - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set environment variables - run: | - echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_ENV - echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV - echo "MASTODON_ACCESS_TOKEN=${{ secrets.MASTODON_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "BLUESKY_PASSWORD=${{ secrets.BLUESKY_PASSWORD }}" >> $GITHUB_ENV - echo "MATRIX_ACCESS_TOKEN=${{ secrets.MATRIX_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "SLACK_ACCESS_TOKEN=${{ secrets.SLACK_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "MASTODON_EU_FR_TOKEN=${{ secrets.MASTODON_EU_FR_TOKEN }}" >> $GITHUB_ENV - echo "LINKEDIN_ACCESS_TOKEN=${{ secrets.LINKEDIN_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "BRC_MASTODON_ACCESS_TOKEN=${{ secrets.BRC_MASTODON_ACCESS_TOKEN }}" >> $GITHUB_ENV - echo "BRC_BLUESKY_PASSWORD=${{ secrets.BRC_BLUESKY_PASSWORD }}" >> $GITHUB_ENV - echo "BRC_LINKEDIN_ACCESS_TOKEN=${{ secrets.BRC_LINKEDIN_ACCESS_TOKEN }}" >> $GITHUB_ENV - - - name: Run composite action - uses: ./.github/actions/galaxy-social diff --git a/README.md b/README.md index 64c7bf1..2436095 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,9 @@ For images just drag and drop them here. they will look like this: 4. **Create a Pull Request:** Once your post is ready, [create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request?tool=webui#creating-the-pull-request) to the main branch from another branch or [from your fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) -5. **Preview and Review:** After each pull request, the "Create Preview" GitHub action will run. It will generate previews of the content as it would appear on each platform listed under "media" in a comment to the pull request and highlight any errors that need to be fixed before merging. +5. **Preview and Review:** After each pull request, the GitHub action will run and will generate previews of the content as it would appear on each platform listed under "media" in a comment to the pull request and highlight any errors that need to be fixed before merging. -6. **Publish Your Content:** Upon merging the pull request, the "Publish Content" GitHub action will run. The results will be added to `processed_files.json` in the processed_files branch. +6. **Publish Your Content:** Upon merging the pull request, the GitHub action will run. The results will be added to `processed_files.json` in the processed_files branch. By following these steps, you can effectively create and publish posts on Galaxy Social. @@ -106,7 +106,7 @@ The `name` is then used in the `media` tag in the post file (posts/\*.md) to det 5. **Enable the Plugin**: Simply set `enabled: true` to enable the new social media platform. This ensures that it will be implemented when creating posts. -6. **Update publish_content.yml**: Finally, update the `publish_content.yml` file to include an environment variable referencing the token saved in GitHub secrets. Use the following template: (Don't put the prefixed `$` in here) +6. **Update galaxy_social.yml**: Finally, update the `galaxy_social.yml` file to include an environment variable referencing the token saved in GitHub secrets. Use the following template: (Don't put the prefixed `$` in here) ```yaml diff --git a/github_run.py b/github_run.py index 43190d6..ae8849e 100644 --- a/github_run.py +++ b/github_run.py @@ -1,5 +1,3 @@ -import argparse -import fnmatch import json import os import re @@ -13,10 +11,14 @@ class github_run: - def __init__(self): + def __init__( + self, pr_number, processed_files_path, processed_files_branch="processed_files" + ): g = github.Github(os.getenv("GITHUB_TOKEN")) self.repo = g.get_repo(os.getenv("GITHUB_REPOSITORY")) - self.pr = self.repo.get_pull(int(os.getenv("PR_NUMBER"))) + self.pr = self.repo.get_pull(pr_number) + self.processed_files_path = processed_files_path + self.processed_files_branch = processed_files_branch def comment(self, comment_text, **kwargs): if not comment_text: @@ -58,30 +60,15 @@ def comment(self, comment_text, **kwargs): return True def get_files(self): + files_to_process = [] for file in self.pr.get_files(): - raw_url = file.raw_url - if raw_url.endswith(".md"): - response = requests.get(raw_url) + file_path = file.filename + if file_path.endswith(".md"): + response = requests.get(file.raw_url) if response.status_code == 200: - changed_file_path = file.filename - os.makedirs( - os.path.dirname(changed_file_path), - exist_ok=True, - ) - with open(changed_file_path, "w") as f: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w") as f: f.write(response.text) - - changed_files = os.environ.get("CHANGED_FILES") - files_to_process = [] - if changed_files: - for file_path in eval(changed_files.replace("\\", "")): - if file_path.endswith(".md"): - files_to_process.append(file_path) - else: - print("No changed files found, processing all files.") - for root, _, files in os.walk("posts"): - for filename in fnmatch.filter(files, "*.md"): - file_path = os.path.join(root, filename) files_to_process.append(file_path) return files_to_process @@ -135,9 +122,8 @@ def new_pr(self, not_posted): head=branch_name, ) - self.repo.get_workflow("preview.yml").create_dispatch( - ref="main", - inputs={"pr_number": new_pr.number}, + self.repo.get_workflow("galaxy_social.yml").create_dispatch( + ref="main", inputs={"pr_number": str(new_pr.number)} ) self.comment( @@ -145,34 +131,80 @@ def new_pr(self, not_posted): f"I created a new PR for failed posts: {new_pr.html_url}" ) + def initialize_processed_files_branch(self): + try: + branch = self.repo.get_branch(self.processed_files_branch) + except: + self.repo.create_git_ref( + ref=f"refs/heads/{self.processed_files_branch}", + sha=self.repo.get_branch("main").commit.sha, + ) + message = f"Initialize {self.processed_files_branch} with {self.processed_files_path}" + self.repo.create_file( + path=self.processed_files_path, + message=message, + content="{}", + branch=self.processed_files_branch, + ) + print(message) + branch = self.repo.get_branch(self.processed_files_branch) + + file_content = self.repo.get_contents( + self.processed_files_path, ref=branch.commit.sha + ) + processed_files = file_content.decoded_content.decode() + with open(self.processed_files_path, "w") as f: + f.write(processed_files) + print(f"processed_files:\n {processed_files}") + + def commit_processed_files(self): + with open(self.processed_files_path, "r") as file: + file_data = file.read() + processed_files_content_file = self.repo.get_contents( + self.processed_files_path, ref=self.processed_files_branch + ) + message = f"Update {self.processed_files_path}" + self.repo.update_file( + path=self.processed_files_path, + message=message, + content=file_data, + sha=processed_files_content_file.sha, + branch=self.processed_files_branch, + ) + print(message) + return json.loads(file_data) + if __name__ == "__main__": - github_instance = github_run() + with open(os.getenv("GITHUB_EVENT_PATH", ""), "r") as f: + event_data = json.load(f) + pr_number = event_data.get("number") or int( + event_data.get("inputs", {}).get("pr_number") + ) + closed = event_data.get("action") == "closed" + merged = event_data.get("pull_request", {}).get("merged", False) + preview = not merged and not closed + if not merged and closed: + print("No action to take") + sys.exit(0) + + github_instance = github_run(pr_number, "processed_files.json") files_to_process = github_instance.get_files() if not files_to_process: github_instance.comment("No files to process.") sys.exit() - parser = argparse.ArgumentParser(description="Galaxy Social.") - parser.add_argument("--preview", action="store_true", help="Preview the post") - parser.add_argument( - "--json-out", - help="Output json file for processed files", - default="processed_files.json", - ) - args = parser.parse_args() - - gs = galaxy_social(args.preview, args.json_out) - + gs = galaxy_social(preview) try: - message = gs.process_files(files_to_process) - github_instance.comment(message, preview=args.preview) - if args.preview: + github_instance.initialize_processed_files_branch() + message = gs.process_files( + files_to_process, github_instance.processed_files_path + ) + github_instance.comment(message, preview=preview) + if preview: sys.exit() - with open(args.json_out, "r") as file: - processed_files = json.load(file) - + processed_files = github_instance.commit_processed_files() not_posted = {} for file_path, social_stat_dict in processed_files.items(): if file_path in files_to_process: @@ -184,12 +216,10 @@ def new_pr(self, not_posted): not_posted[file_path] = media_list github_instance.new_pr(not_posted) - except Exception as e: - if args.preview: + if preview: github_instance.comment("Something went wrong, an Admin will take a look.") else: not_posted = {file_path: [] for file_path in files_to_process} github_instance.new_pr(not_posted) - raise e diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 93fd7d9..db4fe9c 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -12,9 +12,8 @@ class galaxy_social: - def __init__(self, preview: bool, json_out: str): + def __init__(self, preview: bool): self.preview = preview - self.json_out = json_out plugins_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "plugins.yml" ) @@ -188,10 +187,9 @@ def process_markdown_file(self, file_path: str, processed_files: Dict[str, Any]) print(f"Processed {file_path}: {stats}") return processed_files, message - def process_files(self, files_to_process: list): + def process_files(self, files_to_process: list, processed_files_path: str): processed_files = {} messages = "" - processed_files_path = self.json_out if os.path.exists(processed_files_path): with open(processed_files_path, "r") as file: processed_files = json.load(file) @@ -238,6 +236,6 @@ def process_files(self, files_to_process: list): print("No files to process.") sys.exit() print(f"Processing {len(files_to_process)} file(s): {files_to_process}\n") - gs = galaxy_social(args.preview, args.json_out) - message = gs.process_files(files_to_process) + gs = galaxy_social(args.preview) + message = gs.process_files(files_to_process, args.json_out) print(message) From 11402ce2eef988b7662030d28ee3ff4642e101d4 Mon Sep 17 00:00:00 2001 From: Arash Date: Wed, 6 Nov 2024 23:45:20 +0100 Subject: [PATCH 4/4] Refactor file retrieval to use GitHub library instead of requests --- github_run.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/github_run.py b/github_run.py index ae8849e..59e5461 100644 --- a/github_run.py +++ b/github_run.py @@ -4,7 +4,6 @@ import sys import github -import requests import yaml from lib.galaxy_social import galaxy_social @@ -64,12 +63,14 @@ def get_files(self): for file in self.pr.get_files(): file_path = file.filename if file_path.endswith(".md"): - response = requests.get(file.raw_url) - if response.status_code == 200: - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, "w") as f: - f.write(response.text) - files_to_process.append(file_path) + head_branch = self.pr.head + pr_content = head_branch.repo.get_contents( + file_path, ref=head_branch.ref + ).decoded_content.decode() + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w") as f: + f.write(pr_content) + files_to_process.append(file_path) return files_to_process