From 84e72eb7393606fc0cdee6f1f749cb813181eafc Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 23 Oct 2023 15:02:28 -0500 Subject: [PATCH] First attempt to build GitHub Pages site --- .github/workflows/build-pages.yaml | 89 ++++++++ .gitignore | 13 +- Dockerfile | 57 +++-- Gemfile | 10 + ImportPagesScript.py | 135 ------------ Settings.py | 19 -- db.py | 133 ----------- static/favicon.ico => favicon.ico | Bin global_vars.py | 12 - index.html | 59 +++++ jira_ticket.py | 145 ------------ main.py | 208 ------------------ .../des_components/des-dr1/des-dr1-faq.html | 5 - .../des_components/des-pages/des-about.html | 10 +- static/des_components/des-pages/des-help.html | 132 ----------- static/des_components/des-public-main.html | 2 - .../des-y3a2/des-y3a2-key-extensions.html | 4 +- static/des_components/elements.html | 1 - vulcan.py | 71 ------ 19 files changed, 211 insertions(+), 894 deletions(-) create mode 100644 .github/workflows/build-pages.yaml create mode 100644 Gemfile delete mode 100644 ImportPagesScript.py delete mode 100644 Settings.py delete mode 100644 db.py rename static/favicon.ico => favicon.ico (100%) delete mode 100644 global_vars.py create mode 100644 index.html delete mode 100644 jira_ticket.py delete mode 100644 main.py delete mode 100644 static/des_components/des-pages/des-help.html delete mode 100755 vulcan.py diff --git a/.github/workflows/build-pages.yaml b/.github/workflows/build-pages.yaml new file mode 100644 index 00000000..df2a37d9 --- /dev/null +++ b/.github/workflows/build-pages.yaml @@ -0,0 +1,89 @@ +name: build-pages + +on: + push: + branches: + - 'github-pages' + paths: + - '**' + - '.github/workflows/build-pages.yaml' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - + name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - + name: Login to container registry + uses: docker/login-action@v1 + with: + registry: "hub.ncsa.illinois.edu" + username: "${{ secrets.NCSA_HARBOR_DES_PUBLIC_USERNAME }}" + password: "${{ secrets.NCSA_HARBOR_DES_PUBLIC_TOKEN }}" + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: hub.ncsa.illinois.edu/des-public/desdm-public:pages + # From https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#github-cache + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} + + - + uses: shrink/actions-docker-extract@v3 + id: extract + with: + image: hub.ncsa.illinois.edu/des-public/desdm-public:pages + path: /srv/jekyll/. + - + name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: ${{ steps.extract.outputs.destination }} + name: dist + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 diff --git a/.gitignore b/.gitignore index 808837c5..4945da00 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,17 @@ *.pyc -static/files/SVA1/* -static/log.csv __pycache__/ static/bower_components/* .DS_Store .codesync static/des_components/elements-built.html + +# Jekyll + +# Gemfile +*.gem +.jekyll-cache +.bundle +.sass-cache +_site +Gemfile.lock +.idea diff --git a/Dockerfile b/Dockerfile index 6e19c9d8..eef5be4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,46 @@ -FROM ubuntu:20.04 +FROM node:20 as build -RUN apt-get update && \ - apt-get install -y \ - python3-pip \ - wget \ - && rm -rf /var/lib/apt/lists/* +# RUN apt-get update && \ +# apt-get install -y \ +# python3-pip \ +# wget \ +# && rm -rf /var/lib/apt/lists/* # Basic python reqs -RUN pip3 install --no-cache-dir jira -RUN pip3 install --no-cache-dir netaddr -RUN pip3 install --no-cache-dir bcrypt -RUN pip3 install --no-cache-dir pyyaml -RUN pip3 install --no-cache-dir tornado==5.0.1 -RUN pip3 install --no-cache-dir jsmin +# RUN pip3 install --no-cache-dir jira +# RUN pip3 install --no-cache-dir netaddr +# RUN pip3 install --no-cache-dir bcrypt +# RUN pip3 install --no-cache-dir pyyaml +# RUN pip3 install --no-cache-dir tornado==5.0.1 +# RUN pip install --no-cache-dir jsmin -WORKDIR /opt -RUN wget https://nodejs.org/dist/v12.14.1/node-v12.14.1-linux-x64.tar.xz -RUN tar -C /usr/local --strip-components 1 -xf /opt/node-v12.14.1-linux-x64.tar.xz +# WORKDIR /tmp +# ADD https://nodejs.org/dist/v12.14.1/node-v12.14.1-linux-x64.tar.xz +# RUN tar -C /usr/local --strip-components 1 -xf /tmp/node-v12.14.1-linux-x64.tar.xz RUN npm install -g vulcanize@1.16.0 -RUN useradd --create-home --shell /bin/bash des --uid 1001 -USER des -WORKDIR /home/des/ +# RUN useradd --create-home --shell /bin/bash des --uid 1001 +# USER des +WORKDIR /opt COPY --chown=des:des ./ ./ -RUN python3 vulcan.py --build +RUN vulcanize static/des_components/elements.html \ + --exclude static/bower_components/polymer/lib/legacy/ \ + --out-html static/des_components/elements-built.html + +FROM jekyll/jekyll:3.8 +## Currently jekyll/jekyll:4 fails with the error below so tag 3.8 is used instead. + +ENV JEKYLL_UID=1000 +ENV JEKYLL_GID=1000 + +USER ${JEKYLL_UID} + +## Install required gems +COPY ./Gemfile ./Gemfile +RUN bundle install + +## Copy source files +COPY --from=build --chown=${JEKYLL_UID}:${JEKYLL_GID} /opt/ ./ -CMD [ "python3", "main.py" ] +CMD ["bundle", "exec", "jekyll", "serve", "--host=0.0.0.0", "--watch", "--drafts"] diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..324e9307 --- /dev/null +++ b/Gemfile @@ -0,0 +1,10 @@ +source "https://rubygems.org" + +gem "github-pages", '228', group: :jekyll_plugins + +# enable tzinfo-data for local build +# gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw] +# gem 'jekyll-paginate', '1.1.0' +# gem 'faraday', '2.7.4' +# gem 'faraday-retry', '2.0.0' +# gem 'webrick', '1.8.1' diff --git a/ImportPagesScript.py b/ImportPagesScript.py deleted file mode 100644 index aee4f02a..00000000 --- a/ImportPagesScript.py +++ /dev/null @@ -1,135 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import os -import random -import string -from time import sleep - -# opens requests.sessions and inputs in user data for time spent using the get_page_content() -# function -with requests.session() as ssn: - usernameinput = input("What is your XConfluence username?") - passwordinput = input("What is your XConfluence password?") - print("If you get an error saying 'AttributeError: 'NoneType' object has no" - "attribute 'prettify' " - "after entering the URL for the below prompt, your password or username has" - "probably been entered wrong. " - "Please restart the program and enter the correct login credentials.") - payload = { - 'os_username': usernameinput, - 'os_password': passwordinput - } - - ssn.post('https://opensource.ncsa.illinois.edu/confluence/login.action', data=payload) - -root_dir = "/home/matias/Research/des_public_new/static" -root_folder = "/des_components/des-dr1/" - -# gets the content of the inputted URL's main content, saves it as a file, -# and additonally saves any linked images as files -print("PLEASE NOTE: Changing the location of the saved files from" - "imported pages is necessary for them to be displayed correctly!") - - -def get_page_content(): - # goes to website and creates file based off of HTML tile - urlinput = input("What URL do you want to pull from?") - filename = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(1, 10)) - filename += ".html" - page = ssn.get(urlinput) - soup = BeautifulSoup(page.content, 'html.parser') - newfile = open(root_dir + root_folder + filename, "w+") - - # function to add content to the beginning of the file - def line_prepender(file, line): - with open(root_dir + root_folder + file, 'r+') as f: - content = f.read() - f.seek(0, 0) - f.write(line.rstrip('\r\n') + '\n' + content) - - pagecontent = soup.find("div", {"id": "main-content"}) - pagecontent = BeautifulSoup(pagecontent.__repr__(), 'html.parser') - - # saves pictures on the page to seperate files - for item in pagecontent.find_all("img", {"class": "confluence-embedded-image"}): - list_att = list(item.attrs.keys()) - imagesrc = item['src'] - imageURL = "https://opensource.ncsa.illinois.edu" + imagesrc - r = ssn.get(imageURL, allow_redirects=True) - picturesfilename = str(item['data-linked-resource-default-alias']) - open(root_dir+"/images/" + picturesfilename, 'wb').write(r.content) - for att in list_att: - if att not in ['src', 'width', 'height', 'scale']: - del item[att] - item['src'] = '/static/images/'+picturesfilename - - # delete classes - for tag in pagecontent(): - del tag["class"] - - newfile.write(pagecontent.prettify()) - - - # add page identifiers to end of file - header = input("What should the heading be? (Title of the des-card)") - pageid = input("What should the dom-module id be? (des-home, des-data, etc.)") - pageclass = input("What Polymer class should this is labeled as (desHome)?") - newfile = open(root_dir + root_folder + filename, 'a') - endtext = """ - - - - - - """.format(pageclass=pageclass, pageid=pageid) - newfile.write(endtext) - - # renames file - # newfilename = str(soup.title.text)[:-15] + ".html" - # newfilename = re.sub('[/]', '-', newfilename) - - newfilename = pageid+".html" - os.rename(root_dir + root_folder + filename, root_dir + root_folder + newfilename) - - # add page identifiers to beginning of file - initext = """\ - - diff --git a/static/des_components/des-pages/des-about.html b/static/des_components/des-pages/des-about.html index e801e224..69554918 100644 --- a/static/des_components/des-pages/des-about.html +++ b/static/des_components/des-pages/des-about.html @@ -22,20 +22,16 @@

About DES Data Management

The Core DESDM Team at NCSA

-

Use the Help form to contact the DESDM team with questions and feedback. See the list below if you need to contact a specific team member.

Note: This list does not include former members of DESDM, many of whom contributed significantly to the project, and so it should not be interpreted as an acknowledgement section.

diff --git a/static/des_components/des-pages/des-help.html b/static/des_components/des-pages/des-help.html deleted file mode 100644 index 88bca7ac..00000000 --- a/static/des_components/des-pages/des-help.html +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - diff --git a/static/des_components/des-public-main.html b/static/des_components/des-public-main.html index 04fcd113..5c6a4f44 100644 --- a/static/des_components/des-public-main.html +++ b/static/des_components/des-public-main.html @@ -624,7 +624,6 @@ - Get Help Acknowledgements About Us @@ -735,7 +734,6 @@ - diff --git a/static/des_components/des-y3a2/des-y3a2-key-extensions.html b/static/des_components/des-y3a2/des-y3a2-key-extensions.html index 2cc298e1..8170347a 100644 --- a/static/des_components/des-y3a2/des-y3a2-key-extensions.html +++ b/static/des_components/des-y3a2/des-y3a2-key-extensions.html @@ -498,7 +498,7 @@

Cosmology Chains -

Chains not on website, but available upon request through the Help form

+

Chains not on website, but available upon request:

@@ -582,7 +582,7 @@

Scale Cuts (Top)

The Y3ext paper reports quite a bit of information in plots containing a lot of points showing parameter estimates, tension metrics, and model comparison tests. While we believe the plots are the most effective way to communicate this information in the paper, readers or people interested in comparing their work to our analysis may be interested in the specific numbers.

-

Thus, we share sets of files that were automatically generated when producing the plots reporting the associated numbers. We will make an effort to document how to read these files below, but if you have questions about how to interpret the files, please contact DES helpdesk through the Help form .

+

Thus, we share sets of files that were automatically generated when producing the plots reporting the associated numbers. We will make an effort to document how to read these files below, but if you have questions about how to interpret the files, feel free to contact us.