Merge pull request #148 from lsst-sqre/tickets/DM-40446

Rewrite notebook refresher

Dockerfile.template

@@ -53,7 +53,6 @@ COPY user_setups /etc/skel/notebooks/.user_setups
 COPY lsst_kernel.json modify_settings.py \
       lsstlaunch.bash runlab.sh refreshnb.sh \
       prepuller.sh 20-logging.py confmerge.py \
-      00_WARNING_README.md.template warning-tutorial-notebooks.md \
       ${jl}/
 COPY noninteractive /opt/lsst/software/jupyterlab/noninteractive/     
 # Mount configmap at ^^ command/command.json

refreshnb.sh

@@ -1,4 +1,9 @@
 #!/bin/sh
+
+set -e
+
+timeout=30  # Not forever, but long enough to account for some network issues
+
 origdir=$(pwd)
 # Using a different environment variable allows us to retain backwards
 #  compatibility
@@ -10,12 +15,16 @@ fi
 urllist=$(echo ${urls} | tr ',' ' ')
 # Default branch is only used in the absence of a branch spec in a URL
 default_branch=${AUTO_REPO_BRANCH:="prod"}
-# We need to have sourced ${LOADRSPSTACK} before we run this.  In the RSP
-#  container startup environment, we always will have done so already.
-# If LSST_CONDA_ENV_NAME is not set, we have not sourced it...so do.
+
+# We need to have sourced ${LOADRSPSTACK} before we run this, because that
+#  is where git comes from.
+# In the RSP container startup environment, we always will have done so,
+#  but if LSST_CONDA_ENV_NAME is not set, we have not sourced it...so do.
 if [ -z "${LSST_CONDA_ENV_NAME}" ]; then
     source ${LOADRSPSTACK}
 fi
+
+# Loop over our automatically-pulled repositories.
 for url in ${urllist}; do
     branch=$(echo ${url} | cut -d '@' -f 2)
     # Only use default_branch if branch is not specified in the URL
@@ -25,90 +34,64 @@ for url in ${urllist}; do
     repo=$(echo ${url} | cut -d '@' -f 1)
     reponame=$(basename ${repo} .git)
     dirname="${HOME}/notebooks/${reponame}"
-    # If dirname doesn't exist, the user gets a read-only copy.  We also
-    #  take the opportunity to garbage-collect it, in case the clone has
-    #  a whole bunch of removed notebook outputs in it.
-    # We will try to do the right thing if it's r/w, but...at your own risk
-    if ! [ -d "${dirname}" ]; then
-        cd "${HOME}/notebooks" && \
-            (git clone --depth 1 ${repo} -b ${branch} >/dev/null 2>&1) && \
-            chmod -R ugo-w "${dirname}"
-    else
-        cd "${dirname}"
-        if [ "$(pwd)" != "${dirname}" ]; then
-            echo 1>&2 "Could not find repository in ${dirname}"
-        else
-            dirty=0
-            otherbranch=0
-            rw=0
-            upstream=$(git remote -v | grep origin | grep fetch \
-                           | awk '{print $2}')
-            repo=$(basename "${dirname}")
-            firstfile=$(ls -1 | head -1)
-            if [ -n "${firstfile}" ]; then
-                if [ -w "${firstfile}" ]; then
-                    # We were already read/write...shenanigans
-                    rw=1
-                fi
-            fi
-            if [ "${rw}" -eq 0 ]; then
-                # Temporarily make branch writeable
-                chmod -R u+w "${dirname}"
-            fi
-            currentbr=$(git rev-parse --abbrev-ref HEAD)
-            if [ "${currentbr}" != "${branch}" ]; then
-                otherbranch=1
-            fi
-            # If we have uncommited changes, stash, then we will pop back and
-            #  apply after pull
-	    st=$(git status --porcelain=v2)
-            if [ -n "${st}" ]; then
-		# If the only thing wrong is the existence of the README
-		# then we should just delete it, because it probably dates
-		# to when we were doing the test wrong.
-		if [ "${st}" = '? 00_WARNING_README.md' ]; then
-		    rm 00_WARNING_README.md
-		else
-                    ( git stash > /dev/null 2>&1 )
-                    dirty=1
-		fi
-            fi
-            # Do we need to change branches?
-            if [ "${otherbranch}" -ne 0 ]; then
-                ( git checkout ${branch} > /dev/null 2>&1 )
+
+    if [ -d "${dirname}" ]; then
+        if [ -w "${dirname}" ]; then
+            # The repository is writeable.  So we relocate it, and
+            # create/update the relocation readme file in the parent dir.
+            human_now=$(date)
+            now=$(date +%Y%m%d%H%M%S)
+            new="${dirname}-${now}"
+            mv "${dirname}" "${new}"
+            rd="${HOME}/notebooks/00_README_RELOCATION.md"
+            if ! [ -f "${rd}" ]; then
+                # Write the file header if the file didn't exist.
+                echo "## Directory relocation" > ${rd}
+                echo "" >> ${rd}
+                echo -n "The following were read-write, and have" >> ${rd}
+                echo " been relocated as follows:" >> ${rd}
+                echo "" >> ${rd}
             fi
-            git pull
-            if [ "${otherbranch}" -ne 0 ]; then
-                ( git checkout ${currentbr} > /dev/null 2>&1 )
-            fi
-            if [ "${dirty}" -ne 0 ]; then
-                ( git stash apply > /dev/null 2>&1 )
-            fi
-            if [ "${rw}" -ne 0 ] || \
-                   [ "${dirty}" -ne 0 ] || \
-                   [ "${otherbranch}" -ne 0 ]; then
-                # We need to drop the warning in, because the user did not
-                # leave it read-only and on the default branch.
-                #
-                # In short, they are in an unsupported state, and we
-                # recommend they delete the directory and get a new one next
-                # time they launch a lab.
-                jl=/opt/lsst/software/jupyterlab                
-                sed -e "s|{{DIR}}|${repo}|" \
-                    -e "s|{{BRANCH}}|${branch}|" \
-                    -e "s|{{UPSTREAM}}|${upstream}|" \
-                    "${jl}/00_WARNING_README.md.template" > \
-                    ${dirname}/00_WARNING_README.md
-                if [ -f "${jl}/warning-${repo}.md" ]; then
-                    cat "${jl}/warning-${repo}.md" >> \
-                        ${dirname}/00_WARNING_README.md
-                fi
-            fi
-            if [ "${rw}" -eq 0 ]; then
-                # Change it back to read-only
-                chmod -R ugo-w "${dirname}"
+            echo "- ${human_now}: ${dirname} -> ${new}" >> ${rd}
+        else
+            # We might get to go home early.  If the repository exists, and it
+            # is not writeable, see if it has the same last commit as the
+            # remote.
+            #
+            # We're not going to protect against the case where someone
+            # makes it writeable, changes something, and then changes it
+            # back to read-only, but, come on, if you do that, you deserve
+            # what you get.
+            #
+            # We need to be sitting in the repo for git config to work.
+            cd "${dirname}"
+            remote=$(git config --get remote.origin.url)
+            # Set up our comparison to see whether the branch is stale.
+            branch_re="\\srefs/heads/${branch}\$"
+            remote_sha=$(timeout $timeout git ls-remote ${remote} | \
+                             grep "${branch_re}" | \
+                             awk '{print $1}')
+            local_sha=$(git rev-parse HEAD)
+            # Go back to where we were now that we have the commit SHA of
+            # both remote and local.
+            cd "${origdir}"
+            if [ "${local_sha}" = "${remote_sha}" ]; then
+                # We're fine.  Don't do anything.  Skip ahead.
+                continue
             fi
+            # It's not up-to-date, so make it read-write so we can remove it.
+            chmod -R u+w "${dirname}"
+            # And remove it.
+            rm -rf "${dirname}"
         fi
     fi
+    # If we got here, within the loop iteration, either the directory
+    # didn't exist to start with, we moved it aside because it was
+    # writeable, or we removed it because it was stale.
+    timeout ${timeout} \
+            git clone --depth 1 ${repo} -b ${branch} "${dirname}" \
+            >/dev/null 2>&1
+    # Finally, make it read-only
+    chmod -R ugo-w "${dirname}"
 done
 cd "${origdir}" # In case we were sourced and not in a subshell