Add basic etl pipeline (#61)

pages/index.tsx

@@ -108,8 +108,15 @@ export default function Home() {
           </Link>
           <p className="text-xs">Regular PDF</p>
           <h4 className="text-2xl font-bold mt-12 mb-4">Something to geek out on</h4>
-          <p className="mb-4 ">
-              This site and the following files are generated from <a href="https://jsoncrack.com/editor?json=https://raw.githubusercontent.com/yuhonas/clintp.xyz/main/resume/resume.clintp.json" className="underline">resume.clintp.json</a> which conforms to <a href="https://jsonresume.org/" target="_blank" className="underline">JSON Resume</a></p>
+              <ul className="mb-4 list-disc">
+                <li>My resume is stored in <a href="https://jsonresume.org/" className="underline">JSON Schema</a> format, it can be viewed at <a href="https://raw.githubusercontent.com/yuhonas/clintp.xyz/main/resume/resume.clintp.json" className="underline">resume.clintp.json</a></li>
+                <li>The resume is automagically linted/spell checked and built into various formats</li>
+                <li>You can explore the data using tools like <a href="https://jsoncrack.com/editor?json=https://raw.githubusercontent.com/yuhonas/clintp.xyz/main/resume/resume.clintp.json" className="underline">JSON Crack</a> or
+                 &nbsp;<a href="https://lite.datasette.io/?json=https://raw.githubusercontent.com/yuhonas/clintp.xyz/main/resume/resume.clintp.json#/data/resume?_sort=rowid&_facet=name&_facet=location" className="underline">lite.datasette.io</a>
+                </li>
+                <li>As part of the <a href="https://github.com/yuhonas/clintp.xyz/blob/main/.github/workflows/ci.yml" className="underline">resume build</a>, a gist is updated which enables the resume to be published on <a href="https://registry.jsonresume.org/yuhonas" className="underline">registry.jsonresume.com</a>, see also <a href="https://registry.jsonresume.org/yuhonas?theme=stackoverflow" className="underline">here</a> for an example of the resume rethemed </li>
+              </ul>
+              <p className="mb-4">This following files are also available</p>
             <ul className="mb-4 flex gap-3">
               {/* <li>
                 <Link href="https://jsoncrack.com/editor?json=https://raw.githubusercontent.com/yuhonas/clintp.xyz/main/resume/resume.clintp.json" className="hover:underline">

resume/Pipfile

@@ -14,6 +14,12 @@ sumy = "*"
 py-readability-metrics = "*"
 check-jsonschema = "*"
 prefect = "*"
+luigi = "*"
+pandas = "*"
+requests = "*"
+qrcode = "*"
+segno = "*"
+qrcode-artistic = "*"
 
 [requires]
 python_version = "3.9"

resume/examples/transform.py

@@ -0,0 +1,68 @@
+import luigi
+import json
+import segno
+from segno import helpers
+from urllib.request import urlopen, urlparse
+from PIL import Image
+
+
+class ExtractResume(luigi.Task):
+    output_file = luigi.Parameter(default='resume.clintp.json')
+
+    def run(self):
+        pass
+
+    def output(self):
+        return luigi.LocalTarget(self.output_file)
+
+
+class FetchAndConvertProfileImage(luigi.Task):
+    def requires(self):
+        return {
+            'resume': ExtractResume()
+        }
+
+    def run(self):
+        resume = json.load(open(self.input()['resume'].path))
+        profile_image = urlopen(resume['basics']['image'])
+
+        image = Image.open(profile_image).quantize(
+            colors=256, method=2).convert('RGB')
+        pixelated = image.resize((40, 40)).resize(image.size, Image.NEAREST)
+
+        pixelated.save(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget("build/clintp.gif")
+
+
+class GenerateQrCode(luigi.Task):
+    def requires(self):
+        return {
+            'resume': ExtractResume(),
+            'profile_image': FetchAndConvertProfileImage()
+        }
+
+    def run(self):
+        # access the resume that was generated in a previous task
+
+        resume = json.load(open(self.input()['resume'].path))
+        # parse the site_url and remove the protocol as it's included for some reason in the qrcode
+
+        parsed_url = urlparse(resume['basics']['url'])
+        url_without_protocol = parsed_url.netloc + parsed_url.path
+
+        qrcode = helpers.make_mecard(name=resume['basics']['name'],
+                                     email=(resume['basics']['email']),
+                                     url=url_without_protocol)
+
+        qrcode.to_artistic(background=self.input()['profile_image'].path,
+                           target=self.output().path, scale=5)
+
+    def output(self):
+        return luigi.LocalTarget("build/clintp-qrcode.gif")
+
+
+if __name__ == "__main__":
+    # luigi.build([TaskD()], workers=1, local_scheduler=True)
+    luigi.build([GenerateQrCode()], workers=1, local_scheduler=True)

resume/resume.clintp.json

@@ -315,10 +315,6 @@
     }
   ],
   "references": [
-    {
-      "name": "Danny Chin",
-      "reference": "It is my pleasure to recommend Richard, his performance working as a consultant for Main St. Company proved that he will be a valuable addition to any company."
-    }
   ],
   "projects": [
     {