Merge pull request #12 from CivicActions/feature/PGOV-289-Airtable_mi…

…gration_POC

Airtable migration Proof-of-concept

.ddev/.env.pgov-example

@@ -0,0 +1,8 @@
+# Copy this file to `.env` and update the default values for PGOV development.
+
+# Obtain a personal access token at https://airtable.com/create/tokens while
+# logged in as a sufficiently-permissioned user.
+#    Scopes should include:
+#      data.records:read
+#      schema.bases:read
+AIRTABLE_API_KEY=""

.gitignore

@@ -9,7 +9,6 @@
 /web/libraries/
 /private
 # Ignore sensitive information
-/web/sites/*/settings.php
 /web/sites/*/settings.local.php
 
 # Ignore Drupal's file directory
@@ -22,6 +21,7 @@
 /.idea/
 
 # Ignore .env files as they are personal
+/.ddev/.env
 /.env
 keys
 

composer.json

@@ -27,6 +27,7 @@
         "drupal/graphql_compose": "^2.2",
         "drupal/group": "^3.2",
         "drupal/login_gov": "^1.1",
+        "drupal/migrate_plus": "^6.0",
         "drupal/next": "^2.0@beta",
         "drupal/type_tray": "^1.3",
         "drupal/workflow": "^1.8"

web/config/core.extension.yml → config/core.extension.yml

@@ -40,6 +40,7 @@ module:
   menu_link_content: 0
   menu_ui: 0
   migrate: 0
+  migrate_plus: 0
   mysql: 0
   next: 0
   next_extras: 0
@@ -50,6 +51,7 @@ module:
   page_cache: 0
   path: 0
   path_alias: 0
+  pgov_migrate: 0
   search: 0
   serialization: 0
   shortcut: 0
@@ -68,9 +70,9 @@ module:
   views_ui: 0
   pathauto: 1
   views: 10
-  standard: 1000
+  minimal: 1000
 theme:
   claro: 0
   gin: 0
   olivero: 0
-profile: standard
+profile: minimal

config/migrate_plus.migration_group.default.yml

@@ -0,0 +1,10 @@
+uuid: c2910ff5-383f-4bea-ac08-84e7952cdf35
+langcode: en
+status: true
+dependencies: {  }
+id: default
+label: Default
+description: 'A container for any migrations not explicitly assigned to a group.'
+source_type: null
+module: null
+shared_configuration: null

web/modules/custom/pgov_migrate/migrations/migrate_plus.migration.organizations.yml

@@ -0,0 +1,18 @@
+id: organizations
+label: "Strategic Portfolio: Organizations"
+source:
+  plugin: airtable
+  base: 'U.S. Strategic Portfolio'
+  table: 'Organizations'
+destination:
+  plugin: entity:node
+  default_bundle: 'article'
+process:
+  title: name
+  uid:
+    plugin: default_value
+    default_value: 1
+  body/value: description
+  body/format:
+    plugin: default_value
+    default_value: filtered_html

web/modules/custom/pgov_migrate/pgov_migrate.info.yml

@@ -0,0 +1,9 @@
+name: Airtable Migration
+description: Import data from Airtable into content structures on this site.
+package: Custom
+
+type: module
+core_version_requirement: ^10.3 || ^11
+
+dependencies:
+  - migrate_plus:migrate_plus

web/modules/custom/pgov_migrate/src/Plugin/migrate/source/Airtable.php

@@ -0,0 +1,172 @@
+<?php
+
+namespace Drupal\pgov_migrate\Plugin\migrate\source;
+
+use Drupal\migrate\Plugin\MigrationInterface;
+use Drupal\migrate_plus\Plugin\migrate\source\Url;
+
+/**
+ * A source plugin that can migrate records from Airtable.
+ *
+ * Example:
+ *
+ * @code
+ *  source:
+ *    plugin: airtable
+ *    base: Base Name
+ *    table: Goals
+ * @endcode
+ *
+ * @MigrateSource (
+ *   id = "airtable"
+ * )
+ */
+class Airtable extends Url {
+
+  /**
+   * Airtable's API URL; is the base of all requests.
+   */
+  const string AIRTABLE_URL = 'https://api.airtable.com/v0/';
+
+  /**
+   * A list of Airtable bases available for this migration.
+   *
+   * @var array
+   */
+  protected array $airtableBases = [];
+
+  /**
+   * The Airtable base for this migration.
+   *
+   * @var string
+   */
+  protected string $airtableBase;
+
+  /**
+   * The table in the current Airtable base that contains data to be migrated.
+   *
+   * @var string
+   */
+  protected string $table;
+
+  /**
+   * An API key to authorize this connection.
+   *
+   * This should be added to the hosting server's environment, or in
+   * .ddev/env during local development.
+   *
+   * @see https://support.airtable.com/docs/creating-personal-access-tokens
+   *
+   * @var string
+   */
+  private function getAirtableApiKey() {
+    return $_ENV['AIRTABLE_API_KEY'];
+  }
+
+  /**
+   * Authorization headers to validate the API requests.
+   *
+   * @return string[]
+   */
+  private function getAirtableHeaders() {
+    return [
+      'Authorization' => 'Bearer ' . $this->getAirtableApiKey(),
+      'Accept' => 'application/json',
+    ];
+  }
+
+  /**
+   * Get the Airtable-specific base id using its label.
+   *
+   * @param $name
+   *
+   * @return mixed|string|bool
+   */
+  protected function getAirtableBaseId($name) {
+    if (!$this->airtableBases) {
+      $uri = $this::AIRTABLE_URL . 'meta/bases';
+      $result = \Drupal::httpClient()->get($uri, ['headers' => $this->getAirtableHeaders()]);
+
+      $this->airtableBases = [];
+      foreach(json_decode($result->getBody())->bases as $base) {
+        $this->airtableBases[$base->id] = $base->name;
+      }
+    }
+    return array_search($name, $this->airtableBases);
+  }
+
+  /**
+   * Derive the available Airtable fields for a given table in the current base.
+   *
+   * All tables will return `airtable_id` and `airtable_created`, and the rest
+   * of the columns are based on the table contents. Column names are the
+   * machine names as identified in Airtable's API docs for the current base,
+   * and are sufficient for use migration.yaml mappings.
+   *
+   * @return array
+   * @throws \GuzzleHttp\Exception\GuzzleException
+   */
+  private function getAirtableFields() {
+    if ($base = $this->airtableBase) {
+      $uri = $this::AIRTABLE_URL . 'meta/bases/' . $base . '/tables';
+    }
+    $result = \Drupal::httpClient()->get($uri, ['headers' => $this->getAirtableHeaders()]);
+
+    // There doesn't seem to be a way to get fields for _a_ table, so find ours.
+    foreach(json_decode($result->getBody())->tables as $table) {
+      if ($table->name == $this->table) {
+        $fields = [];
+        $fields['airtable_id'] = ['name'  => 'airtable_id', 'selector' => 'id'];
+        $fields['airtable_created_time'] = ['name'  => 'airtable_created', 'selector' => 'createdTime'];
+        foreach ($table->fields as $field) {
+          $fields[$field->name] = (array) $field;
+          $fields[$field->name]['selector'] = 'fields/' . $field->name;
+        }
+        return $fields;
+      }
+    }
+    return [];
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) {
+
+    // Require base and table configurations.
+    if (empty($configuration['table'])) {
+      throw new \InvalidArgumentException('Missing required configuration: "table"');
+    }
+    $this->table = $configuration['table'];
+
+    if (empty($configuration['base'])) {
+      throw new \InvalidArgumentException('Missing required configuration: "base"');
+    }
+    if (!$base = $this->getAirtableBaseId($configuration['base'])) {
+      throw new \InvalidArgumentException('"base" must be one of: ');
+    }
+    $this->airtableBase = $base;
+
+    // Ensure we're using expected plugins.
+    $configuration['data_fetcher_plugin'] = 'http';
+    $configuration['data_parser_plugin'] = 'airtable_data';
+
+    // Set source URLs based on configuration settings for base and table.
+    $url = $this::AIRTABLE_URL . $base;
+    if (isset($configuration['table'])) {
+     $url .= '/' . $configuration['table'];
+    }
+    $configuration['urls'] = [$url];
+
+    // Set Auth headers.
+    $configuration['headers'] = $this->getAirtableHeaders();
+
+    // The ID field is the same for all Airtable tables.
+    $configuration['ids'] = ['id' => ['type' => 'string']];
+
+    // Automatically populate the list of available fields for this base/table.
+    $configuration['fields'] = $this->getAirtableFields();
+
+    parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
+  }
+}