Add diacritics setting

artefactual · Jul 13, 2023 · 875adde · 875adde
1 parent c741f38
commit 875adde
Show file tree

Hide file tree

Showing 7 changed files with 216 additions and 8 deletions.
diff --git a/apps/qubit/modules/settings/actions/diacriticsAction.class.php b/apps/qubit/modules/settings/actions/diacriticsAction.class.php
@@ -0,0 +1,103 @@
+<?php
+/*
+ * This file is part of the Access to Memory (AtoM) software.
+ *
+ * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Access to Memory (AtoM) is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+class SettingsDiacriticsAction extends SettingsEditAction
+{
+    // Arrays not allowed in class constants
+    public static $NAMES = [
+        'diacritics',
+    ];
+
+    public function earlyExecute()
+    {
+        parent::earlyExecute();
+
+        $this->updateMessage = $this->i18n->__('Diacritics settings saved.');
+
+        $this->settingDefaults = [
+            'diacritics' => QubitSetting::getByName('diacritics')->getValue(['sourceCulture' => true]),
+        ];
+    }
+
+    public function execute($request)
+    {
+        DefaultEditAction::execute($request);
+
+        if ($request->isMethod('post')) {
+            $this->form->bind($request->getPostParameters(), $request->getFiles());
+
+            if ($this->form->isValid()) {
+                if (0 === intval($request->getPostParameters()['diacritics']) || (1 === intval($request->getPostParameters()['diacritics']) && 0 !== $request->getFiles()['mappings']['size'])) {
+                    $this->processForm($request);
+                    $this->uploadDiacritics($request);
+
+                    $this->getUser()->setFlash('notice', $this->context->i18n->__('Diacritics setting updated. Rebuild the search index for the changes to be applied.'));
+                    $this->redirect(['module' => 'settings', 'action' => 'diacritics']);
+                } else {
+                    $this->getUser()->setFlash('error', $this->context->i18n->__('To enable diacritics setting, please upload a char_filter mapping YAML.'));
+                    $this->redirect(['module' => 'settings', 'action' => 'diacritics']);
+                }
+            }
+        }
+    }
+
+    public function processForm($request)
+    {
+        sfConfig::set('app_diacritics', $request->getPostParameters()['diacritics']);
+        QubitSetting::findAndSave('diacritics', $request->getPostParameters()['diacritics'], ['sourceCulture' => true]);
+    }
+
+    public function uploadDiacritics($request)
+    {
+        $file = $request->getFiles('mappings');
+        $diacriticsMappingPath = '/atom/src/plugins/arElasticSearchPlugin/config/diacritics_mapping.yml';
+
+        if (1 === (int) $request->getPostParameters()['diacritics']) {
+            if ('application/x-yaml' === $request->getFiles()['mappings']['type']) {
+                // Move uploaded file to replace diacritics_mapping.yml.
+                try {
+                    $file = Qubit::moveUploadFile($file);
+                } catch (sfException $e) {
+                    $this->getUser()->setFlash('error', $e->getMessage());
+                }
+
+                // Replace /config/diacritics_mapping.yml with uploaded file
+                $uploadMappingPath = $file['tmp_name'];
+                $diacriticsMappings = file_get_contents($uploadMappingPath);
+                file_put_contents($diacriticsMappingPath, $diacriticsMappings);
+            }
+        } else {
+            // Reset diacritics yaml when disabling the setting
+            file_put_contents($diacriticsMappingPath, '');
+        }
+    }
+
+    protected function addField($name)
+    {
+        switch ($name) {
+            case 'diacritics':
+                $this->form->setDefault($name, $this->settingDefaults[$name]);
+                $this->form->setWidget($name, new sfWidgetFormSelectRadio(['choices' => [0 => $this->i18n->__('Disabled'), 1 => $this->i18n->__('Enabled')]], ['class' => 'radio']));
+                $this->form->setValidator($name, new sfValidatorChoice(['choices' => [1, 0]]));
+                $this->form->setWidget('mappings', new sfWidgetFormInputFile());
+                $this->form->setValidator('mappings', new sfValidatorFile());
+
+                break;
+        }
+    }
+}
diff --git a/apps/qubit/modules/settings/actions/menuComponent.class.php b/apps/qubit/modules/settings/actions/menuComponent.class.php
@@ -39,6 +39,10 @@ public function execute($request)
                 'label' => $i18n->__('Default template'),
                 'action' => 'template',
             ],
+            [
+                'label' => $i18n->__('Diacritics'),
+                'action' => 'diacritics',
+            ],
             [
                 'label' => $i18n->__('Digital object derivatives'),
                 'action' => 'digitalObjectDerivatives',

diff --git a/apps/qubit/modules/settings/templates/diacriticsSuccess.php b/apps/qubit/modules/settings/templates/diacriticsSuccess.php
@@ -0,0 +1,59 @@
+<?php decorate_with('layout_2col.php'); ?>
+
+<?php slot('sidebar'); ?>
+
+  <?php echo get_component('settings', 'menu'); ?>
+
+<?php end_slot(); ?>
+
+<?php slot('title'); ?>
+    <h1><?php echo __('Diacritics settings'); ?></h1>
+<?php end_slot(); ?>
+
+<?php slot('content'); ?>
+
+  <div class="alert alert-info">
+    <p><?php echo __('Please rebuild the search index after uploading diacritics mappings.'); ?></p>
+    <pre>$ php symfony search:populate</pre>
+  </div>
+
+  <?php echo $form->renderGlobalErrors(); ?>
+
+  <?php echo $form->renderFormTag(url_for(['module' => 'settings', 'action' => 'diacritics']), ['method' => 'post']); ?>
+
+    <?php echo $form->renderHiddenFields(); ?>
+
+    <div id="content">
+
+      <table class="table sticky-enabled">
+        <thead>
+          <tr>
+            <th width="30%"><?php echo __('Name'); ?></th>
+            <th><?php echo __('Value'); ?></th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr>
+            <td><?php echo __('Diacritics'); ?></td>
+            <td><?php echo $form->diacritics; ?></td>
+          </tr>
+          <tr>
+            <td><?php echo __('Mappings YAML'); ?></td>
+            <td>
+              <span><?php echo __('YAML file containing char_filter mappings.'); ?></span></br>
+              <?php echo $form->mappings; ?><br/>
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    <section class="actions">
+      <ul>
+        <li><input class="c-btn c-btn-submit" type="submit" value="<?php echo __('Save'); ?>"/></li>
+      </ul>
+    </section>
+
+  </form>
+
+<?php end_slot(); ?>
diff --git a/data/fixtures/settings.yml b/data/fixtures/settings.yml
@@ -3,7 +3,7 @@ QubitSetting:
     name: version
     editable: 0
     deleteable: 0
-    value: 193
+    value: 194
   milestone:
     name: milestone
     editable: 0
@@ -1351,6 +1351,11 @@ QubitSetting:
   Qubit_Settings_defaultArchivalDescriptionBrowseView:
     name: default_archival_description_browse_view
     value: table
+  Qubit_Settings_diacritics:
+    name: diacritics
+    editable: 1
+    deleteable: 0
+    value: 0
   Qubit_Settings_digitalObjectDerivativesPdfPageNumber:
     name: digital_object_derivatives_pdf_page_number
     value: 1

diff --git a/...rate/migrations/arMigration0193.class.php → ...rate/migrations/arMigration0194.class.php b/...rate/migrations/arMigration0193.class.php → ...rate/migrations/arMigration0194.class.php
@@ -18,20 +18,25 @@
  */
 
 /*
- * Modify database for large PDF text support
+ * Add diacritics settings.
  *
  * @package    AccesstoMemory
  * @subpackage migration
  */
-class arMigration0193
+class arMigration0194
 {
-    public const VERSION = 193;
+    public const VERSION = 194;
     public const MIN_MILESTONE = 2;
 
     public function up($configuration)
     {
-        $sql = 'ALTER TABLE `property_i18n` MODIFY `value` MEDIUMTEXT;';
-        QubitPdo::modify($sql);
+        if (null === QubitSetting::getByName('diacritics')) {
+            $setting = new QubitSetting();
+            $setting->setName = 'diacritics';
+            $setting->setValue = 0;
+            $setting->setEditable = 'en';
+            $setting->save();
+        }
 
         return true;
     }

diff --git a/plugins/arElasticSearchPlugin/config/diacritics_mapping.yml b/plugins/arElasticSearchPlugin/config/diacritics_mapping.yml
diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php
@@ -1,5 +1,4 @@
 <?php
-
 /*
  * This file is part of the Access to Memory (AtoM) software.
  *
@@ -51,6 +50,13 @@ class arElasticSearchPlugin extends QubitSearchEngine
      */
     protected $mappings;
 
+    /**
+     * Diacritics mappings configuration, diacritics_mapping.yml.
+     *
+     * @var mixed defaults to null
+     */
+    protected $globalDiacritics;
+
     /**
      * If false, this plugin will perform a trial run with no changes made.
      *
@@ -117,6 +123,8 @@ public function __destruct()
 
     public static function loadMappings()
     {
+        global $globalDiacritics;
+
         // Find mapping.yml
         $finder = sfFinder::type('file')->name('mapping.yml');
         $files = array_unique(array_merge(
@@ -132,6 +140,22 @@ public static function loadMappings()
         $esMapping = new arElasticSearchMapping();
         $esMapping->loadYAML(array_shift($files));
 
+        // Check for diacritics_mapping.yml
+        if (sfConfig::get('app_diacritics')) {
+            // Find diacritics_mapping.yml
+            $diacriticsFinder = sfFinder::type('file')->name('diacritics_mapping.yml');
+            $diacriticsFiles = array_unique(array_merge(
+                $diacriticsFinder->in(sfConfig::get('sf_config_dir')),
+                $diacriticsFinder->in(ProjectConfiguration::getActive()->getPluginSubPaths('/config'))
+            ));
+
+            if (!count($diacriticsFiles)) {
+                throw new sfException('You must create a diacritics_mapping.yml file.');
+            }
+
+            $globalDiacritics = sfYaml::load(array_shift($diacriticsFiles));
+        }
+
         return $esMapping;
     }
 
@@ -468,6 +492,9 @@ public static function modelClassFromQubitObjectClass($className)
      */
     protected function initialize()
     {
+        global $globalDiacritics;
+        $this->loadMappings();
+
         try {
             $this->index->open();
         } catch (Exception $e) {
@@ -479,7 +506,12 @@ protected function initialize()
                     && isset($this->config['index']['configuration']['analysis']['char_filter']['strip_md'])
                 ) {
                     foreach ($this->config['index']['configuration']['analysis']['analyzer'] as $key => $analyzer) {
-                        $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = ['strip_md'];
+                        if ((in_array($key, ['english', 'french', 'default', 'autocomplete'])) && sfConfig::get('app_diacritics')) {
+                            $this->config['index']['configuration']['analysis']['char_filter']['diacritics_lowercase'] = $globalDiacritics;
+                            $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = ['strip_md', 'diacritics_lowercase'];
+                        } else {
+                            $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = ['strip_md'];
+                        }
                     }
                 }