From c0c1a91243ada240f3ad1277a257e6750f70675b Mon Sep 17 00:00:00 2001 From: Mike Cantelon Date: Wed, 29 Jun 2022 00:39:15 -0700 Subject: [PATCH] Add term CSV import task, refs #13622 --- lib/task/import/csvTermImportTask.class.php | 282 ++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 lib/task/import/csvTermImportTask.class.php diff --git a/lib/task/import/csvTermImportTask.class.php b/lib/task/import/csvTermImportTask.class.php new file mode 100644 index 0000000000..353938e347 --- /dev/null +++ b/lib/task/import/csvTermImportTask.class.php @@ -0,0 +1,282 @@ +. + */ + +/** + * Import CSV term data. + * + * @author Mike Cantelon + */ +class csvTermImportTask extends csvImportBaseTask +{ + protected $namespace = 'csv'; + protected $name = 'term-import'; + protected $briefDescription = 'Import csv term data'; + protected $detailedDescription = <<<'EOF' +Import CSV term data +EOF; + + /** + * @see sfTask + * + * @param mixed $arguments + * @param mixed $options + */ + public function execute($arguments = [], $options = []) + { + parent::execute($arguments, $options); + + $this->validateOptions($options); + + $skipRows = ($options['skip-rows']) ? $options['skip-rows'] : 0; + + $sourceName = ($options['source-name']) + ? $options['source-name'] + : basename($arguments['filename']); + + if (false === $fh = fopen($arguments['filename'], 'rb')) { + throw new sfException('You must specify a valid filename'); + } + + $databaseManager = new sfDatabaseManager($this->configuration); + $conn = $databaseManager->getDatabase('propel')->getConnection(); + + $import = new QubitFlatfileImport([ + 'status' => [ + 'context' => $this->context, + 'task' => $this, + 'cliOptions' => $options, + 'sourceName' => $sourceName, + 'legacyIdToInternalId' => [], + 'termRelations' => [], + ], + + // How many rows should import until we display an import status update? + 'rowsUntilProgressDisplay' => $options['rows-until-update'], + + // Where to log errors to + 'errorLog' => $options['error-log'], + + 'saveLogic' => function (&$self) { + $legacyId = ($self->columnExists('legacyId')) ? $self->columnValue('legacyId') : false; + $taxonomyId = $self->columnValue('taxonomyId'); + $culture = $self->columnValue('culture'); + + // Set parent ID + if ($self->columnExists('parentName')) { + // Look up parent ID using name, taxonomy ID, and culture + $criteria = new Criteria(); + $criteria->add(QubitTerm::TAXONOMY_ID, $taxonomyId); + $criteria->addJoin(QubitTerm::ID, QubitTermI18n::ID); + $criteria->add(QubitTermI18n::CULTURE, $culture); + $criteria->add(QubitTermI18n::NAME, $self->columnValue('parentName')); + + $term = QubitTerm::getOne($criteria); + + if (null !== $term) { + $parentId = $term->id; + } else { + $parentId = QubitTerm::ROOT_ID; + } + } elseif ($self->columnExists('parentId')) { + $parentId = $self->columnValue('parentId'); + } + + // Set culture + $self->status['context']->getUser()->setCulture($culture); + + // Attempt to load pre-existing term and take note of taxonomy + $term = null; + + if (!empty($legacyId) && isset($self->status['legacyIdToInternalId'][$legacyId])) { + // This row is a translation rather than a new term + $term = QubitTerm::getById($self->status['legacyIdToInternalId'][$legacyId]); + } else { + // Check to see if a term with the same name already exists in the DB + $criteria = new Criteria(); + $criteria->add(QubitTerm::TAXONOMY_ID, $taxonomyId); + $criteria->addJoin(QubitTerm::ID, QubitTermI18n::ID); + $criteria->add(QubitTermI18n::CULTURE, $culture); + $criteria->add(QubitTermI18n::NAME, $self->columnValue('name')); + + $term = QubitTerm::getOne($criteria); + } + + // Create term if an existing term hasn't been found + if (null === $term) { + $term = new QubitTerm(); + $term->taxonomyId = $self->columnValue('taxonomyId'); + + if (!empty($parentId)) { + if ( + $mapEntry = $self->fetchKeymapEntryBySourceAndTargetName( + $parentId, + $self->getStatus('sourceName'), + 'term' + ) + ) { + // Use parent ID reference in keymap entry + $term->parentId = $mapEntry->target_id; + } elseif (null !== QubitTerm::getById($parentId)) { + // Assume parent ID is the ID of an existing term + $term->parentId = $parentId; + } else { + $message = 'Parent ID not found in keymap data or existing data. Setting to root.'; + $self->getStatus('task')->log($self->logError($message)); + } + } + } + + // Set term name and code and save + $term->setName($self->columnValue('name'), ['culture' => $culture]); + + if ($self->columnExists('code')) { + $term->code = $self->columnValue('code'); + } + + $term->save(); + + // Store new internal ID in case there are subsequent translations to import + if (!empty($legacyId) && !isset($self->status['legacyIdToInternalId'][$legacyId])) { + $self->status['legacyIdToInternalId'][$legacyId] = $term->id; + } + + // Add notes, if any + $noteTypes = [ + 'scopeNote' => QubitTerm::SCOPE_NOTE_ID, + 'sourceNote' => QubitTerm::SOURCE_NOTE_ID, + 'displayNote' => QubitTerm::DISPLAY_NOTE_ID, + ]; + + foreach ($noteTypes as $column => $typeId) { + if ($self->columnExists('')) { + $noteDataText = trim($self->columnValue($column)); + + if (!empty($noteDataText)) { + $noteData = explode('|', $noteDataText); + + foreach ($noteData as $noteText) { + $note = new QubitNote(); + $note->objectId = $term->id; + $note->typeId = $typeId; + $note->content = $noteText; + + $note->save(); + } + } + } + } + + // Add other names + if ($self->columnExists('otherFormsOfName')) { + $otherNameDataText = trim($self->columnValue('otherFormsOfName')); + + if (!empty($otherNameDataText)) { + $otherNameData = explode('|', $otherNameDataText); + + foreach ($otherNameData as $name) { + $otherName = new QubitOtherName(); + $otherName->objectId = $term->id; + $otherName->typeId = QubitTerm::ALTERNATIVE_LABEL_ID; + $otherName->name = $name; + + $otherName->save(); + } + } + } + + // Add related terms + $self->status['termRelations'][$term->id] = []; + + if ($self->columnExists('relatedTerms')) { + $relatedTermsDataText = trim($self->columnValue('relatedTerms')); + + if (!empty($relatedTermsDataText)) { + $relatedTermsData = explode('|', $relatedTermsDataText); + + foreach ($relatedTermsData as $related) { + $relatedData = [ + 'name' => $related, + 'taxonomyId' => $taxonomyId, + 'culture' => $culture, + ]; + + $self->status['termRelations'][$term->id][] = $relatedData; + } + } + } + + // Add keymap entry + if ($self->columnExists('legacyId')) { + $self->createKeymapEntry($self->getStatus('sourceName'), $self->columnValue('legacyId'), $term); + } + }, + + 'completeLogic' => function (&$self) { + // Add term relations + foreach ($self->status['termRelations'] as $termId => $relatedTermsData) { + foreach ($relatedTermsData as $relatedData) { + // Search for term in the DB + $criteria = new Criteria(); + $criteria->add(QubitTerm::TAXONOMY_ID, $relatedData['taxonomyId']); + $criteria->addJoin(QubitTerm::ID, QubitTermI18n::ID); + $criteria->add(QubitTermI18n::CULTURE, $relatedData['culture']); + $criteria->add(QubitTermI18n::NAME, $relatedData['name']); + + if (null !== $relatedTerm = QubitTerm::getOne($criteria)) { + // Create relation + $relation = new QubitRelation(); + $relation->objectId = $termId; + $relation->subjectId = $relatedTerm->id; + $relation->typeId = QubitTerm::TERM_RELATION_ASSOCIATIVE_ID; + $relation->save(); + } + } + } + }, + ]); + + // Allow search indexing to be enabled via a CLI option + $import->searchIndexingDisabled = ($options['index']) ? false : true; + + $import->csv($fh, $skipRows); + } + + /** + * @see csvImportBaseTask + */ + protected function configure() + { + parent::configure(); + + $this->addOptions([ + new sfCommandOption( + 'source-name', + null, + sfCommandOption::PARAMETER_OPTIONAL, + 'Source name to use when inserting keymap entries.' + ), + new sfCommandOption( + 'index', + null, + sfCommandOption::PARAMETER_NONE, + 'Index for search during import.' + ), + ]); + } +}