Skip to content

Commit

Permalink
Merge pull request #348 from BioKIC/master
Browse files Browse the repository at this point in the history 
Merge BioKIC/Symbiota developments
  • Loading branch information
@egbot
egbot authored Oct 14, 2023
2 parents 719568b + 669feef commit 8061560
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 99 deletions.
106 changes: 54 additions & 52 deletions classes/OccurrenceCleaner.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,60 +23,67 @@ public function __destruct(){
}

//Search and resolve duplicate specimen records
public function getDuplicateCatalogNumber($type,$start,$limit = 500){
//Search is not available for personal specimen management
public function getDuplicateCatalogNumber($type, $start, $limit = 500){
$dupArr = array();
$catArr = array();
$cnt = 0;
if($type=='cat'){
$sql1 = 'SELECT catalognumber '.
'FROM omoccurrences '.
'WHERE catalognumber IS NOT NULL AND collid = '.$this->collid;
$sql = 'SELECT catalogNumber as catnum, count(occid) as cnt FROM omoccurrences WHERE catalognumber IS NOT NULL AND collid = '.$this->collid.' GROUP BY catalogNumber ';
}
else{
$sql1 = 'SELECT otherCatalogNumbers '.
'FROM omoccurrences '.
'WHERE otherCatalogNumbers IS NOT NULL AND collid = '.$this->collid;
$sql = 'SELECT o.otherCatalogNumbers as catnum, count(o.occid) as cnt
FROM omoccurrences o LEFT JOIN omoccuridentifiers i ON o.occid = i.occid
WHERE i.occid IS NULL AND o.otherCatalogNumbers IS NOT NULL AND o.collid = '.$this->collid.' GROUP BY o.otherCatalogNumbers ';
}
//echo $sql1;
$rs = $this->conn->query($sql1);
$sql .= 'HAVING cnt > 1 ';
$rs = $this->conn->query($sql);
$cnt = -1*$start;
while($r = $rs->fetch_object()){
$cn = ($type=='cat'?$r->catalognumber:$r->otherCatalogNumbers);
if(array_key_exists($cn,$catArr)){
//Dupe found
$cnt++;
if($start < $cnt && !array_key_exists($cn,$dupArr)){
//Add dupe to array
$dupArr[$this->cleanInStr($cn)] = '';
if(count($dupArr) > $limit) break;
}
$cnt++;
if($cnt > 0) $dupArr[] = $this->cleanInStr($r->catnum);
if(count($dupArr) > $limit) break;
}
$rs->free();

$retArr = array();
if($dupArr){
$sqlFrag = '';
if($type=='cat'){
$sqlFrag = 'occid, otherCatalogNumbers, catalognumber AS dupid FROM omoccurrences WHERE collid = '.$this->collid.' AND catalognumber IN("'.implode('","', $dupArr).'") ORDER BY catalognumber';
}
else{
$catArr[$cn] = '';
$sqlFrag = 'occid, otherCatalogNumbers, otherCatalogNumbers AS dupid FROM omoccurrences WHERE collid = '.$this->collid.' AND otherCatalogNumbers IN("'.implode('","', $dupArr).'") ORDER BY otherCatalogNumbers';
}
$retArr = $this->getDuplicates($sqlFrag);
}
$rs->free();

$retArr = array();
if($type=='cat'){
$sql = 'SELECT o.catalognumber AS dupid, o.occid, o.catalognumber, o.othercatalognumbers, o.family, o.sciname, '.
'o.recordedby, o.recordnumber, o.associatedcollectors, o.eventdate, o.verbatimeventdate, '.
'o.country, o.stateprovince, o.county, o.municipality, o.locality, o.datelastmodified '.
'FROM omoccurrences o '.
'WHERE o.collid = '.$this->collid.' AND o.catalognumber IN("'.implode('","',array_keys($dupArr)).'") '.
'ORDER BY o.catalognumber';
if($type=='other' && count($dupArr) < $limit){
$retArr = array_merge($retArr, $this->setAdditionalIdentifiers($cnt, ($limit - count($dupArr))));
}
else{
$sql = 'SELECT o.otherCatalogNumbers AS dupid, o.occid, o.catalognumber, o.othercatalognumbers, o.family, o.sciname, '.
'o.recordedby, o.recordnumber, o.associatedcollectors, o.eventdate, o.verbatimeventdate, '.
'o.country, o.stateprovince, o.county, o.municipality, o.locality, o.datelastmodified '.
'FROM omoccurrences o '.
'WHERE o.collid = '.$this->collid.' AND o.otherCatalogNumbers IN("'.implode('","',array_keys($dupArr)).'") '.
'ORDER BY o.otherCatalogNumbers';

return $retArr;
}

private function setAdditionalIdentifiers($cnt, $limit){
$retArr = array();
$start = 0;
if($cnt < 0) $start = -1*$cnt;
$dupArr = array();
$sql = 'SELECT i.identifierName, i.identifierValue, COUNT(i.occid) as cnt
FROM omoccurrences o INNER JOIN omoccuridentifiers i ON o.occid = i.occid
WHERE o.collid = '.$this->collid.' GROUP BY i.identifiername, i.identifiervalue
HAVING cnt > 1 LIMIT '.$start.','.$limit;
$rs = $this->conn->query($sql);
while($r = $rs->fetch_object()){
$dupArr[$r->identifierName][] = $this->cleanInStr($r->identifierValue);
}
//echo $sql;
$rs->free();

$retArr = $this->getDuplicates($sql);
foreach($dupArr as $idName => $idValueArr){
$sqlFrag = 'o.occid, CONCAT(i.identifierName, IF(i.identifierName = "","",": "), i.identifierValue) as otherCatalogNumbers, CONCAT(i.identifierName, ": ", i.identifierValue) as dupid
FROM omoccurrences o INNER JOIN omoccuridentifiers i ON o.occid = i.occid
WHERE o.collid = '.$this->collid.' AND i.identifierName = "'.$this->cleanInStr($idName).'" AND i.identifierValue IN("'.implode('","', $idValueArr).'")
ORDER BY i.identifierName, i.identifierValue';
$retArr = array_merge($retArr, $this->getDuplicates($sqlFrag));
}
return $retArr;
}

Expand Down Expand Up @@ -116,20 +123,16 @@ public function getDuplicateCollectorNumber($start){
$rs->free();

//Collection duplicate clusters
$occidArr = array();
$cnt = 0;
foreach($collArr as $ed => $arr1){
foreach($arr1 as $rn => $arr2){
foreach($arr2 as $ln => $dupArr){
if(count($dupArr) > 1){
//Skip records until start is reached
if($cnt >= $start){
$sql = 'SELECT '.$cnt.' AS dupid, o.occid, o.catalognumber, o.othercatalognumbers, o.othercatalognumbers, o.family, o.sciname, o.recordedby, o.recordnumber, '.
'o.associatedcollectors, o.eventdate, o.verbatimeventdate, o.country, o.stateprovince, o.county, o.municipality, o.locality, datelastmodified '.
'FROM omoccurrences o '.
'WHERE occid IN('.implode(',',$dupArr).') ';
$sqlFragment = $cnt.' AS dupid FROM omoccurrences WHERE occid IN('.implode(',',$dupArr).') ';
//echo $sql;
$retArr = array_merge($retArr,$this->getDuplicates($sql));
$retArr = array_merge($retArr,$this->getDuplicates($sqlFragment));
}
if($cnt > ($start+200)) break 3;
$cnt++;
Expand All @@ -140,15 +143,14 @@ public function getDuplicateCollectorNumber($start){
return $retArr;
}

private function getDuplicates($sql){
private function getDuplicates($sqlFragment){
$retArr = array();
$cnt = 0;
$dupid = '';
$sql = 'SELECT catalognumber, family, sciname, recordedby, recordnumber, associatedcollectors,
eventdate, verbatimeventdate, country, stateprovince, county, municipality, locality, datelastmodified, '.
$sqlFragment;
$rs = $this->conn->query($sql);
while($row = $rs->fetch_assoc()){
if($dupid != $row['dupid']) $cnt++;
$retArr[$cnt][$row['occid']] = array_change_key_case($row);
$dupid = $row['dupid'];
$retArr[$row['dupid']][$row['occid']] = array_change_key_case($row);
}
$rs->free();
return $retArr;
Expand Down
4 changes: 2 additions & 2 deletions classes/OccurrenceDuplicate.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,8 @@ public function getDupeList($recordedBy, $recordNumber, $eventDate, $catNum, $oc
if(is_numeric($occid)) $queryTerms[] = 'o.occid = '.$occid;
$sql = 'SELECT c.institutioncode, c.collectioncode, c.collectionname, o.occid, o.catalognumber, '.
'o.recordedby, o.recordnumber, o.eventdate, o.verbatimeventdate, o.country, o.stateprovince, o.county, o.locality '.
'FROM omoccurrences o LEFT JOIN omcollections c ON o.collid = c.collid ';
if($recordedBy) $sql .= 'LEFT JOIN omoccurrencesfulltext f ON o.occid = f.occid ';
'FROM omoccurrences o INNER JOIN omcollections c ON o.collid = c.collid ';
if($recordedBy) $sql .= 'INNER JOIN omoccurrencesfulltext f ON o.occid = f.occid ';
$sql .= 'WHERE o.occid != '.$currentOccid;
if($queryTerms){
$sql .= ' AND ('.implode(') AND (', $queryTerms).') ';
Expand Down
4 changes: 2 additions & 2 deletions classes/OccurrenceEditorManager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1577,8 +1577,8 @@ public function mergeRecords($targetOccid,$sourceOccid){
//Remap identifiers
$sql = 'UPDATE omoccuridentifiers SET occid = '.$targetOccid.' WHERE occid = '.$sourceOccid;
if(!$this->conn->query($sql)){
$this->errorArr[] .= '; '.$LANG['ERROR_REMAPPING_OCCIDS'].': '.$this->conn->error;
$status = false;
//$this->errorArr[] .= '; '.$LANG['ERROR_REMAPPING_OCCIDS'].': '.$this->conn->error;
//$status = false;
}

//Remap exsiccati
Expand Down
2 changes: 1 addition & 1 deletion classes/OccurrenceTaxaManager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ public function getTaxaSearchTerm(){
}

public function cleanOutStr($str){
if(!is_string($str)) $str = '';
if(!is_string($str) && !is_numeric($str) && !is_bool($str)) $str = '';
if(strpos($str, '=') !== false) $str = '';
return htmlspecialchars($str);
}
Expand Down
82 changes: 40 additions & 42 deletions collections/cleaning/duplicatesearch.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,35 +22,34 @@
if($collid) $cleanManager->setCollId($collid);
$collMap = current($cleanManager->getCollMap());

$statusStr = '';
$isEditor = 0;
if($IS_ADMIN || (array_key_exists("CollAdmin",$USER_RIGHTS) && in_array($collid,$USER_RIGHTS["CollAdmin"])) || ($collMap['colltype'] == 'General Observations')){
$isEditor = 1;
}
if($collMap){
if($IS_ADMIN || (array_key_exists("CollAdmin",$USER_RIGHTS) && in_array($collid,$USER_RIGHTS["CollAdmin"])) || ($collMap['colltype'] == 'General Observations')){
$isEditor = 1;
}

//If collection is a general observation project, limit to User
if($collMap['colltype'] == 'General Observations'){
$cleanManager->setObsUid($SYMB_UID);
}
//If collection is a general observation project, limit to User
if($collMap['colltype'] == 'General Observations'){
$cleanManager->setObsUid($SYMB_UID);
}

$limit = ini_get('max_input_vars')*0.2;
if(!$limit || $limit > 1000) $limit = 1000;
$limit = ini_get('max_input_vars')*0.2;
if(!$limit || $limit > 1000) $limit = 1000;

$dupArr = array();
if($action == 'listdupscatalog'){
$dupArr = $cleanManager->getDuplicateCatalogNumber('cat',$start,$limit);
}
if($action == 'listdupsothercatalog'){
$dupArr = $cleanManager->getDuplicateCatalogNumber('other',$start,$limit);
$dupArr = array();
if($action == 'listdupscatalog'){
$dupArr = $cleanManager->getDuplicateCatalogNumber('cat', $start, $limit);
}
if($action == 'listdupsothercatalog'){
$dupArr = $cleanManager->getDuplicateCatalogNumber('other', $start, $limit);
}
elseif($action == 'listdupsrecordedby'){
$dupArr = $cleanManager->getDuplicateCollectorNumber($start);
}
}
elseif($action == 'listdupsrecordedby'){
$dupArr = $cleanManager->getDuplicateCollectorNumber($start);
}

?>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=<?php echo $CHARSET; ?>">
<title><?php echo $DEFAULT_TITLE.' '.$LANG['OCC_CLEANER']; ?></title>
<?php
include_once($SERVER_ROOT.'/includes/head.php');
Expand Down Expand Up @@ -96,38 +95,37 @@ function batchSwitchTargetSpecimens(cbElem){
}
</script>
</head>
<body style="margin-left:0px;margin-right:0px">
<div class='navpath'>
<body style="margin-left:10px; width: 100%">
<div class='navpath' style="margin:10px">
<a href="../../index.php"><?php echo $LANG['HOME']; ?></a> &gt;&gt;
<a href="../misc/collprofiles.php?collid=<?php echo $collid; ?>&emode=1"><?php echo $LANG['COL_MAN']; ?></a> &gt;&gt;
<a href="index.php?collid=<?php echo $collid; ?>"><?php echo $LANG['CLEAN_MOD_INDEX']; ?></a> &gt;&gt;
<b><?php echo $LANG['DUP_OCCS']; ?></b>
</div>

<!-- inner text -->
<div id="innertext" style="background-color:white;">
<div id="innertext" style="background-color:white; margin:10px; width: 100%; max-width: 100%; padding: 0px;">
<?php
if($isEditor){
if($IS_ADMIN && $limit < 900) echo '<div>'.$LANG['SUPERADMIN_NOTICE'].'</div>';
if($collMap && $isEditor){
if($IS_ADMIN && $limit < 900) echo '<div style="max-width: 1000px">'.$LANG['SUPERADMIN_NOTICE'].'</div>';
if($action == 'listdupscatalog' || $action == 'listdupsothercatalog' || $action == 'listdupsrecordedby'){
//Look for duplicate catalognumbers
if($dupArr){
$recCnt = count($dupArr);
//Build table
?>
<div style="margin-bottom:10px;">
<div style="max-width: 1000px; margin-bottom:10px;">
<b><?php echo $LANG['DUP_INSTRUCTIONS']; ?></b>
</div>
<form name="mergeform" action="duplicatesearch.php" method="post" onsubmit="return validateMergeForm(this);">
<input name="collid" type="hidden" value="<?php echo $collid; ?>" />
<?php
if($recCnt > $limit){
$href = 'duplicatesearch.php?collid='.$collid.'&action='.$action.'&start='.($start+$limit);
echo '<div style="float:right;"><a href="'.$href.'"><b>'.$LANG['NEXT'].' '.$limit.' '.$LANG['RECORDS'].' &gt;&gt;</b></a></div>';
}
echo '<div style="float:left;margin-bottom:4px;margin-left:15px;"><input name="action" type="submit" value="Merge Duplicate Records" /></div>';
echo '<div style="float:left;margin-left:15px;"><b>'.($start+1).' '.$LANG['TO'].' '.($start+$recCnt).' '.$LANG['DUP_CLUSTERS'].' </b></div>';
?>
<div style="max-width: 1000px">
<?php
$recCnt = count($dupArr);
if($recCnt > $limit){
$href = 'duplicatesearch.php?collid='.$collid.'&action='.$action.'&start='.($start+$limit);
echo '<div style="float:right;"><a href="'.$href.'"><b>'.$LANG['NEXT'].' '.$limit.' '.$LANG['RECORDS'].' &gt;&gt;</b></a></div>';
}
echo '<div style="float:left;margin-bottom:4px;margin-left:15px;"><input name="action" type="submit" value="Merge Duplicate Records" /></div>';
echo '<div style="float:left;margin-left:15px;"><b>'.($start+1).' '.$LANG['TO'].' '.($start+$recCnt).' '.$LANG['DUP_CLUSTERS'].' </b></div>';
?>
</div>
<div style="clear: both">
<table class="styledtable" style="font-family:Arial;font-size:12px;">
<tr>
Expand Down Expand Up @@ -156,7 +154,7 @@ function batchSwitchTargetSpecimens(cbElem){
foreach($occArr as $occId => $occArr){
echo '<tr '.(($setCnt % 2) == 1?'class="alt"':'').'>';
echo '<td><a href="../editor/occurrenceeditor.php?occid='.$occId.'" target="_blank">'.$occId.'</a></td>'."\n";
echo '<td><input name="dupid[]" type="checkbox" value="'.$dupKey.':'.$occId.'" /></td>'."\n";
echo '<td><input name="dupid[]" type="checkbox" value="'.$dupKey.'|'.$occId.'" /></td>'."\n";
echo '<td><input name="dup'.$dupKey.'target" type="radio" value="'.$occId.'" '.($first?'checked':'').'/></td>'."\n";
echo '<td>'.$occArr['catalognumber'].'</td>'."\n";
echo '<td>'.$occArr['othercatalognumbers'].'</td>'."\n";
Expand Down Expand Up @@ -199,9 +197,9 @@ function batchSwitchTargetSpecimens(cbElem){
<?php
$dupArr = array();
foreach($_POST['dupid'] as $v){
$vArr = explode(':',$v);
$vArr = explode('|',$v);
if(count($vArr) > 1){
$target = $_POST['dup'.$vArr[0].'target'];
$target = $_POST['dup'.str_replace(' ', '_', $vArr[0]).'target'];
if($target != $vArr[1]) $dupArr[$target][] = $vArr[1];
}
}
Expand Down

0 comments on commit 8061560

Please sign in to comment.