From 8f3d1d8cf959df3fafaf09f4783daa7315d1ec6f Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 13 Sep 2024 16:15:52 +0200 Subject: [PATCH 1/7] initial version of new besluit handling, tested with the cases in the mock route --- app.ts | 6 +- .../delta-context-config.js | 4 +- .../mandataris-besluit/beleidsdomein.ts | 137 ++++++ controllers/mandataris-besluit/fractie.ts | 238 +++++++++++ controllers/mandataris-besluit/mandataris.ts | 106 +++++ controllers/mandataris-besluit/persoon.ts | 77 ++++ controllers/mandatees-decisions.ts | 106 +++-- cron/handle-decision-queue.ts | 103 +++++ data-access/lidmaatschap.ts | 51 +++ data-access/mandataris.ts | 32 +- data-access/mandatees-decisions.ts | 357 +++++++++++++++- data-access/persoon.ts | 3 +- docker-compose.debug.yml | 7 +- routes/delta-decisions.ts | 63 +++ routes/delta.ts | 30 -- routes/mock.ts | 393 ++++++++++++++++++ util/find-uuid-in-uri.ts | 10 - util/uuid-for-uri.ts | 60 +++ 18 files changed, 1680 insertions(+), 103 deletions(-) create mode 100644 controllers/mandataris-besluit/beleidsdomein.ts create mode 100644 controllers/mandataris-besluit/fractie.ts create mode 100644 controllers/mandataris-besluit/mandataris.ts create mode 100644 controllers/mandataris-besluit/persoon.ts create mode 100644 cron/handle-decision-queue.ts create mode 100644 data-access/lidmaatschap.ts create mode 100644 routes/delta-decisions.ts delete mode 100644 routes/delta.ts create mode 100644 routes/mock.ts delete mode 100644 util/find-uuid-in-uri.ts create mode 100644 util/uuid-for-uri.ts diff --git a/app.ts b/app.ts index 24d12ff..7b83b23 100644 --- a/app.ts +++ b/app.ts @@ -3,13 +3,15 @@ import { app } from 'mu'; import express, { Request, ErrorRequestHandler } from 'express'; import bodyParser from 'body-parser'; -import { deltaRouter } from './routes/delta'; +import { deltaRouter } from './routes/delta-decisions'; import { mandatarissenRouter } from './routes/mandatarissen'; import { fractiesRouter } from './routes/fractie'; import { personenRouter } from './routes/persoon'; import { burgemeesterRouter } from './routes/burgemeester-benoeming'; import { installatievergaderingRouter } from './routes/installatievergadering'; import { mandatenRouter } from './routes/mandaten'; +import { mockRouter } from './routes/mock'; +import { cronjob } from './cron/handle-decision-queue'; import { cronjob as notificationBekrachtigdMandataris } from './cron/notification-for-bekrachtigde-mandataris'; import { electionResultsRouter } from './routes/verkiezingsresultaten'; @@ -37,6 +39,7 @@ app.use('/mandaten', mandatenRouter); app.use('/burgemeester-benoeming', burgemeesterRouter); app.use('/installatievergadering-api', installatievergaderingRouter); app.use('/election-results-api', electionResultsRouter); +app.use('/mock', mockRouter); const errorHandler: ErrorRequestHandler = function (err, _req, res, _next) { // custom error handler to have a default 500 error code instead of 400 as in the template @@ -49,3 +52,4 @@ const errorHandler: ErrorRequestHandler = function (err, _req, res, _next) { app.use(errorHandler); notificationBekrachtigdMandataris.start(); +setTimeout(() => cronjob.start(), 10000); diff --git a/config/custom-dispatching/delta-context-config.js b/config/custom-dispatching/delta-context-config.js index 39960c7..35f6536 100644 --- a/config/custom-dispatching/delta-context-config.js +++ b/config/custom-dispatching/delta-context-config.js @@ -11,7 +11,7 @@ PREFIX euro: PREFIX euvoc: PREFIX ext: PREFIX foaf: -PREFIX generiek: +PREFIX generiek: PREFIX lblodlg: PREFIX locn: PREFIX mandaat: @@ -19,7 +19,7 @@ PREFIX mu: PREFIX org: PREFIX organisatie: PREFIX person: -PREFIX persoon: +PREFIX persoon: PREFIX prov: PREFIX rdfs: PREFIX regorg: diff --git a/controllers/mandataris-besluit/beleidsdomein.ts b/controllers/mandataris-besluit/beleidsdomein.ts new file mode 100644 index 0000000..57f8ebd --- /dev/null +++ b/controllers/mandataris-besluit/beleidsdomein.ts @@ -0,0 +1,137 @@ +import { + copySimpleInstanceToGraph, + getBeleidsdomeinTriplesInStagingGraph, + getGraphsWhereInstanceExists, + insertTriplesInGraph, +} from '../../data-access/mandatees-decisions'; +import { MandatarisFullInfo, Triple } from '../../types'; +import { createMandatarisBesluitNotification } from '../../util/create-notification'; +import { getUuidForUri } from '../../util/uuid-for-uri'; + +export async function copyBeleidsdomeinInfo( + mandatarisFullInfo: MandatarisFullInfo, +) { + const beleidsDomeinen = mandatarisFullInfo.triples + .filter( + (triple) => + triple.predicate.value === + 'http://data.vlaanderen.be/ns/mandaat#beleidsdomein', + ) + .map((triple) => triple.object.value); + + for (const beleidsDomein of beleidsDomeinen) { + await copyBeleidsDomein( + mandatarisFullInfo, + beleidsDomein, + mandatarisFullInfo.graph, + ); + } +} + +async function copyBeleidsDomein( + mandatarisFullInfo: MandatarisFullInfo, + beleidsDomein: string, + graph: string, +): Promise { + const graphsForBeleidsDomein = + await getGraphsWhereInstanceExists(beleidsDomein); + const inAppropriateGraph = graphsForBeleidsDomein.find((g) => + ['http://mu.semte.ch/graphs/public', graph].includes(g.graph.value), + ); + if (graphsForBeleidsDomein.length === 0) { + await createBeleidsDomein(beleidsDomein, graph); + await createMandatarisBesluitNotification({ + title: 'Beleidsdomein aangemaakt', + description: `Een nieuw beleidsdomein met uri ${beleidsDomein} werd aangemaakt op basis van de informatie in het Besluit.`, + type: 'info', + info: mandatarisFullInfo, + }); + } else if (!inAppropriateGraph) { + await copySimpleInstanceToGraph(beleidsDomein, graph); + } else { + // beleidsdomein exists in an appropriate graph. nothing to do + } +} + +async function createBeleidsDomein(beleidsdomeinUri: string, graph: string) { + const triplesForBeleidsdomein = + await getBeleidsdomeinTriplesInStagingGraph(beleidsdomeinUri); + const id = await getUuidForUri(beleidsdomeinUri, { + allowCheckingUri: true, + allowGenerateUuid: true, + }); + const extraTriples: Triple[] = [ + { + subject: { + value: beleidsdomeinUri, + type: 'uri', + }, + predicate: { + value: 'http://mu.semte.ch/vocabularies/core/uuid', + type: 'uri', + }, + object: { + value: id, + type: 'string', + }, + }, + { + subject: { + value: beleidsdomeinUri, + type: 'uri', + }, + predicate: { + value: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + type: 'uri', + }, + object: { + value: 'http://mu.semte.ch/vocabularies/ext/BeleidsdomeinCode', + type: 'uri', + }, + }, + { + subject: { + value: beleidsdomeinUri, + type: 'uri', + }, + predicate: { + value: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + type: 'uri', + }, + object: { + value: 'http://www.w3.org/2004/02/skos/core#Concept', + type: 'uri', + }, + }, + { + subject: { + value: beleidsdomeinUri, + type: 'uri', + }, + predicate: { + value: 'http://www.w3.org/2004/02/skos/core#inScheme', + type: 'uri', + }, + object: { + value: 'http://data.vlaanderen.be/id/conceptscheme/BeleidsdomeinCode', + type: 'uri', + }, + }, + { + subject: { + value: beleidsdomeinUri, + type: 'uri', + }, + predicate: { + value: 'http://www.w3.org/2004/02/skos/core#topConceptOf', + type: 'uri', + }, + object: { + value: 'http://data.vlaanderen.be/id/conceptscheme/BeleidsdomeinCode', + type: 'uri', + }, + }, + ]; + const allTriples = [...triplesForBeleidsdomein, ...extraTriples]; + await insertTriplesInGraph(allTriples, graph); +} diff --git a/controllers/mandataris-besluit/fractie.ts b/controllers/mandataris-besluit/fractie.ts new file mode 100644 index 0000000..a5da568 --- /dev/null +++ b/controllers/mandataris-besluit/fractie.ts @@ -0,0 +1,238 @@ +import { querySudo } from '@lblod/mu-auth-sudo'; +import { sparqlEscapeUri } from 'mu'; +import { fixLidmaatschapTijdsinterval } from '../../data-access/lidmaatschap'; +import { BESLUIT_STAGING_GRAPH } from '../../data-access/mandatees-decisions'; +import { MandatarisFullInfo } from '../../types'; +import { createMandatarisBesluitNotification } from '../../util/create-notification'; +import { sparqlEscapeString } from '../../util/mu'; +import { + getBooleanSparqlResult, + getSparqlResults, +} from '../../util/sparql-result'; +import { getUuidForUri } from '../../util/uuid-for-uri'; + +export async function copyFractionInfo(mandatarisFullInfo: MandatarisFullInfo) { + const nonExistingFractions = + await checkForFractionsThatDontExist(mandatarisFullInfo); + if (nonExistingFractions) { + return; + } + await updateFractionName(mandatarisFullInfo); + const duplicateMemberships = + await checkForDuplicateMemberships(mandatarisFullInfo); + if (duplicateMemberships) { + return; + } + await copyMembership(mandatarisFullInfo); + await fixLidmaatschapTijdsinterval( + mandatarisFullInfo.mandatarisUri, + mandatarisFullInfo.graph, + ); +} + +const checkForFractionsThatDontExist = async ( + mandatarisFullInfo: MandatarisFullInfo, +) => { + const mandatarisUri = sparqlEscapeUri(mandatarisFullInfo.mandatarisUri); + const graph = mandatarisFullInfo.graph; + const query = ` + PREFIX org: + PREFIX mandaat: + + ASK WHERE { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${mandatarisUri} org:hasMembership ?membership . + ?membership org:organisation ?fractie . + } + FILTER NOT EXISTS { + GRAPH ${sparqlEscapeUri(graph)} { + ?fractie a mandaat:Fractie . + } + } + } + `; + const result = await querySudo(query); + const hasUnknownFractions = getBooleanSparqlResult(result); + if (hasUnknownFractions) { + await createMandatarisBesluitNotification({ + title: 'Onbekende fractie', + description: `Mandataris met uri ${mandatarisFullInfo.mandatarisUri} heeft een fractie die niet gekend is in de applicatie. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, + type: 'error', + info: mandatarisFullInfo, + }); + return true; + } else { + return false; + } +}; + +const updateFractionName = async (mandatarisFullInfo: MandatarisFullInfo) => { + const mandatarisUri = sparqlEscapeUri(mandatarisFullInfo.mandatarisUri); + const graph = sparqlEscapeUri(mandatarisFullInfo.graph); + const query = ` + PREFIX org: + PREFIX mandaat: + PREFIX regorg: + + DELETE { + GRAPH ${graph} { + ?fractie regorg:legalName ?oldName . + } + } + INSERT { + GRAPH ${graph} { + ?fractie regorg:legalName ?name . + } + } + WHERE { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${mandatarisUri} org:hasMembership ?membership . + ?membership org:organisation ?fractie . + ?fractie regorg:legalName ?name . + } + GRAPH ${graph} { + ?fractie regorg:legalName ?oldName . + } + FILTER(?oldName != ?name) + } + `; + await querySudo(query); +}; + +const checkForDuplicateMemberships = async ( + mandatarisFullInfo: MandatarisFullInfo, +) => { + const mandatarisUri = sparqlEscapeUri(mandatarisFullInfo.mandatarisUri); + const query = ` + PREFIX org: + ASK WHERE { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${mandatarisUri} org:hasMembership ?membership . + ${mandatarisUri} org:hasMembership ?membership2 . + FILTER(?membership != ?membership2) + } + } + `; + const result = await querySudo(query); + const hasUnknownFractions = getBooleanSparqlResult(result); + if (hasUnknownFractions) { + await createMandatarisBesluitNotification({ + title: 'Dubbele fractie', + description: `Mandataris met uri ${mandatarisFullInfo.mandatarisUri} heeft meerdere fracties op hetzelfde moment in het besluit. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, + type: 'error', + info: mandatarisFullInfo, + }); + return true; + } else { + return false; + } +}; + +const copyMembership = async (mandatarisFullInfo: MandatarisFullInfo) => { + const hasNewMemberships = + await hasMembershipsWithDifferences(mandatarisFullInfo); + if (!hasNewMemberships) { + // no different memberships specified. keep the original data + return; + } + + await removeOldMemberships(mandatarisFullInfo); + await addNewMemberships(mandatarisFullInfo); +}; + +const hasMembershipsWithDifferences = async ( + mandatarisFullInfo: MandatarisFullInfo, +) => { + const mandatarisUri = sparqlEscapeUri(mandatarisFullInfo.mandatarisUri); + const query = ` + PREFIX org: + ASK WHERE { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${mandatarisUri} org:hasMembership ?subject . + ?subject ?predicate ?fractie . + VALUES ?predicate { + org:organisation + } + } + FILTER NOT EXISTS { + GRAPH ${sparqlEscapeUri(mandatarisFullInfo.graph)} { + ?subject ?predicate ?fractie . + } + } + } + `; + const result = await querySudo(query); + return getBooleanSparqlResult(result); +}; + +const removeOldMemberships = async (mandatarisFullInfo: MandatarisFullInfo) => { + const mandatarisUri = sparqlEscapeUri(mandatarisFullInfo.mandatarisUri); + const query = ` + PREFIX org: + DELETE { + GRAPH ${sparqlEscapeUri(mandatarisFullInfo.graph)} { + ${mandatarisUri} org:hasMembership ?membership . + ?membership ?p ?o . + ?membership org:memberDuring ?interval. + ?interval ?p2 ?o2 . + } + } WHERE { + GRAPH ${sparqlEscapeUri(mandatarisFullInfo.graph)} { + ${mandatarisUri} org:hasMembership ?membership . + ?membership ?p ?o . + OPTIONAL { + ?membership org:memberDuring ?interval. + ?interval ?p2 ?o2 . + } + } + } + `; + await querySudo(query); +}; + +const addNewMemberships = async (mandatarisFullInfo: MandatarisFullInfo) => { + const membershipUri = await getMembershipUri( + mandatarisFullInfo.mandatarisUri, + ); + const mandatarisUri = sparqlEscapeUri(mandatarisFullInfo.mandatarisUri); + const graph = sparqlEscapeUri(mandatarisFullInfo.graph); + const id = await getUuidForUri(membershipUri, { + allowCheckingUri: true, + allowGenerateUuid: true, + }); + + const query = ` + PREFIX org: + PREFIX mu: + + INSERT { + GRAPH ${graph} { + ${mandatarisUri} org:hasMembership ?membership . + ?membership ?p ?o ; + a org:Membership ; + mu:uuid ${sparqlEscapeString(id)} . + } + } + WHERE { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${mandatarisUri} org:hasMembership ?membership . + ?membership ?p ?o . + } + } + `; + await querySudo(query); +}; + +const getMembershipUri = async (mandatarisUri: string) => { + const query = ` + PREFIX org: + SELECT ?membership WHERE { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${sparqlEscapeUri(mandatarisUri)} org:hasMembership ?membership . + } + } + `; + const result = await querySudo(query); + const results = getSparqlResults(result); + return results[0]?.membership?.value; +}; diff --git a/controllers/mandataris-besluit/mandataris.ts b/controllers/mandataris-besluit/mandataris.ts new file mode 100644 index 0000000..0495f53 --- /dev/null +++ b/controllers/mandataris-besluit/mandataris.ts @@ -0,0 +1,106 @@ +import { + checkIfAllPropertiesAccountedFor, + checkIfMandatarisExists, + insertTriplesInGraph, + replacePropertiesOnInstance, +} from '../../data-access/mandatees-decisions'; +import { MandatarisFullInfo, Triple } from '../../types'; +import { createMandatarisBesluitNotification } from '../../util/create-notification'; +import { getUuidForUri } from '../../util/uuid-for-uri'; + +export async function copyMandatarisInfo( + mandatarisFullInfo: MandatarisFullInfo, +) { + const mandatarisSubject = mandatarisFullInfo.mandatarisUri; + const mandatarisExists = await checkIfMandatarisExists(mandatarisSubject); + if (mandatarisExists) { + await copyMandatarisToExisting(mandatarisFullInfo); + } else { + await createNewMandataris(mandatarisFullInfo); + } +} + +async function copyMandatarisToExisting( + mandatarisFullInfo: MandatarisFullInfo, +) { + const mandatarisSubject = mandatarisFullInfo.mandatarisUri; + const mandatarisTriples = mandatarisFullInfo.triples; + const graph = mandatarisFullInfo.graph; + const allPropertiesAccountedFor = await checkIfAllPropertiesAccountedFor( + mandatarisSubject, + mandatarisTriples, + graph, + ); + if (!allPropertiesAccountedFor) { + await replacePropertiesOnInstance( + mandatarisSubject, + mandatarisTriples, + graph, + ); + await createMandatarisBesluitNotification({ + title: 'Mandataris aangepast', + description: `Mandataris met uri ${mandatarisSubject} werd aangepast op basis van de informatie in een Besluit.`, + type: 'info', + info: mandatarisFullInfo, + }); + } +} + +async function createNewMandataris(mandatarisFullInfo: MandatarisFullInfo) { + const mandatarisSubject = mandatarisFullInfo.mandatarisUri; + const mandatarisTriples = mandatarisFullInfo.triples; + const graph = mandatarisFullInfo.graph; + + const uuid = await getUuidForUri(mandatarisSubject, { + allowCheckingUri: true, + allowGenerateUuid: true, + }); + + const extraTriples: Triple[] = [ + { + subject: { + value: mandatarisSubject, + type: 'uri', + }, + predicate: { + value: 'http://mu.semte.ch/vocabularies/core/uuid', + type: 'uri', + }, + object: { + value: uuid, + type: 'string', + }, + }, + { + subject: { + value: mandatarisSubject, + type: 'uri', + }, + predicate: { + value: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + type: 'uri', + }, + object: { + value: 'http://data.vlaanderen.be/ns/mandaat#Mandataris', + type: 'uri', + }, + }, + ]; + const allTriples = [...mandatarisTriples, ...extraTriples]; + + const statusFound = allTriples.find( + (triple) => + triple.predicate.value === 'http://data.vlaanderen.be/ns/mandaat#status', + ); + const statusWarning = statusFound + ? '' + : ' Let op: status van de mandataris werd niet gevonden in het Besluit!'; + + await insertTriplesInGraph(allTriples, graph); + await createMandatarisBesluitNotification({ + title: 'Mandataris aangemaakt', + description: `Een nieuwe Mandataris met uri ${mandatarisSubject} werd aangemaakt op basis van de informatie in een Besluit.${statusWarning}`, + type: 'warning', + info: mandatarisFullInfo, + }); +} diff --git a/controllers/mandataris-besluit/persoon.ts b/controllers/mandataris-besluit/persoon.ts new file mode 100644 index 0000000..04481a5 --- /dev/null +++ b/controllers/mandataris-besluit/persoon.ts @@ -0,0 +1,77 @@ +import { + copyPersonToGraph, + getGraphsWhereInstanceExists, + getPersonTriplesInStagingGraph, + insertTriplesInGraph, +} from '../../data-access/mandatees-decisions'; +import { MandatarisFullInfo, Triple } from '../../types'; +import { createMandatarisBesluitNotification } from '../../util/create-notification'; +import { getUuidForUri } from '../../util/uuid-for-uri'; + +export async function copyPersonInfo(mandatarisFullInfo: MandatarisFullInfo) { + // this must exist because we checked earlier if the minimal info was available + const persoonUri = mandatarisFullInfo.triples.find( + (triple) => + triple.predicate.value === + 'http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan', + )?.object?.value as string; + + const graphsForPerson = await getGraphsWhereInstanceExists(persoonUri); + const graph = mandatarisFullInfo.graph; + const inAppropriateGraph = graphsForPerson.find((g) => + ['http://mu.semte.ch/graphs/public', graph].includes(g.graph.value), + ); + if (graphsForPerson.length === 0) { + await createPerson(persoonUri, graph); + await createMandatarisBesluitNotification({ + title: 'Persoon aangemaakt', + description: `Een nieuwe Persoon met uri ${persoonUri} werd aangemaakt op basis van de informatie in het Besluit. Deze Persoon zal onvolledige informatie bevatten aangezien e.g. rijksregisternummer niet gepubliceerd wordt in het Besluit.`, + type: 'warning', + info: mandatarisFullInfo, + }); + } else if (!inAppropriateGraph) { + await copyPersonToGraph(persoonUri, graph); + } else { + // person exists in an appropriate graph. nothing to do + } +} + +async function createPerson(persoonUri: string, graph: string) { + const triplesForPerson = await getPersonTriplesInStagingGraph(persoonUri); + const id = await getUuidForUri(persoonUri, { + allowCheckingUri: true, + allowGenerateUuid: true, + }); + const extraTriples: Triple[] = [ + { + subject: { + value: persoonUri, + type: 'uri', + }, + predicate: { + value: 'http://mu.semte.ch/vocabularies/core/uuid', + type: 'uri', + }, + object: { + value: id, + type: 'string', + }, + }, + { + subject: { + value: persoonUri, + type: 'uri', + }, + predicate: { + value: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + type: 'uri', + }, + object: { + value: 'http://www.w3.org/ns/person#Person', + type: 'uri', + }, + }, + ]; + const allTriples = [...triplesForPerson, ...extraTriples]; + await insertTriplesInGraph(allTriples, graph); +} diff --git a/controllers/mandatees-decisions.ts b/controllers/mandatees-decisions.ts index 3f1f04d..f6587a7 100644 --- a/controllers/mandatees-decisions.ts +++ b/controllers/mandatees-decisions.ts @@ -3,68 +3,112 @@ import { findBestuurseenheidForMandaat } from '../data-access/bestuurseenheid'; import { endExistingMandataris, findDecisionForMandataris, - findStartDateOfMandataris as findStartDateOfMandataris, + findStartDateOfMandataris, updatePublicationStatusOfMandataris, } from '../data-access/mandataris'; import { - findPersoonForMandatarisInGraph, + TERM_MANDATARIS_TYPE, + TERM_STAGING_GRAPH, + checkIfMinimalMandatarisInfoAvailable, getMandateOfMandataris as findMandateOfMandataris, - getQuadsInLmbFromTriples, + findNameOfPersoonFromStaging, findOverlappingMandataris, + findPersoonForMandatarisInGraph, + getMandatarisTriplesInStagingGraph, + getQuadsInLmbFromTriples, + getTriplesOfSubject, insertTriplesInGraph, isMandatarisInTarget as isMandatarisInLmbDatabase, - updateDifferencesOfMandataris, - getTriplesOfSubject, - TERM_STAGING_GRAPH, - findNameOfPersoonFromStaging, isSubjectOfType, - TERM_MANDATARIS_TYPE, + updateDifferencesOfMandataris, } from '../data-access/mandatees-decisions'; import { checkPersonExistsAllGraphs, copyPerson, createrPersonFromUri, } from '../data-access/persoon'; -import { mandatarisQueue } from '../routes/delta'; -import { Term } from '../types'; +import { MandatarisBesluitLookup, MandatarisFullInfo } from '../types'; import { PUBLICATION_STATUS } from '../util/constants'; +import { copyBeleidsdomeinInfo } from './mandataris-besluit/beleidsdomein'; +import { copyFractionInfo } from './mandataris-besluit/fractie'; +import { copyMandatarisInfo } from './mandataris-besluit/mandataris'; +import { copyPersonInfo } from './mandataris-besluit/persoon'; export async function processMandatarisForDecisions( - mandatarisSubject: Term, + mandatarisSubject: string, ): Promise { - const isMandataris = await isSubjectOfType( - TERM_MANDATARIS_TYPE, + const { valid, besluitUri, type } = + await isValidMandataris(mandatarisSubject); + if (!valid || !besluitUri) { + return; + } + const mandatarisPointer: MandatarisBesluitLookup = { + mandatarisUri: mandatarisSubject, + besluitUri, + type, + }; + await handleMandatarisSubject(mandatarisPointer); + await updatePublicationStatusOfMandataris( mandatarisSubject, + PUBLICATION_STATUS.BEKRACHTIGD, + ); +} + +async function isValidMandataris(mandataris: string) { + const isMandataris = await isSubjectOfType( + TERM_MANDATARIS_TYPE.value, + mandataris, ); if (!isMandataris) { console.log( - `|> URI: ${mandatarisSubject.value} is not of type: ${TERM_MANDATARIS_TYPE.value}`, + `|> URI: ${mandataris} is not of type: ${TERM_MANDATARIS_TYPE.value}`, ); - return; + return { valid: false, besluitUri: null, type: null }; } - // The decision is actually a besluit:Artikel this + // The decision can also be a besluit:Artikel this // because the besluit does not have a direct relation to the mandataris yet - const decision = await findDecisionForMandataris(mandatarisSubject); - if (!decision) { - console.log( - `|> Could not find a decision for mandataris: ${mandatarisSubject.value}`, - ); - mandatarisQueue.addToManualQueue(mandatarisSubject); - return; + const result = await findDecisionForMandataris(mandataris); + if (!result) { + console.log(`|> Could not find a decision for mandataris: ${mandataris}`); + return { valid: false, besluitUri: null, type: null }; } + return { + valid: true, + besluitUri: result.besluit, + type: + result.link.toLocaleLowerCase().indexOf('ontslag') >= 0 + ? ('ontslag' as const) + : ('aanstelling' as const), + }; +} - await handleTriplesForMandatarisSubject(mandatarisSubject); - await updatePublicationStatusOfMandataris( - mandatarisSubject, - PUBLICATION_STATUS.BEKRACHTIGD, +export async function handleMandatarisSubject( + mandatarisBesluitInfo: MandatarisBesluitLookup, +) { + const { graph, minimalInfoAvailable } = + await checkIfMinimalMandatarisInfoAvailable(mandatarisBesluitInfo); + if (!minimalInfoAvailable || !graph) { + return; + } + const mandatarisTriples = await getMandatarisTriplesInStagingGraph( + mandatarisBesluitInfo.mandatarisUri, ); + + const mandatarisFullInfo: MandatarisFullInfo = { + ...mandatarisBesluitInfo, + triples: mandatarisTriples, + graph, + }; + + await copyMandatarisInfo(mandatarisFullInfo); + await copyPersonInfo(mandatarisFullInfo); + await copyFractionInfo(mandatarisFullInfo); + await copyBeleidsdomeinInfo(mandatarisFullInfo); } -export async function handleTriplesForMandatarisSubject( - mandatarisSubject: Term, -) { - console.log(`|> Mandataris uri: ${mandatarisSubject.value}`); +export async function oldHandleMandatarisSubject(mandatarisSubject: string) { + console.log(`|> Mandataris uri: ${mandatarisSubject}`); const isExitingInLmbDatabase = await isMandatarisInLmbDatabase(mandatarisSubject); console.log( diff --git a/cron/handle-decision-queue.ts b/cron/handle-decision-queue.ts new file mode 100644 index 0000000..e509d3a --- /dev/null +++ b/cron/handle-decision-queue.ts @@ -0,0 +1,103 @@ +import { CronJob } from 'cron'; +import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; +import { sparqlEscapeDateTime, sparqlEscapeUri } from 'mu'; +import { processMandatarisForDecisions } from '../controllers/mandatees-decisions'; +import { createNotification } from '../util/create-notification'; + +const BESLUIT_CRON_PATTERN = + process.env.BESLUIT_CRON_PATTERN || '0 */5 * * * *'; // Every 5 minutes +const BESLUIT_BUFFER_TIME = parseInt( + process.env.BESLUIT_BUFFER_TIME || '300000', +); // 5 minutes in milliseconds +const BESLUIT_BATCH_SIZE = parseInt(process.env.BESLUIT_BATCH_SIZE || '100'); +let running = false; +export const cronjob = CronJob.from({ + cronTime: BESLUIT_CRON_PATTERN, + onTick: async () => { + if (running) { + return; + } + running = true; + await handleBesluitQueue(); + running = false; + }, +}); + +async function handleBesluitQueue() { + const batch = await fetchBatchOfMandatarisInstances(); + + if (batch.length === 0) { + return; + } + + console.log(`Processing ${batch.length} mandataris instances`); + for (const match of batch) { + await safeProcessMandatarisForDecisions(match); + } + await cleanInstancesFromQueue(batch); +} + +async function safeProcessMandatarisForDecisions(match) { + await processMandatarisForDecisions(match.mandataris).catch(async (e) => { + console.log( + `ERROR processing mandataris decision instance ${match.instance}: ${e.message}`, + ); + await createNotification({ + title: 'Error tijdens verwerken van Besluit voor mandataris', + description: `Error tijdens verwerken van Besluit voor ${match.instance}. Gelieve de logs na te kijken voor meer informatie.`, + type: 'error', + graph: match.instance, + links: [ + { + type: 'mandataris', + uri: match.mandataris, + }, + ], + }).catch((e) => { + console.log( + `ERROR creating notification for error to process for mandataris decision instance: ${e.message}`, + ); + }); + }); +} + +async function fetchBatchOfMandatarisInstances() { + const bufferTimeAgo = new Date(Date.now() - BESLUIT_BUFFER_TIME); + const query = ` + PREFIX ext: + + SELECT ?instance ?mandataris WHERE { + GRAPH { + ?instance ext:queueInstance ?mandataris ; + ext:queueTime ?time . + + FILTER(?time < ${sparqlEscapeDateTime(bufferTimeAgo)}) + } + } LIMIT ${BESLUIT_BATCH_SIZE}`; + + const result = await querySudo(query); + return result.results.bindings.map((binding) => { + return { + mandataris: binding.mandataris.value, + instance: binding.instance.value, + }; + }); +} + +async function cleanInstancesFromQueue(batch) { + const query = ` + DELETE { + GRAPH { + ?instance ?p ?o . + } + } WHERE { + GRAPH { + VALUES ?instance { + ${batch.map((match) => sparqlEscapeUri(match.instance)).join(' ')} + } + ?instance ?p ?o . + } + }`; + + await updateSudo(query); +} diff --git a/data-access/lidmaatschap.ts b/data-access/lidmaatschap.ts new file mode 100644 index 0000000..fc2bfb2 --- /dev/null +++ b/data-access/lidmaatschap.ts @@ -0,0 +1,51 @@ +import { v4 as uuidv4 } from 'uuid'; +import { sparqlEscapeUri, sparqlEscapeString } from 'mu'; +import { updateSudo } from '@lblod/mu-auth-sudo'; + +export async function fixLidmaatschapTijdsinterval( + mandatarisUri: string, + graph: string, +) { + const id = uuidv4(); + const intervalUri = `http://data.lblod.info/id/tijdsintervallen//${id}`; + const safeMandatarisUri = sparqlEscapeUri(mandatarisUri); + const safeGraph = sparqlEscapeUri(graph); + const safeIntervalUri = sparqlEscapeUri(intervalUri); + + const query = ` + PREFIX org: + PREFIX dct: + PREFIX generiek: + PREFIX mu: + PREFIX mandaat: + + DELETE { + GRAPH ${safeGraph} { + ?membership org:memberDuring ?interval. + ?interval ?p ?o. + } + } + INSERT { + GRAPH ${safeGraph} { + ?membership org:memberDuring ${safeIntervalUri}. + ${safeIntervalUri} a dct:PeriodOfTime ; + mu:uuid ${sparqlEscapeString(id)} ; + generiek:begin ?start ; + generiek:einde ?end . + } + } + WHERE { + GRAPH ${safeGraph} { + ${safeMandatarisUri} org:hasMembership ?membership ; + mandaat:start ?start . + OPTIONAL { + ${safeMandatarisUri} mandaat:einde ?end . + } + OPTIONAL { + ?membership org:memberDuring ?interval. + ?interval ?p ?o. + } + } + }`; + await updateSudo(query); +} diff --git a/data-access/mandataris.ts b/data-access/mandataris.ts index 8f4625f..e4b7c7c 100644 --- a/data-access/mandataris.ts +++ b/data-access/mandataris.ts @@ -543,35 +543,41 @@ export async function findStartDateOfMandataris( } export async function findDecisionForMandataris( - mandataris: Term, -): Promise { - const mandatarisSubject = sparqlEscapeTermValue(mandataris); + mandataris: string, +): Promise<{ besluit: string; link: string } | null> { + const mandatarisSubject = sparqlEscapeUri(mandataris); const besluiteQuery = ` - PREFIX mandaat: + PREFIX mandaat: - SELECT ?artikel + SELECT ?artikel ?link WHERE { - OPTIONAL { ?artikel mandaat:bekrachtigtAanstellingVan ${mandatarisSubject}. } - OPTIONAL { ?artikel mandaat:bekrachtigtOntslagVan ${mandatarisSubject}. } + VALUES ?link { + mandaat:bekrachtigtAanstellingVan + mandaat:bekrachtigtOntslagVan + } + ?artikel ?link ${mandatarisSubject}. } `; - const result = await updateSudo(besluiteQuery); + const result = await querySudo(besluiteQuery); const sparqlresult = findFirstSparqlResult(result); if (sparqlresult?.artikel) { - return sparqlresult.artikel; + return { + besluit: sparqlresult.artikel.value, + link: sparqlresult.link.value, + }; } return null; } export async function updatePublicationStatusOfMandataris( - mandataris: Term, + mandataris: string, status: PUBLICATION_STATUS, ): Promise { const escaped = { - mandataris: sparqlEscapeTermValue(mandataris), + mandataris: sparqlEscapeUri(mandataris), status: sparqlEscapeUri(status), mandatarisType: sparqlEscapeTermValue(TERM_MANDATARIS_TYPE), }; @@ -601,11 +607,11 @@ export async function updatePublicationStatusOfMandataris( try { await updateSudo(updateStatusQuery); console.log( - `|> Updated status to ${status} for mandataris: ${mandataris.value}.`, + `|> Updated status to ${status} for mandataris: ${mandataris}.`, ); } catch (error) { console.log( - `|> Could not update mandataris: ${mandataris.value} status to ${status}`, + `|> Could not update mandataris: ${mandataris} status to ${status}`, ); } } diff --git a/data-access/mandatees-decisions.ts b/data-access/mandatees-decisions.ts index 6bfe0ab..44edd76 100644 --- a/data-access/mandatees-decisions.ts +++ b/data-access/mandatees-decisions.ts @@ -1,6 +1,12 @@ import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; import { sparqlEscapeUri } from 'mu'; -import { Quad, Term, TermProperty, Triple } from '../types'; +import { + MandatarisBesluitLookup, + Quad, + Term, + TermProperty, + Triple, +} from '../types'; import { findFirstSparqlResult, getBooleanSparqlResult, @@ -8,6 +14,15 @@ import { } from '../util/sparql-result'; import { TERM_TYPE, sparqlEscapeTermValue } from '../util/sparql-escape'; import { MANDATARIS_STATUS, PUBLICATION_STATUS } from '../util/constants'; +import { + createNotification, + getMandatarisNotificationGraph, +} from '../util/create-notification'; +import { getUuidForUri } from '../util/uuid-for-uri'; + +export const BESLUIT_STAGING_GRAPH = + process.env.BESLUIT_STAGING_GRAPH || + 'http://mu.semte.ch/graphs/besluiten-consumed'; export const TERM_STAGING_GRAPH = { type: TERM_TYPE.URI, @@ -19,12 +34,12 @@ export const TERM_MANDATARIS_TYPE = { } as Term; export async function isSubjectOfType( - rdfType: Term, - subject: Term, + rdfType: string, + subject: string, ): Promise { const queryForType = ` ASK { - ${sparqlEscapeTermValue(subject)} a ${sparqlEscapeTermValue(rdfType)} . + ${sparqlEscapeUri(subject)} a ${sparqlEscapeUri(rdfType)} . } `; @@ -68,7 +83,7 @@ export async function getQuadsInLmbFromTriples( const query = ` SELECT ?subject ?predicate ?object ?graph WHERE { - GRAPH ?graph { + GRAPH ?graph { VALUES (?subject ?predicate) { ${useAsValues.join('')} } @@ -170,7 +185,7 @@ export async function updateDifferencesOfMandataris( } WHERE { GRAPH ${escaped.graph} { ${escaped.subject} ${escaped.predicate} ${escaped.currentObject} . - } + } MINUS { GRAPH ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)} { ${subjectPredicate} ${escaped.currentObject} @@ -248,7 +263,7 @@ export async function findOverlappingMandataris( ${sparqlEscapeUri(PUBLICATION_STATUS.DRAFT)} } ?mandataris a ${sparqlEscapeTermValue(TERM_MANDATARIS_TYPE)} ; - mandaat:status ?status; + mandaat:status ?status; mandaat:isBestuurlijkeAliasVan ${sparqlEscapeTermValue(persoon)} ; org:holds ${sparqlEscapeTermValue(mandaat)} . } @@ -261,7 +276,7 @@ export async function findOverlappingMandataris( export async function insertTriplesInGraph( triples: Array, - graph: Term, + graph: string, ): Promise { if (triples.length === 0) { return; @@ -277,7 +292,7 @@ export async function insertTriplesInGraph( const insertQuery = ` INSERT DATA { - GRAPH ${sparqlEscapeTermValue(graph)} { + GRAPH ${sparqlEscapeUri(graph)} { ${insertTriples.join('\n')} } } @@ -286,11 +301,11 @@ export async function insertTriplesInGraph( try { await updateSudo(insertQuery, {}, { mayRetry: true }); console.log( - `|> Inserted new mandataris triples (${triples.length}) in graph (${graph.value}).`, + `|> Inserted ${triples.length} new triples in graph (${graph}).`, ); } catch (error) { throw Error( - `Could not insert ${triples.length} triples in graph (${graph.value}).`, + `Could not insert ${triples.length} triples in graph (${graph}).`, ); } } @@ -301,13 +316,13 @@ export async function findNameOfPersoonFromStaging( const mandatarisUri = sparqlEscapeTermValue(mandataris); const queryMandatarisPerson = ` PREFIX foaf: - PREFIX persoon: + PREFIX persoon: SELECT ?persoonUri ?firstname ?lastname WHERE { GRAPH ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)} { ${mandatarisUri} a ${sparqlEscapeTermValue(TERM_MANDATARIS_TYPE)} . - + ${mandatarisUri} persoon:isBestuurlijkeAliasVan ?persoonUri . ${mandatarisUri} persoon:gebruikteVoornaam ?firstname . ${mandatarisUri} foaf:familyName ?lastname . @@ -318,3 +333,319 @@ export async function findNameOfPersoonFromStaging( return findFirstSparqlResult(result); } + +export async function checkIfMinimalMandatarisInfoAvailable( + mandatarisBesluitInfo: MandatarisBesluitLookup, +) { + const mandataris = mandatarisBesluitInfo.mandatarisUri; + const besluitUri = mandatarisBesluitInfo.besluitUri; + + const query = ` + PREFIX mandaat: + PREFIX org: + PREFIX foaf: + PREFIX persoon: + PREFIX ext: + PREFIX person: + + SELECT * { + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${sparqlEscapeUri(mandataris)} a mandaat:Mandataris ; + mandaat:start ?start ; + mandaat:isBestuurlijkeAliasVan ?person ; + org:holds ?mandaat. + FILTER NOT EXISTS { + ${sparqlEscapeUri(mandataris)} org:holds ?otherMandaat. + FILTER (?otherMandaat != ?mandaat) + } + } + # person can be in any graph, either provided, public or bestuurseenheid graph + ?person a person:Person ; + foaf:familyName ?familyName ; + persoon:gebruikteVoornaam ?firstName . + GRAPH ?bestuurseenheidGraph { + ?mandaat a mandaat:Mandaat . + } + ?bestuurseenheidGraph ext:ownedBy ?bestuurseenheid. + } LIMIT 1 + `; + const result = await querySudo(query); + const typedResult = getSparqlResults(result); + if (typedResult.length) { + return { + minimalInfoAvailable: true, + graph: typedResult[0].bestuurseenheidGraph.value, + }; + } else { + const graph = await getMandatarisNotificationGraph(mandataris); + await createNotification({ + title: 'Besluit met Mandataris zonder minimale info', + description: `Mandataris ${mandataris} uit Besluit ${besluitUri} heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een start datum, een persson en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.`, + type: 'error', + graph, + links: [ + { + type: 'mandataris', + uri: mandataris, + }, + { + type: 'besluit', + uri: besluitUri, + }, + ], + }); + return { minimalInfoAvailable: false, graph: null }; + } +} + +export async function checkIfMandatarisExists(mandatarisUri) { + const query = ` + PREFIX ext: + ASK { + GRAPH ?g { + ${sparqlEscapeUri( + mandatarisUri, + )} a . + } + ?g ext:ownedBy ?bestuurseenheid. + } + `; + const result = await querySudo(query); + return getBooleanSparqlResult(result); +} + +export async function getGraphsWhereInstanceExists(instanceUri) { + const query = ` + PREFIX ext: + SELECT ?graph WHERE { + GRAPH ?graph { + ${sparqlEscapeUri(instanceUri)} a ?thing. + } + OPTIONAL { + ?graph ext:ownedBy ?bestuurseenheid. + } + FILTER(?graph = || BOUND(?bestuurseenheid)) + } + `; + const result = await querySudo(query); + return getSparqlResults(result); +} + +export async function getMandatarisTriplesInStagingGraph( + mandatarisUri, +): Promise { + // note: not asking for uuid, keep ours, not asking for membership: done when handling fracties + const query = ` + PREFIX mandaat: + PREFIX org: + + SELECT ?subject ?predicate ?object WHERE { + VALUES ?subject { + ${sparqlEscapeUri(mandatarisUri)} + } + VALUES ?predicate { + mandaat:start + mandaat:einde + mandaat:rangorde + mandaat:beleidsdomein + mandaat:isBestuurlijkeAliasVan + mandaat:status + org:holds + } + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${sparqlEscapeUri(mandatarisUri)} ?predicate ?object . + } + + } + `; + const result = await querySudo(query); + return getSparqlResults(result) as Triple[]; +} + +export async function getPersonTriplesInStagingGraph( + personUri, +): Promise { + const query = ` + PREFIX foaf: + PREFIX persoon: + + SELECT ?subject ?predicate ?object WHERE { + VALUES ?subject { + ${sparqlEscapeUri(personUri)} + } + VALUES ?predicate { + foaf:familyName + foaf:name + persoon:gebruikteVoornaam + } + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${sparqlEscapeUri(personUri)} ?predicate ?object . + } + + } + `; + const result = await querySudo(query); + return getSparqlResults(result) as Triple[]; +} + +export async function getBeleidsdomeinTriplesInStagingGraph( + beleidsdomeinUri, +): Promise { + const query = ` + PREFIX skos: + + SELECT ?subject ?predicate ?object WHERE { + VALUES ?subject { + ${sparqlEscapeUri(beleidsdomeinUri)} + } + VALUES ?predicate { + skos:prefLabel + } + GRAPH ${sparqlEscapeUri(BESLUIT_STAGING_GRAPH)} { + ${sparqlEscapeUri(beleidsdomeinUri)} ?predicate ?object . + } + + } + `; + const result = await querySudo(query); + return getSparqlResults(result) as Triple[]; +} + +export async function replacePropertiesOnInstance( + subject: string, + triples: Triple[], + graph: string, +) { + // just some safety in case someone passes us triples with a different subject + const subjectTriples = triples.filter((t) => t.subject.value == subject); + // This only removes the properties that are in the staging graph and keeps the old ones + // That way we don't accidentally delete information that we want to keep if they are + // not specified in the staging graph like beleidsdomeinen + const predicates = subjectTriples.map((triple) => triple.predicate); + const query = ` + DELETE { + GRAPH ${sparqlEscapeUri(graph)} { + ${sparqlEscapeUri(subject)} ?p ?o . + } + } INSERT { + GRAPH ${sparqlEscapeUri(graph)} { + ${subjectTriples + .map((triple) => { + return `${sparqlEscapeUri(subject)} ${sparqlEscapeUri( + triple.predicate.value, + )} ${sparqlEscapeTermValue(triple.object)} .`; + }) + .join('\n')} + } + } WHERE { + GRAPH ${sparqlEscapeUri(graph)} { + ${sparqlEscapeUri(subject)} ?p ?o . + VALUES ?p { + ${predicates.map((p) => sparqlEscapeUri(p.value)).join(' ')} + } + } + } + `; + + await updateSudo(query); +} + +export async function checkIfAllPropertiesAccountedFor( + subject: string, + triples: Triple[], + graph, +) { + // just some safety in case someone passes us triples with a different subject + const subjectTriples = triples.filter((t) => t.subject.value == subject); + const predicates = subjectTriples.map((triple) => triple.predicate); + + const query = ` + SELECT ?p ?o WHERE { + GRAPH ${sparqlEscapeUri(graph)} { + ${sparqlEscapeUri(subject)} ?p ?o . + VALUES ?p { + ${predicates.map((p) => sparqlEscapeUri(p.value)).join(' ')} + } + } + }`; + const result = await querySudo(query); + const results = getSparqlResults(result); + + const differentLength = results.length !== subjectTriples.length; + if (differentLength) { + return false; + } + const originValueMap = {}; + subjectTriples.forEach((t) => { + originValueMap[t.predicate.value] = originValueMap[t.predicate.value] || []; + originValueMap[t.predicate.value].push(t.object.value); + }); + + let missingResult = false; + results.forEach((r) => { + if ( + !originValueMap[r.p.value] || + !originValueMap[r.p.value].includes(r.o.value) + ) { + missingResult = true; + } + }); + return !missingResult; +} + +export async function copyPersonToGraph(personUri: string, graph: string) { + const safePersonUri = sparqlEscapeUri(personUri); + const query = ` + PREFIX persoon: + PREFIX adms: + PREFIX ext: + + INSERT { + GRAPH ${sparqlEscapeUri(graph)} { + ${safePersonUri} ?p ?o . + ${safePersonUri} persoon:heeftGeboorte ?geboorte . + ?geboorte ?geboorteP ?geboorteO. + ${safePersonUri} adms:identifier ?identifier . + ?identifier ?idP ?idO. + } + } WHERE { + GRAPH ?g { + ${safePersonUri} ?p ?o . + OPTIONAL { + ${safePersonUri} persoon:heeftGeboorte ?geboorte . + ?geboorte ?geboorteP ?geboorteO. + } + OPTIONAL { + ${safePersonUri} adms:identifier ?identifier . + ?identifier ?idP ?idO. + } + } + ?g ext:ownedBy ?bestuurseenheid. + FILTER (?g = || BOUND(?bestuurseenheid)) + }`; + await updateSudo(query); +} + +export async function copySimpleInstanceToGraph( + instanceUri: string, + graph: string, +) { + const safeInstanceUri = sparqlEscapeUri(instanceUri); + const query = ` + PREFIX persoon: + PREFIX adms: + PREFIX ext: + + INSERT { + GRAPH ${sparqlEscapeUri(graph)} { + ${safeInstanceUri} ?p ?o . + } + } WHERE { + GRAPH ?g { + ${safeInstanceUri} ?p ?o . + } + ?g ext:ownedBy ?bestuurseenheid. + FILTER (?g = || BOUND(?bestuurseenheid)) + }`; + await updateSudo(query); +} diff --git a/data-access/persoon.ts b/data-access/persoon.ts index aec8042..4fed5f8 100644 --- a/data-access/persoon.ts +++ b/data-access/persoon.ts @@ -14,7 +14,7 @@ import { findFirstSparqlResult, getBooleanSparqlResult, } from '../util/sparql-result'; -import { getIdentifierFromPersonUri } from '../util/find-uuid-in-uri'; +import { getIdentifierFromPersonUri } from '../util/uuid-for-uri'; // note since we use the regular query, not sudo queries, be sure to log in when using this endpoint. E.g. use the vendor login @@ -139,6 +139,7 @@ export async function createrPersonFromUri( lastname: Term, graph: Term, ): Promise { + // TODO danger! const personIdentifier = getIdentifierFromPersonUri(personUri.value); const createQuery = ` PREFIX mu: diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index a4f6b62..d2d83be 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -2,15 +2,18 @@ services: mandataris: image: local-js-template #build: ./ + #image: semtech/mu-javascript-template:1.8.0 restart: 'no' labels: - 'logging=true' environment: NODE_ENV: development NO_BABEL_NODE: true - EMAIL_FROM_MANDATARIS_EFFECTIEF: "email@address.com" - NOTIFICATION_CRON_PATTERN: "* * * * *" # Every minute + EMAIL_FROM_MANDATARIS_EFFECTIEF: 'email@address.com' + NOTIFICATION_CRON_PATTERN: '* * * * *' # Every minute SEND_EMAIL_FOR_MANDATARIS_EFFECTIEF: false + BESLUIT_CRON_PATTERN: '*/5 * * * * *' + BESLUIT_BUFFER_TIME: 10000 ports: - '8082:80' - '9225:9229' diff --git a/routes/delta-decisions.ts b/routes/delta-decisions.ts new file mode 100644 index 0000000..977c619 --- /dev/null +++ b/routes/delta-decisions.ts @@ -0,0 +1,63 @@ +import Router from 'express-promise-router'; +import { sparqlEscapeUri, sparqlEscapeDateTime } from 'mu'; + +import { Request, Response } from 'express'; + +import { Changeset, Quad } from '../types'; +import { ProcessingQueue } from '../services/processing-queue'; +import { updateSudo } from '@lblod/mu-auth-sudo'; +import { v4 as uuid } from 'uuid'; +import { BESLUIT_STAGING_GRAPH } from '../data-access/mandatees-decisions'; + +const deltaRouter = Router(); +export const mandatarisQueue = new ProcessingQueue(); + +deltaRouter.post('/decisions', async (req: Request, res: Response) => { + console.log('|>Triggered the decisions endpoint!'); + const changesets: Changeset[] = req.body; + const insertTriples = changesets + .map((changeset: Changeset) => changeset.inserts) + .flat() + .filter( + (quad) => + quad.graph.value.startsWith(BESLUIT_STAGING_GRAPH) && + quad.predicate.value.startsWith( + 'http://data.vlaanderen.be/ns/mandaat#bekrachtigt', + ), + ); + + const mandatarisSubjects = Array.from( + new Set(insertTriples.map((quad: Quad) => quad.object.value)), + ); + + // keep these mandataris instances in the database instead of memory + // so if we crash we know they should still be processed + // wait BUFFER_TIME to process the mandataris so we are reasonably sure that we have all the info + const query = ` + PREFIX ext: + + INSERT DATA { + GRAPH { + ${mandatarisSubjects + .map((mandataris: string) => { + const id = uuid(); + const instanceUri = sparqlEscapeUri( + `http://mu.semte.ch/vocabularies/ext/queueInstance/${id}`, + ); + const mandatarisUri = sparqlEscapeUri(mandataris); + return `${instanceUri} ext:queueInstance ${mandatarisUri} ; + ext:queueTime ${sparqlEscapeDateTime(new Date())} .`; + }) + .join('\n')} + } + }`; + + await updateSudo(query).catch((e) => { + res.status(500).send({ status: 'error', message: e.message }); + return; + }); + + res.status(200).send({ status: 'ok' }); +}); + +export { deltaRouter }; diff --git a/routes/delta.ts b/routes/delta.ts deleted file mode 100644 index 8413e0a..0000000 --- a/routes/delta.ts +++ /dev/null @@ -1,30 +0,0 @@ -import Router from 'express-promise-router'; - -import { Request, Response } from 'express'; - -import { Changeset, Quad } from '../types'; -import { processMandatarisForDecisions } from '../controllers/mandatees-decisions'; -import { ProcessingQueue } from '../services/processing-queue'; - -const deltaRouter = Router(); -export const mandatarisQueue = new ProcessingQueue(); - -deltaRouter.post('/decisions', async (req: Request, res: Response) => { - console.log('|>Triggered the decisions endpoint!'); - const changesets: Changeset[] = req.body; - const insertTriples = changesets - .map((changeset: Changeset) => changeset.inserts) - .flat(); - - const mandatarisSubjects = Array.from( - new Set(insertTriples.map((quad: Quad) => quad.object)), - ); - - mandatarisQueue.setMethodToExecute(processMandatarisForDecisions); - mandatarisQueue.moveManualQueueToQueue(); - mandatarisQueue.addToQueue(mandatarisSubjects); - - return res.status(200).send({ status: 'ok' }); -}); - -export { deltaRouter }; diff --git a/routes/mock.ts b/routes/mock.ts new file mode 100644 index 0000000..4658837 --- /dev/null +++ b/routes/mock.ts @@ -0,0 +1,393 @@ +import Router from 'express-promise-router'; + +import { Request, Response } from 'express'; +import { sparqlEscapeDateTime } from 'mu'; +import { updateSudo } from '@lblod/mu-auth-sudo'; +import { v4 as uuidv4 } from 'uuid'; + +export const mockRouter = Router(); + +mockRouter.get('/add-decision', async (req: Request, res: Response) => { + console.log('\n \t|>Triggered the add decision endpoint'); + + let action = createDecisionForKnownMandatarisAndPerson; + + if (req.query.new) { + action = createDecisionForNewMandatarisAndPerson; + } else if (req.query.fraction) { + action = createDecisionForUnknownFraction; + } else if (req.query.unknownMandaat) { + action = createDecisionForUnknownMandaat; + } else if (req.query.incompletePerson) { + action = createDecisionForIncompleteNewPerson; + } else if (req.query.changeFractionName) { + action = createDecisionForKnownMandatarisChangeFractionName; + } else if (req.query.newBeleidsdomein) { + action = createDecisionForKnownMandatarisNewBeleidsdomein; + } + try { + await action(); + } catch (error) { + res.status(500).send({ status: 'error', error: error.message }); + return; + } + res.status(200).send({ status: 'ok' }); +}); + +mockRouter.get('/clear-decisions', async (_req: Request, res: Response) => { + //the mock functions here will create inconsistent states in the consumer graph. Clear it using this endpoint + const query = ` + DELETE { + GRAPH { + ?s ?p ?o. + } + } WHERE { + GRAPH { + ?s ?p ?o. + } + }`; + await updateSudo(query); + res.status(200).send({ status: 'ok' }); +}); + +const createDecisionForKnownMandatarisAndPerson = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = '5FF2DC1278A81C0009000A15'; + const mandataris = ``; + const persoonId = + '1656cfde62b97fe365c5bc3813a8d7d4a76f0e14b1b9aacf4ae9e8558347aeb6'; + const persoon = ``; + const mandaat = + ''; + const fractie = + ''; + + const membershipId = uuidv4(); + const membership = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX foaf: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + } + } + `; + await updateSudo(insertQuery); +}; + +const createDecisionForKnownMandatarisNewBeleidsdomein = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = '5FF2DC1278A81C0009000A15'; + const mandataris = ``; + const persoonId = + '1656cfde62b97fe365c5bc3813a8d7d4a76f0e14b1b9aacf4ae9e8558347aeb6'; + const persoon = ``; + const mandaat = + ''; + const fractie = + ''; + + const membershipId = uuidv4(); + const membership = ``; + + const beleidsdomeinId = uuidv4(); + const beleidsdomeinName = 'foobar'; + const beleidsdomeinUri = ``; + + const existingBeleidsdomeinUri = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX foaf: + PREFIX skos: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + + ${mandataris} mandaat:beleidsdomein ${beleidsdomeinUri}, ${existingBeleidsdomeinUri}. + ${beleidsdomeinUri} a mandaat:Beleidsdomein; + skos:prefLabel "${beleidsdomeinName}" . + ${existingBeleidsdomeinUri} a mandaat:Beleidsdomein; + skos:prefLabel "FAIL SHOULD NOT CHANGE" . + } + } + `; + await updateSudo(insertQuery); +}; + +const createDecisionForKnownMandatarisChangeFractionName = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = '5FF2DC1278A81C0009000A15'; + const mandataris = ``; + const persoonId = + '1656cfde62b97fe365c5bc3813a8d7d4a76f0e14b1b9aacf4ae9e8558347aeb6'; + const persoon = ``; + const mandaat = + ''; + const fractie = + ''; + + const membershipId = uuidv4(); + const membership = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX foaf: + PREFIX regorg: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + + ${fractie} regorg:legalName "New name" ; + a mandaat:Fractie . + } + } + `; + await updateSudo(insertQuery); +}; + +const createDecisionForNewMandatarisAndPerson = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = uuidv4(); + const mandataris = ``; + const persoonId = uuidv4(); + const persoon = ``; + const mandaat = + ''; + const fractie = + ''; + + const membershipId = uuidv4(); + const membership = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX person: + PREFIX foaf: + PREFIX persoon: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + + ${persoon} a person:Person; + foaf:familyName "Doe"; + persoon:gebruikteVoornaam "John"; + foaf:name "Johnny" . + + } + } + `; + await updateSudo(insertQuery); +}; + +const createDecisionForUnknownFraction = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = '5FF2DC1278A81C0009000A15'; + const mandataris = ``; + const persoonId = + '1656cfde62b97fe365c5bc3813a8d7d4a76f0e14b1b9aacf4ae9e8558347aeb6'; + const persoon = ``; + const mandaat = + ''; + const fractieId = uuidv4(); + const fractie = ``; + + const membershipId = uuidv4(); + const membership = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX foaf: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + } + } + `; + await updateSudo(insertQuery); +}; + +const createDecisionForUnknownMandaat = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = '5FF2DC1278A81C0009000A15'; + const mandataris = ``; + const persoonId = + '1656cfde62b97fe365c5bc3813a8d7d4a76f0e14b1b9aacf4ae9e8558347aeb6'; + const persoon = ``; + const mandaat = ``; + const fractie = + ''; + + const membershipId = uuidv4(); + const membership = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX foaf: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + } + } + `; + await updateSudo(insertQuery); +}; + +const createDecisionForIncompleteNewPerson = async () => { + const today = sparqlEscapeDateTime(new Date()); + const id = uuidv4(); + const mandatarisId = uuidv4(); + const mandataris = ``; + const persoonId = uuidv4(); + const persoon = ``; + const mandaat = + ''; + const fractie = + ''; + + const membershipId = uuidv4(); + const membership = ``; + + const insertQuery = ` + PREFIX mandaat: + PREFIX mu: + PREFIX besluit: + PREFIX org: + PREFIX xsd: + PREFIX person: + PREFIX foaf: + + INSERT DATA { + GRAPH { + a besluit:Besluit; + mu:uuid """${id}"""; + mandaat:bekrachtigtOntslagVan ${mandataris}. + + ${mandataris} a mandaat:Mandataris; + mu:uuid """${mandatarisId}"""; + mandaat:isBestuurlijkeAliasVan ${persoon}; + mandaat:start ${today} ; + org:holds ${mandaat}. + + ${mandataris} org:hasMembership ${membership}. + ${membership} a org:Membership; + org:organisation ${fractie} . + + ${persoon} a person:Person; + foaf:name "Johnny" . + + } + } + `; + await updateSudo(insertQuery); +}; diff --git a/util/find-uuid-in-uri.ts b/util/find-uuid-in-uri.ts deleted file mode 100644 index 16aed35..0000000 --- a/util/find-uuid-in-uri.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { v4 as uuidv4 } from 'uuid'; - -export function getIdentifierFromPersonUri(uri: string) { - const personBaseUri = 'http://data.lblod.info/id/personen/'; - if (!uri.includes(personBaseUri)) { - return uuidv4(); - } - - return uri.replace(personBaseUri, '').trim(); -} diff --git a/util/uuid-for-uri.ts b/util/uuid-for-uri.ts new file mode 100644 index 0000000..724a01d --- /dev/null +++ b/util/uuid-for-uri.ts @@ -0,0 +1,60 @@ +import { v4 as uuidv4 } from 'uuid'; +import { sparqlEscapeUri } from 'mu'; +import { querySudo } from '@lblod/mu-auth-sudo'; + +export function getIdentifierFromPersonUri(uri: string) { + const personBaseUri = 'http://data.lblod.info/id/personen/'; + if (!uri.includes(personBaseUri)) { + return uuidv4(); + } + + return uri.replace(personBaseUri, '').trim(); +} + +export function getIdentifierFromUri(uri: string) { + const uuid = uri.split('/').pop(); + if ( + uuid && + uuid + .toLocaleLowerCase() + .match( + '(^[a-f0-9]{8}(-)?[a-f0-9]{4}(-)?[a-f0-9]{4}(-)?[a-f0-9]{4}(-)?[a-f0-9]{12}$)', + ) + ) { + return uuid; + } else { + return null; + } +} + +export async function getUuidForUri( + uri: string, + options?: { + allowCheckingUri: boolean; + allowGenerateUuid: boolean; + }, +) { + const query = ` + PREFIX mu: + + SELECT ?uuid WHERE { + ${sparqlEscapeUri(uri)} mu:uuid ?uuid . + } LIMIT 1`; + + const result = await querySudo(query); + if (result.results.bindings.length == 1) { + return result.results.bindings[0].uuid.value; + } + if (!options?.allowCheckingUri && !options?.allowGenerateUuid) { + return null; + } + + let id: string | null = null; + if (options?.allowCheckingUri) { + id = getIdentifierFromUri(uri); + } + if (!id && options?.allowGenerateUuid) { + id = uuidv4(); + } + return id; +} From 3efdc16c3309925f4012153e041c0f777370c83e Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 13 Sep 2024 16:27:52 +0200 Subject: [PATCH 2/7] remove old way of handling decisions --- controllers/mandatees-decisions.ts | 153 ---------------- data-access/bestuurseenheid.ts | 78 --------- data-access/mandataris.ts | 22 --- data-access/mandatees-decisions.ts | 269 +---------------------------- data-access/persoon.ts | 134 +------------- routes/delta-decisions.ts | 2 - routes/mock.ts | 3 +- services/processing-queue.ts | 80 --------- 8 files changed, 10 insertions(+), 731 deletions(-) delete mode 100644 data-access/bestuurseenheid.ts delete mode 100644 services/processing-queue.ts diff --git a/controllers/mandatees-decisions.ts b/controllers/mandatees-decisions.ts index f6587a7..b1f27b7 100644 --- a/controllers/mandatees-decisions.ts +++ b/controllers/mandatees-decisions.ts @@ -1,32 +1,13 @@ -import { findBestuurseenheidForMandaat } from '../data-access/bestuurseenheid'; - import { - endExistingMandataris, findDecisionForMandataris, - findStartDateOfMandataris, updatePublicationStatusOfMandataris, } from '../data-access/mandataris'; import { TERM_MANDATARIS_TYPE, - TERM_STAGING_GRAPH, checkIfMinimalMandatarisInfoAvailable, - getMandateOfMandataris as findMandateOfMandataris, - findNameOfPersoonFromStaging, - findOverlappingMandataris, - findPersoonForMandatarisInGraph, getMandatarisTriplesInStagingGraph, - getQuadsInLmbFromTriples, - getTriplesOfSubject, - insertTriplesInGraph, - isMandatarisInTarget as isMandatarisInLmbDatabase, isSubjectOfType, - updateDifferencesOfMandataris, } from '../data-access/mandatees-decisions'; -import { - checkPersonExistsAllGraphs, - copyPerson, - createrPersonFromUri, -} from '../data-access/persoon'; import { MandatarisBesluitLookup, MandatarisFullInfo } from '../types'; import { PUBLICATION_STATUS } from '../util/constants'; import { copyBeleidsdomeinInfo } from './mandataris-besluit/beleidsdomein'; @@ -106,137 +87,3 @@ export async function handleMandatarisSubject( await copyFractionInfo(mandatarisFullInfo); await copyBeleidsdomeinInfo(mandatarisFullInfo); } - -export async function oldHandleMandatarisSubject(mandatarisSubject: string) { - console.log(`|> Mandataris uri: ${mandatarisSubject}`); - const isExitingInLmbDatabase = - await isMandatarisInLmbDatabase(mandatarisSubject); - console.log( - `|> Mandataris exists in LMB database? ${isExitingInLmbDatabase}`, - ); - - const mandaat = await findMandateOfMandataris(mandatarisSubject); - console.log('|> Mandaat for mandataris', mandaat?.value); - if (!mandaat) { - console.log( - `|> No mandaat found for mandataris with subject: ${mandatarisSubject.value} \n|>\n`, - ); - mandatarisQueue.addToManualQueue(mandatarisSubject); - return; - } - - const mandatarisGraph = await findBestuurseenheidForMandaat(mandaat); - console.log(`|> mandataris graph: ${mandatarisGraph?.value ?? undefined}.`); - - if (!mandatarisGraph) { - console.log( - `|> Could not find graph from mandaat: ${mandaat.value}. Continueing to the next subject.\n|>\n`, - ); - mandatarisQueue.addToManualQueue(mandatarisSubject); - return; - } - - const incomingTriples = await getTriplesOfSubject( - mandatarisSubject, - TERM_STAGING_GRAPH, - ); - console.log( - `|> Found ${incomingTriples.length} triples in the staging graph for mandataris.`, - ); - if (isExitingInLmbDatabase) { - const currentQuads = await getQuadsInLmbFromTriples(incomingTriples); - console.log('|> Updating mandataris predicate values.'); - await updateDifferencesOfMandataris( - currentQuads, - incomingTriples, - mandatarisGraph, - ); - - console.log( - '|> Going to the next mandataris subeject as triples are updated. \n|>\n', - ); - return; - } - - // Looking for persoon in graph of the mandataris - const persoonOfMandataris = await findPersoonForMandatarisInGraph( - mandatarisSubject, - TERM_STAGING_GRAPH, - ); - - if (!persoonOfMandataris) { - console.log( - `|> Could not find person of mandataris: ${mandatarisSubject.value}. Continuing to the next subject.\n|>\n`, - ); - mandatarisQueue.addToManualQueue(mandatarisSubject); - return; - } - - console.log( - `|> Persoon from mandataris: ${persoonOfMandataris?.value ?? undefined}.`, - ); - - const persoonInLMBGraph = await findPersoonForMandatarisInGraph( - mandatarisSubject, - mandatarisGraph, - ); - console.log( - `|> Is persoon in graph of mandataris (LMB)? ${ - persoonInLMBGraph ? true : false - }`, - ); - if (persoonInLMBGraph) { - const overlappingMandataris = await findOverlappingMandataris( - persoonOfMandataris, - mandaat, - ); - console.log( - `|> Persoon has overlapping mandaat? ${ - overlappingMandataris?.value ?? false - }`, - ); - - if (overlappingMandataris) { - const startDate = await findStartDateOfMandataris(mandatarisSubject); - console.log(`|> Found start date for incoming mandataris? ${startDate}`); - if (startDate) { - await endExistingMandataris( - mandatarisGraph, - overlappingMandataris, - startDate, - ); - } - } - - console.log('|> Inserting incoming triples'); - await insertTriplesInGraph(incomingTriples, mandatarisGraph); - - return; - } - - // If person exists in another graph, copy that person. - const personInOtherGraph = - await checkPersonExistsAllGraphs(persoonOfMandataris); - console.log( - `|> Is persoon in other graphs of the LMB application?: ${personInOtherGraph}`, - ); - if (personInOtherGraph) { - await copyPerson(persoonOfMandataris, mandatarisGraph); - return; - } - - // Create new person with given firstname and lastname - const persoon = await findNameOfPersoonFromStaging(mandatarisSubject); - console.log('|> Looking for persoon names', persoon); - if (!persoon) { - mandatarisQueue.addToManualQueue(mandatarisSubject); - return; - } - - await createrPersonFromUri( - persoon.persoonUri, - persoon.firstname, - persoon.lastname, - mandatarisGraph, - ); -} diff --git a/data-access/bestuurseenheid.ts b/data-access/bestuurseenheid.ts deleted file mode 100644 index d81880f..0000000 --- a/data-access/bestuurseenheid.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { TERM_TYPE, sparqlEscapeTermValue } from '../util/sparql-escape'; -import { findFirstSparqlResult } from '../util/sparql-result'; -import { Term } from '../types'; -import { querySudo } from '@lblod/mu-auth-sudo'; -import { sparqlEscapeUri } from '../util/mu'; - -export const bestuurseenheid_sudo = { - getContactEmailFromMandataris, -}; - -export async function findBestuurseenheidForMandaat( - mandaat: Term, -): Promise { - const queryForId = ` - PREFIX org: - PREFIX besluit: - PREFIX mandaat: - PREFIX person: - PREFIX skos: - PREFIX mu: - - SELECT ?id WHERE { - ?bestuurseenheid a besluit:Bestuurseenheid. - ?bestuurseenheid mu:uuid ?id. - ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . - ?bestuursorgaanInTijd mandaat:isTijdspecialisatieVan ?bestuursorgaan . - ?bestuursorgaanInTijd org:hasPost ${sparqlEscapeTermValue(mandaat)} . - } - `; - - const idResult = await querySudo(queryForId); - const result = findFirstSparqlResult(idResult); - - if (!result) { - return null; - } - - return { - type: TERM_TYPE.URI, - value: - 'http://mu.semte.ch/graph/organizations/' + - result.id.value + - '/LoketLB-mandaatGebruiker', - } as Term; -} - -async function getContactEmailFromMandataris(mandatarisUri: string) { - const query = ` - PREFIX org: - PREFIX besluit: - PREFIX mandaat: - PREFIX person: - PREFIX skos: - PREFIX mu: - PREFIX ext: - PREFIX schema: - - SELECT ?email WHERE { - ${sparqlEscapeUri(mandatarisUri)} a mandaat:Mandataris; - org:holds ?mandaat. - - ?bestuurseenheid a besluit:Bestuurseenheid. - ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . - ?bestuursorgaanInTijd mandaat:isTijdspecialisatieVan ?bestuursorgaan . - ?bestuursorgaanInTijd org:hasPost ?mandaat . - - OPTIONAL { - ?contact a ext:BestuurseenheidContact ; - ext:contactVoor ?bestuurseenheid ; - schema:email ?email . - } - } LIMIT 1 - `; - const sparqlResult = await querySudo(query); - const result = findFirstSparqlResult(sparqlResult); - - return result?.email.value; -} diff --git a/data-access/mandataris.ts b/data-access/mandataris.ts index e4b7c7c..9cf6d42 100644 --- a/data-access/mandataris.ts +++ b/data-access/mandataris.ts @@ -520,28 +520,6 @@ export async function endExistingMandataris( } } -export async function findStartDateOfMandataris( - mandataris: Term, -): Promise { - const startDateQuery = ` - PREFIX mandaat: - - SELECT ?startDate - WHERE { - ${sparqlEscapeTermValue(mandataris)} mandaat:start ?startDate . - } - `; - - const dateResult = await querySudo(startDateQuery); - const result = findFirstSparqlResult(dateResult); - - if (result) { - return new Date(result.startDate.value); - } - - return null; -} - export async function findDecisionForMandataris( mandataris: string, ): Promise<{ besluit: string; link: string } | null> { diff --git a/data-access/mandatees-decisions.ts b/data-access/mandatees-decisions.ts index 44edd76..e04e487 100644 --- a/data-access/mandatees-decisions.ts +++ b/data-access/mandatees-decisions.ts @@ -1,24 +1,15 @@ import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; import { sparqlEscapeUri } from 'mu'; +import { MandatarisBesluitLookup, Term, Triple } from '../types'; import { - MandatarisBesluitLookup, - Quad, - Term, - TermProperty, - Triple, -} from '../types'; + createNotification, + getMandatarisNotificationGraph, +} from '../util/create-notification'; +import { TERM_TYPE, sparqlEscapeTermValue } from '../util/sparql-escape'; import { - findFirstSparqlResult, getBooleanSparqlResult, getSparqlResults, } from '../util/sparql-result'; -import { TERM_TYPE, sparqlEscapeTermValue } from '../util/sparql-escape'; -import { MANDATARIS_STATUS, PUBLICATION_STATUS } from '../util/constants'; -import { - createNotification, - getMandatarisNotificationGraph, -} from '../util/create-notification'; -import { getUuidForUri } from '../util/uuid-for-uri'; export const BESLUIT_STAGING_GRAPH = process.env.BESLUIT_STAGING_GRAPH || @@ -48,232 +39,6 @@ export async function isSubjectOfType( return getBooleanSparqlResult(isOfSubject); } -export async function getTriplesOfSubject( - subject: Term, - graph: Term, -): Promise> { - const queryForsubject = ` - SELECT ?predicate ?object ?graph - WHERE { - GRAPH ${sparqlEscapeTermValue(graph)} { - ${sparqlEscapeTermValue(subject)} ?predicate ?object . - } - } - `; - - const results = await querySudo(queryForsubject); - - return getSparqlResults(results).map((po) => { - return { - subject: subject, - predicate: po.predicate, - object: po.object, - } as Triple; - }); -} - -export async function getQuadsInLmbFromTriples( - triples: Array, -): Promise> { - const useAsValues = triples.map((triple: Triple) => { - return `(${sparqlEscapeTermValue(triple.subject)} ${sparqlEscapeTermValue( - triple.predicate, - )}) \n`; - }); - const query = ` - SELECT ?subject ?predicate ?object ?graph - WHERE { - GRAPH ?graph { - VALUES (?subject ?predicate) { - ${useAsValues.join('')} - } - OPTIONAL { - ?subject ?predicate ?object . - } - MINUS { - GRAPH ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)} { - ?subject ?predicate ?object . - } - } - } - } - `; - const resultsInTarget = await querySudo(query); - - return getSparqlResults(resultsInTarget) as Array; -} - -export async function isMandatarisInTarget(subject: Term) { - const mandatarisType = sparqlEscapeTermValue(TERM_MANDATARIS_TYPE); - const askIfMandataris = ` - ASK { - ${sparqlEscapeTermValue(subject)} a ${mandatarisType} . - MINUS { - GRAPH ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)} { - ${sparqlEscapeTermValue(subject)} a ${mandatarisType} . - } - } - } - `; - const result = await querySudo(askIfMandataris); - - return getBooleanSparqlResult(result); -} - -export async function findPersoonForMandatarisInGraph( - subject: Term, - graph: Term, -): Promise { - const escaped = { - graph: sparqlEscapeTermValue(graph), - mandataris: sparqlEscapeTermValue(subject), - }; - - const queryForPersoon = ` - PREFIX mandaat: - - SELECT ?persoon - WHERE { - GRAPH ${escaped.graph}{ - ${escaped.mandataris} mandaat:isBestuurlijkeAliasVan ?persoon . - } - } - `; - const persoonForMandataris = await querySudo(queryForPersoon); - const result = findFirstSparqlResult(persoonForMandataris); - - return result?.persoon ?? null; -} - -export async function updateDifferencesOfMandataris( - currentQuads: Array, - incomingTriples: Array, - insertGraph: Term, -): Promise { - for (const incomingTriple of incomingTriples) { - const currentQuad = currentQuads.find( - (quad: Quad) => - quad.subject.value == incomingTriple.subject.value && - quad.predicate.value == incomingTriple.predicate.value, - ); - if (currentQuad) { - if (incomingTriple.object.value !== currentQuad.object.value) { - console.log( - `|> Value for predicate (${incomingTriple.predicate.value}) differ. Current: ${currentQuad.object.value} incoming: ${incomingTriple.object.value}. Updating value.`, - ); - const escaped = { - subject: sparqlEscapeTermValue(currentQuad.subject), - predicate: sparqlEscapeTermValue(currentQuad.predicate), - graph: sparqlEscapeTermValue(insertGraph), - currentObject: - currentQuad.object.value ?? - sparqlEscapeTermValue(currentQuad.object), - incomingObject: - incomingTriple.object.value ?? - sparqlEscapeTermValue(incomingTriple.object), - }; - const subjectPredicate = `${escaped.subject} ${escaped.predicate}`; - const updateObjectValueQuery = ` - DELETE { - GRAPH ${escaped.graph} { - ${subjectPredicate} ${escaped.currentObject} - } - } INSERT { - GRAPH ${escaped.graph} { - ${subjectPredicate} ${escaped.incomingObject} . - } - } WHERE { - GRAPH ${escaped.graph} { - ${escaped.subject} ${escaped.predicate} ${escaped.currentObject} . - } - MINUS { - GRAPH ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)} { - ${subjectPredicate} ${escaped.currentObject} - } - } - } - `; - - try { - await updateSudo(updateObjectValueQuery, {}, { mayRetry: true }); - console.log( - `|> Updated value for predicate (${incomingTriple.predicate.value}) to ${incomingTriple.object.value}.`, - ); - } catch (error) { - throw Error( - `Could not update mandataris predicate value: ${subjectPredicate}`, - ); - } - } - } else { - const escaped = { - subject: sparqlEscapeTermValue(incomingTriple.subject), - predicate: sparqlEscapeTermValue(incomingTriple.predicate), - incomingObject: sparqlEscapeTermValue(incomingTriple.object), - graph: sparqlEscapeTermValue(insertGraph), - }; - const insertIncomingQuery = ` - INSERT DATA { - GRAPH ${escaped.graph} { - ${escaped.subject} ${escaped.predicate} ${escaped.incomingObject} . - } - } - `; - try { - await updateSudo(insertIncomingQuery, {}, { mayRetry: true }); - console.log(`|> Inserted triple: ${JSON.stringify(incomingTriple)}`); - } catch (error) { - throw Error( - `Could not insert incoming triple: ${JSON.stringify(incomingTriple)}`, - ); - } - } - } -} - -export async function getMandateOfMandataris( - mandataris: Term, -): Promise { - const queryForMandatarisMandate = ` - PREFIX org: - - SELECT ?mandaat - WHERE { - ${sparqlEscapeTermValue(mandataris)} org:holds ?mandaat . - } - `; - const mandateResult = await querySudo(queryForMandatarisMandate); - const mandaatQuad = findFirstSparqlResult(mandateResult); - - return mandaatQuad?.mandaat ?? null; -} - -export async function findOverlappingMandataris( - persoon: Term, - mandaat: Term, -): Promise { - const queryMandataris = ` - PREFIX mandaat: - PREFIX org: - - SELECT DISTINCT ?mandataris - WHERE { - VALUES ?status { - ${sparqlEscapeUri(MANDATARIS_STATUS.EFFECTIEF)} - ${sparqlEscapeUri(PUBLICATION_STATUS.DRAFT)} - } - ?mandataris a ${sparqlEscapeTermValue(TERM_MANDATARIS_TYPE)} ; - mandaat:status ?status; - mandaat:isBestuurlijkeAliasVan ${sparqlEscapeTermValue(persoon)} ; - org:holds ${sparqlEscapeTermValue(mandaat)} . - } - `; - - const mandatarisResult = await querySudo(queryMandataris); - - return findFirstSparqlResult(mandatarisResult)?.mandataris ?? null; -} - export async function insertTriplesInGraph( triples: Array, graph: string, @@ -310,30 +75,6 @@ export async function insertTriplesInGraph( } } -export async function findNameOfPersoonFromStaging( - mandataris: Term, -): Promise { - const mandatarisUri = sparqlEscapeTermValue(mandataris); - const queryMandatarisPerson = ` - PREFIX foaf: - PREFIX persoon: - - SELECT ?persoonUri ?firstname ?lastname - WHERE { - GRAPH ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)} { - ${mandatarisUri} a ${sparqlEscapeTermValue(TERM_MANDATARIS_TYPE)} . - - ${mandatarisUri} persoon:isBestuurlijkeAliasVan ?persoonUri . - ${mandatarisUri} persoon:gebruikteVoornaam ?firstname . - ${mandatarisUri} foaf:familyName ?lastname . - } - } - `; - const result = await querySudo(queryMandatarisPerson); - - return findFirstSparqlResult(result); -} - export async function checkIfMinimalMandatarisInfoAvailable( mandatarisBesluitInfo: MandatarisBesluitLookup, ) { diff --git a/data-access/persoon.ts b/data-access/persoon.ts index 4fed5f8..6e39dba 100644 --- a/data-access/persoon.ts +++ b/data-access/persoon.ts @@ -1,20 +1,18 @@ +import { querySudo } from '@lblod/mu-auth-sudo'; import { query, - update, + sparqlEscapeDateTime, sparqlEscapeString, sparqlEscapeUri, - sparqlEscapeDateTime, + update, } from 'mu'; -import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; import { v4 as uuidv4 } from 'uuid'; import { Term, TermProperty } from '../types'; import { sparqlEscapeTermValue } from '../util/sparql-escape'; -import { TERM_STAGING_GRAPH } from './mandatees-decisions'; import { findFirstSparqlResult, getBooleanSparqlResult, } from '../util/sparql-result'; -import { getIdentifierFromPersonUri } from '../util/uuid-for-uri'; // note since we use the regular query, not sudo queries, be sure to log in when using this endpoint. E.g. use the vendor login @@ -110,132 +108,6 @@ export const personExistsInGraph = async ( return getBooleanSparqlResult(result); }; -// All graphs except the staging graph -export async function checkPersonExistsAllGraphs( - subject: Term, -): Promise { - const escaped = { - person: sparqlEscapeTermValue(subject), - }; - - const askIfPersoonExists = ` - PREFIX mandaat: - - ASK { - GRAPH ?g { - ${escaped.person} ?p ?o. - } - FILTER (?g != ${sparqlEscapeTermValue(TERM_STAGING_GRAPH)}). - } - `; - const result = await querySudo(askIfPersoonExists); - - return getBooleanSparqlResult(result); -} - -export async function createrPersonFromUri( - personUri: Term, - firstname: Term, - lastname: Term, - graph: Term, -): Promise { - // TODO danger! - const personIdentifier = getIdentifierFromPersonUri(personUri.value); - const createQuery = ` - PREFIX mu: - PREFIX person: - PREFIX persoon: - PREFIX foaf: - - INSERT DATA { - GRAPH ${sparqlEscapeTermValue(graph)} { - ${sparqlEscapeTermValue(personUri)} a person:Person; - mu:uuid ${sparqlEscapeString(personIdentifier)}; - persoon:gebruikteVoornaam ${sparqlEscapeTermValue(firstname)}; - foaf:familyName ${sparqlEscapeTermValue(lastname)}. - } - } - `; - const baseLogText = `person with uri ${personUri.value} for ${firstname.value} ${lastname.value}`; - - try { - await updateSudo(createQuery); - console.log('|> Created ' + baseLogText); - } catch (error) { - console.log('|> Could not create ' + baseLogText); - } -} - -export async function copyPerson(subject: Term, graph: Term) { - const escaped = { - graph: sparqlEscapeTermValue(graph), - person: sparqlEscapeTermValue(subject), - }; - - const q = ` - PREFIX mu: - PREFIX person: - PREFIX persoon: - PREFIX foaf: - PREFIX adms: - PREFIX skos: - - INSERT { - GRAPH ${escaped.graph} { - ${escaped.person} a person:Person; - mu:uuid ?uuid; - persoon:gebruikteVoornaam ?voornaam; - adms:identifier ?identifier; - foaf:familyName ?achternaam; - persoon:geslacht ?geslacht; - foaf:name ?altName; - persoon:heeftGeboorte ?geboorte. - - ?identifier a adms:Identifier; - mu:uuid ?idUuid; - skos:notation ?rrn. - - ?geboorte a persoon:Geboorte; - mu:uuid ?geboorteUuid; - persoon:datum ?geboorteDatum. - } - } - WHERE { - GRAPH ?g { - ${escaped.person} a person:Person; - mu:uuid ?uuid; - persoon:gebruikteVoornaam ?voornaam; - adms:identifier ?identifier; - foaf:familyName ?achternaam. - - ?identifier a adms:Identifier; - mu:uuid ?idUuid; - skos:notation ?rrn. - OPTIONAL { - ${escaped.person} persoon:geslacht ?geslacht. - } - OPTIONAL { - ${escaped.person} foaf:name ?altName. - } - OPTIONAL { - ${escaped.person} persoon:heeftGeboorte ?geboorte. - ?geboorte a persoon:Geboorte; - mu:uuid ?geboorteUuid; - persoon:datum ?geboorteDatum. - } - } - }`; - - try { - await updateSudo(q); - console.log( - `|> Copied person with uri ${escaped.person} to graph ${escaped.graph}.`, - ); - } catch (error) { - throw Error(`Could not copy person with uri: ${escaped.person}`); - } -} - async function getFractie( id: string, bestuursperiodeId: string, diff --git a/routes/delta-decisions.ts b/routes/delta-decisions.ts index 977c619..4ec68ae 100644 --- a/routes/delta-decisions.ts +++ b/routes/delta-decisions.ts @@ -4,13 +4,11 @@ import { sparqlEscapeUri, sparqlEscapeDateTime } from 'mu'; import { Request, Response } from 'express'; import { Changeset, Quad } from '../types'; -import { ProcessingQueue } from '../services/processing-queue'; import { updateSudo } from '@lblod/mu-auth-sudo'; import { v4 as uuid } from 'uuid'; import { BESLUIT_STAGING_GRAPH } from '../data-access/mandatees-decisions'; const deltaRouter = Router(); -export const mandatarisQueue = new ProcessingQueue(); deltaRouter.post('/decisions', async (req: Request, res: Response) => { console.log('|>Triggered the decisions endpoint!'); diff --git a/routes/mock.ts b/routes/mock.ts index 4658837..b7042eb 100644 --- a/routes/mock.ts +++ b/routes/mock.ts @@ -115,7 +115,8 @@ const createDecisionForKnownMandatarisNewBeleidsdomein = async () => { const beleidsdomeinName = 'foobar'; const beleidsdomeinUri = ``; - const existingBeleidsdomeinUri = ``; + const existingBeleidsdomeinUri = + ''; const insertQuery = ` PREFIX mandaat: diff --git a/services/processing-queue.ts b/services/processing-queue.ts deleted file mode 100644 index 5e7ca1e..0000000 --- a/services/processing-queue.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { Term } from '../types'; - -/* eslint-disable @typescript-eslint/no-explicit-any */ -export class ProcessingQueue { - toExecute: (args: any) => Promise | null; - queue: Array; - manualQueue: Array; - isExecuting: boolean; - - constructor() { - this.queue = []; - this.manualQueue = []; - this.run(); - this.isExecuting = false; - } - - async run() { - if (this.queue.length >= 1 && !this.isExecuting) { - try { - this.isExecuting = true; - const subject = this.queue?.shift(); - if (subject) { - console.log(`|> TASK start for ${subject.value}`); - await this.toExecute(subject); - } - console.log( - `|> Remaining number of tasks in queue:${this.queue.length} in manual queue:${this.manualQueue.length} \n`, - ); - } catch (error) { - console.error(`|> Error while processing delta in queue ${error}`); - } finally { - this.isExecuting = false; - console.log('|> TASK done \n|>\n'); - this.run(); - } - } - } - - addToQueue(subjects: Array) { - if (!this.toExecute) { - throw Error('|> No method is set to execute the queue items on.'); - } - - const subjectsInQueue = this.queue.map((subject: Term) => subject.value); - const nonDuplicates = subjects.filter( - (term: Term) => !subjectsInQueue.includes(term.value), - ); - - console.log( - `|> [${new Date().toISOString()}] Added ${ - nonDuplicates.length - } to queue.`, - ); - - this.queue.push(...nonDuplicates); - console.log(`|> Currently ${this.queue.length} items in queue.`); - - if (this.queue.length >= 1 && !this.isExecuting) { - console.log('|> Queue was not empty triggering run()'); - this.run(); - } - } - - addToManualQueue(subject: Term) { - this.manualQueue.push(subject); - console.log(`|> Added to manual queue: ${JSON.stringify(subject)}`); - } - - moveManualQueueToQueue() { - console.log( - `|> Moving ${this.manualQueue.length} items from manual queue to the acutal executing queue.`, - ); - this.addToQueue(this.manualQueue); - this.manualQueue = []; - } - - setMethodToExecute(method: (args: any) => Promise) { - this.toExecute = method; - } -} From 54540a6cab39e499e44789d77914853d54e39a84 Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 13 Sep 2024 16:35:00 +0200 Subject: [PATCH 3/7] linting --- data-access/mandatees-decisions.ts | 20 ++++++++++---------- routes/delta-decisions.ts | 24 +++++++++++++----------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/data-access/mandatees-decisions.ts b/data-access/mandatees-decisions.ts index e04e487..b6890ed 100644 --- a/data-access/mandatees-decisions.ts +++ b/data-access/mandatees-decisions.ts @@ -140,13 +140,12 @@ export async function checkIfMinimalMandatarisInfoAvailable( } export async function checkIfMandatarisExists(mandatarisUri) { + const safeMandatarisUri = sparqlEscapeUri(mandatarisUri); const query = ` PREFIX ext: ASK { GRAPH ?g { - ${sparqlEscapeUri( - mandatarisUri, - )} a . + ${safeMandatarisUri} a . } ?g ext:ownedBy ?bestuurseenheid. } @@ -263,6 +262,13 @@ export async function replacePropertiesOnInstance( // That way we don't accidentally delete information that we want to keep if they are // not specified in the staging graph like beleidsdomeinen const predicates = subjectTriples.map((triple) => triple.predicate); + const newData = subjectTriples + .map((triple) => { + return `${sparqlEscapeUri(subject)} ${sparqlEscapeUri( + triple.predicate.value, + )} ${sparqlEscapeTermValue(triple.object)} .`; + }) + .join('\n'); const query = ` DELETE { GRAPH ${sparqlEscapeUri(graph)} { @@ -270,13 +276,7 @@ export async function replacePropertiesOnInstance( } } INSERT { GRAPH ${sparqlEscapeUri(graph)} { - ${subjectTriples - .map((triple) => { - return `${sparqlEscapeUri(subject)} ${sparqlEscapeUri( - triple.predicate.value, - )} ${sparqlEscapeTermValue(triple.object)} .`; - }) - .join('\n')} + ${newData} } } WHERE { GRAPH ${sparqlEscapeUri(graph)} { diff --git a/routes/delta-decisions.ts b/routes/delta-decisions.ts index 4ec68ae..7d1d898 100644 --- a/routes/delta-decisions.ts +++ b/routes/delta-decisions.ts @@ -28,6 +28,18 @@ deltaRouter.post('/decisions', async (req: Request, res: Response) => { new Set(insertTriples.map((quad: Quad) => quad.object.value)), ); + const newData = mandatarisSubjects + .map((mandataris: string) => { + const id = uuid(); + const instanceUri = sparqlEscapeUri( + `http://mu.semte.ch/vocabularies/ext/queueInstance/${id}`, + ); + const mandatarisUri = sparqlEscapeUri(mandataris); + return `${instanceUri} ext:queueInstance ${mandatarisUri} ; + ext:queueTime ${sparqlEscapeDateTime(new Date())} .`; + }) + .join('\n'); + // keep these mandataris instances in the database instead of memory // so if we crash we know they should still be processed // wait BUFFER_TIME to process the mandataris so we are reasonably sure that we have all the info @@ -36,17 +48,7 @@ deltaRouter.post('/decisions', async (req: Request, res: Response) => { INSERT DATA { GRAPH { - ${mandatarisSubjects - .map((mandataris: string) => { - const id = uuid(); - const instanceUri = sparqlEscapeUri( - `http://mu.semte.ch/vocabularies/ext/queueInstance/${id}`, - ); - const mandatarisUri = sparqlEscapeUri(mandataris); - return `${instanceUri} ext:queueInstance ${mandatarisUri} ; - ext:queueTime ${sparqlEscapeDateTime(new Date())} .`; - }) - .join('\n')} + ${newData} } }`; From 2a180cd300a369f4a140003d92f8e17492b4bed5 Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 4 Oct 2024 09:00:42 +0200 Subject: [PATCH 4/7] less techy notifications --- controllers/mandataris-besluit/beleidsdomein.ts | 2 +- controllers/mandataris-besluit/fractie.ts | 4 ++-- controllers/mandataris-besluit/mandataris.ts | 4 ++-- controllers/mandataris-besluit/persoon.ts | 2 +- cron/handle-decision-queue.ts | 4 ++-- data-access/mandatees-decisions.ts | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/controllers/mandataris-besluit/beleidsdomein.ts b/controllers/mandataris-besluit/beleidsdomein.ts index 57f8ebd..0714edd 100644 --- a/controllers/mandataris-besluit/beleidsdomein.ts +++ b/controllers/mandataris-besluit/beleidsdomein.ts @@ -42,7 +42,7 @@ async function copyBeleidsDomein( await createBeleidsDomein(beleidsDomein, graph); await createMandatarisBesluitNotification({ title: 'Beleidsdomein aangemaakt', - description: `Een nieuw beleidsdomein met uri ${beleidsDomein} werd aangemaakt op basis van de informatie in het Besluit.`, + description: `Een nieuw beleidsdomein werd aangemaakt op basis van de informatie in het Besluit.`, type: 'info', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris-besluit/fractie.ts b/controllers/mandataris-besluit/fractie.ts index a5da568..079443f 100644 --- a/controllers/mandataris-besluit/fractie.ts +++ b/controllers/mandataris-besluit/fractie.ts @@ -56,7 +56,7 @@ const checkForFractionsThatDontExist = async ( if (hasUnknownFractions) { await createMandatarisBesluitNotification({ title: 'Onbekende fractie', - description: `Mandataris met uri ${mandatarisFullInfo.mandatarisUri} heeft een fractie die niet gekend is in de applicatie. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, + description: `Mandataris uit een besluit heeft een fractie die niet gekend is in de applicatie. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, type: 'error', info: mandatarisFullInfo, }); @@ -118,7 +118,7 @@ const checkForDuplicateMemberships = async ( if (hasUnknownFractions) { await createMandatarisBesluitNotification({ title: 'Dubbele fractie', - description: `Mandataris met uri ${mandatarisFullInfo.mandatarisUri} heeft meerdere fracties op hetzelfde moment in het besluit. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, + description: `Een mandataris heeft meerdere fracties op hetzelfde moment in het besluit. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, type: 'error', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris-besluit/mandataris.ts b/controllers/mandataris-besluit/mandataris.ts index 0495f53..d6ab626 100644 --- a/controllers/mandataris-besluit/mandataris.ts +++ b/controllers/mandataris-besluit/mandataris.ts @@ -39,7 +39,7 @@ async function copyMandatarisToExisting( ); await createMandatarisBesluitNotification({ title: 'Mandataris aangepast', - description: `Mandataris met uri ${mandatarisSubject} werd aangepast op basis van de informatie in een Besluit.`, + description: `Mandataris werd aangepast op basis van de informatie in een Besluit.`, type: 'info', info: mandatarisFullInfo, }); @@ -99,7 +99,7 @@ async function createNewMandataris(mandatarisFullInfo: MandatarisFullInfo) { await insertTriplesInGraph(allTriples, graph); await createMandatarisBesluitNotification({ title: 'Mandataris aangemaakt', - description: `Een nieuwe Mandataris met uri ${mandatarisSubject} werd aangemaakt op basis van de informatie in een Besluit.${statusWarning}`, + description: `Een nieuwe Mandataris werd aangemaakt op basis van de informatie in een Besluit.${statusWarning}`, type: 'warning', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris-besluit/persoon.ts b/controllers/mandataris-besluit/persoon.ts index 04481a5..934b85e 100644 --- a/controllers/mandataris-besluit/persoon.ts +++ b/controllers/mandataris-besluit/persoon.ts @@ -25,7 +25,7 @@ export async function copyPersonInfo(mandatarisFullInfo: MandatarisFullInfo) { await createPerson(persoonUri, graph); await createMandatarisBesluitNotification({ title: 'Persoon aangemaakt', - description: `Een nieuwe Persoon met uri ${persoonUri} werd aangemaakt op basis van de informatie in het Besluit. Deze Persoon zal onvolledige informatie bevatten aangezien e.g. rijksregisternummer niet gepubliceerd wordt in het Besluit.`, + description: `Een nieuwe Persoon werd aangemaakt op basis van de informatie in het Besluit. Deze Persoon zal onvolledige informatie bevatten aangezien e.g. rijksregisternummer niet gepubliceerd wordt in het Besluit.`, type: 'warning', info: mandatarisFullInfo, }); diff --git a/cron/handle-decision-queue.ts b/cron/handle-decision-queue.ts index e509d3a..e075d19 100644 --- a/cron/handle-decision-queue.ts +++ b/cron/handle-decision-queue.ts @@ -43,8 +43,8 @@ async function safeProcessMandatarisForDecisions(match) { `ERROR processing mandataris decision instance ${match.instance}: ${e.message}`, ); await createNotification({ - title: 'Error tijdens verwerken van Besluit voor mandataris', - description: `Error tijdens verwerken van Besluit voor ${match.instance}. Gelieve de logs na te kijken voor meer informatie.`, + title: 'Fout tijdens verwerken van Besluit voor mandataris', + description: `Fout tijdens verwerken van Besluit voor ${match.instance}. Gelieve de logs na te kijken voor meer informatie.`, type: 'error', graph: match.instance, links: [ diff --git a/data-access/mandatees-decisions.ts b/data-access/mandatees-decisions.ts index b6890ed..2cb0398 100644 --- a/data-access/mandatees-decisions.ts +++ b/data-access/mandatees-decisions.ts @@ -121,7 +121,7 @@ export async function checkIfMinimalMandatarisInfoAvailable( const graph = await getMandatarisNotificationGraph(mandataris); await createNotification({ title: 'Besluit met Mandataris zonder minimale info', - description: `Mandataris ${mandataris} uit Besluit ${besluitUri} heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een start datum, een persson en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.`, + description: `Een Mandataris uit het Besluit heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een startdatum, een persson en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.`, type: 'error', graph, links: [ From c973638b054a670ff6bc843624759d4055903789 Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 4 Oct 2024 10:29:37 +0200 Subject: [PATCH 5/7] forgot to add a uuid to the link --- util/create-notification.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/util/create-notification.ts b/util/create-notification.ts index 702eaa4..62d443e 100644 --- a/util/create-notification.ts +++ b/util/create-notification.ts @@ -40,6 +40,7 @@ export async function createNotification({ ); return `${uri} ext:notificationLink ${linkUri} . ${linkUri} a ext:SystemNotificationLink ; + mu:uuid ${sparqlEscapeString(linkId)} ; ext:linkedType ${sparqlEscapeString(link.type)} ; ext:linkedTo ${sparqlEscapeUri(link.uri)} .`; }) From 6f1fe421335987e85f05fcd9fe96d3947ec48184 Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 4 Oct 2024 11:02:39 +0200 Subject: [PATCH 6/7] typo --- data-access/mandatees-decisions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-access/mandatees-decisions.ts b/data-access/mandatees-decisions.ts index 2cb0398..97bb114 100644 --- a/data-access/mandatees-decisions.ts +++ b/data-access/mandatees-decisions.ts @@ -121,7 +121,7 @@ export async function checkIfMinimalMandatarisInfoAvailable( const graph = await getMandatarisNotificationGraph(mandataris); await createNotification({ title: 'Besluit met Mandataris zonder minimale info', - description: `Een Mandataris uit het Besluit heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een startdatum, een persson en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.`, + description: `Een Mandataris uit het Besluit heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een startdatum, een persoon en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.`, type: 'error', graph, links: [ From f93e979c1b8575b2ad8faa864e2da0adf9559d4a Mon Sep 17 00:00:00 2001 From: karel kremer Date: Fri, 25 Oct 2024 15:23:59 +0200 Subject: [PATCH 7/7] fix linting --- controllers/mandataris-besluit/beleidsdomein.ts | 3 ++- controllers/mandataris-besluit/fractie.ts | 6 ++++-- controllers/mandataris-besluit/mandataris.ts | 3 ++- controllers/mandataris-besluit/persoon.ts | 3 ++- controllers/mandataris.ts | 13 +++++-------- data-access/mandatees-decisions.ts | 3 ++- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/controllers/mandataris-besluit/beleidsdomein.ts b/controllers/mandataris-besluit/beleidsdomein.ts index 0714edd..f81ab1f 100644 --- a/controllers/mandataris-besluit/beleidsdomein.ts +++ b/controllers/mandataris-besluit/beleidsdomein.ts @@ -42,7 +42,8 @@ async function copyBeleidsDomein( await createBeleidsDomein(beleidsDomein, graph); await createMandatarisBesluitNotification({ title: 'Beleidsdomein aangemaakt', - description: `Een nieuw beleidsdomein werd aangemaakt op basis van de informatie in het Besluit.`, + description: + 'Een nieuw beleidsdomein werd aangemaakt op basis van de informatie in het Besluit.', type: 'info', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris-besluit/fractie.ts b/controllers/mandataris-besluit/fractie.ts index 079443f..55c47f5 100644 --- a/controllers/mandataris-besluit/fractie.ts +++ b/controllers/mandataris-besluit/fractie.ts @@ -56,7 +56,8 @@ const checkForFractionsThatDontExist = async ( if (hasUnknownFractions) { await createMandatarisBesluitNotification({ title: 'Onbekende fractie', - description: `Mandataris uit een besluit heeft een fractie die niet gekend is in de applicatie. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, + description: + 'Mandataris uit een besluit heeft een fractie die niet gekend is in de applicatie. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.', type: 'error', info: mandatarisFullInfo, }); @@ -118,7 +119,8 @@ const checkForDuplicateMemberships = async ( if (hasUnknownFractions) { await createMandatarisBesluitNotification({ title: 'Dubbele fractie', - description: `Een mandataris heeft meerdere fracties op hetzelfde moment in het besluit. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.`, + description: + 'Een mandataris heeft meerdere fracties op hetzelfde moment in het besluit. Deze informatie is niet overgezet. Gelieve de Mandataris manueel na te kijken en eventueel aan te passen.', type: 'error', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris-besluit/mandataris.ts b/controllers/mandataris-besluit/mandataris.ts index d6ab626..bb0b99a 100644 --- a/controllers/mandataris-besluit/mandataris.ts +++ b/controllers/mandataris-besluit/mandataris.ts @@ -39,7 +39,8 @@ async function copyMandatarisToExisting( ); await createMandatarisBesluitNotification({ title: 'Mandataris aangepast', - description: `Mandataris werd aangepast op basis van de informatie in een Besluit.`, + description: + 'Mandataris werd aangepast op basis van de informatie in een Besluit.', type: 'info', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris-besluit/persoon.ts b/controllers/mandataris-besluit/persoon.ts index 934b85e..9c14ec7 100644 --- a/controllers/mandataris-besluit/persoon.ts +++ b/controllers/mandataris-besluit/persoon.ts @@ -25,7 +25,8 @@ export async function copyPersonInfo(mandatarisFullInfo: MandatarisFullInfo) { await createPerson(persoonUri, graph); await createMandatarisBesluitNotification({ title: 'Persoon aangemaakt', - description: `Een nieuwe Persoon werd aangemaakt op basis van de informatie in het Besluit. Deze Persoon zal onvolledige informatie bevatten aangezien e.g. rijksregisternummer niet gepubliceerd wordt in het Besluit.`, + description: + 'Een nieuwe Persoon werd aangemaakt op basis van de informatie in het Besluit. Deze Persoon zal onvolledige informatie bevatten aangezien e.g. rijksregisternummer niet gepubliceerd wordt in het Besluit.', type: 'warning', info: mandatarisFullInfo, }); diff --git a/controllers/mandataris.ts b/controllers/mandataris.ts index 846e15c..910e2e5 100644 --- a/controllers/mandataris.ts +++ b/controllers/mandataris.ts @@ -158,20 +158,17 @@ async function copyOverNonResourceDomainPredicates( }; } -async function findDecision(mandatarisId: string): Promise { - const isMandataris = await mandataris.isValidId(mandatarisId); +async function findDecision(mandatarisUri: string): Promise { + const isMandataris = await mandataris.isValidId(mandatarisUri); if (!isMandataris) { throw new HttpError( - `Mandataris with id ${mandatarisId} not found.`, + `Mandataris with id ${mandatarisUri} not found.`, STATUS_CODE.BAD_REQUEST, ); } - const decision = await findDecisionForMandataris({ - type: 'uri', - value: mandatarisId, - } as Term); + const decision = await findDecisionForMandataris(mandatarisUri); - return decision ? decision.value : null; + return decision ? decision.besluit : null; } export async function handleBulkSetPublicationStatus( diff --git a/data-access/mandatees-decisions.ts b/data-access/mandatees-decisions.ts index 97bb114..35589b7 100644 --- a/data-access/mandatees-decisions.ts +++ b/data-access/mandatees-decisions.ts @@ -121,7 +121,8 @@ export async function checkIfMinimalMandatarisInfoAvailable( const graph = await getMandatarisNotificationGraph(mandataris); await createNotification({ title: 'Besluit met Mandataris zonder minimale info', - description: `Een Mandataris uit het Besluit heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een startdatum, een persoon en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.`, + description: + 'Een Mandataris uit het Besluit heeft niet alle minimale informatie. Een Mandataris in een Besluit moet minstens een startdatum, een persoon en een mandaat bevatten. Het mandaat moet gekend zijn bij ABB.', type: 'error', graph, links: [