Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refinery reduction #49

Merged
merged 13 commits into from
Nov 11, 2024
Merged

Refinery reduction #49

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
4060e81
Removes overview charts but label distribution
JWittmeyer Nov 6, 2024
3866fb9
Remove overview card for inter annotator
JWittmeyer Nov 6, 2024
b050659
Remove Record IDE
JWittmeyer Nov 7, 2024
59f6c60
Remove bricks integrator
JWittmeyer Nov 7, 2024
d5c42a1
Merge remote-tracking branch 'origin/dev' into refinery-reduction
JWittmeyer Nov 7, 2024
4cd1486
Merge branch 'dev' into refinery-reduction
JWittmeyer Nov 8, 2024
fece5d2
Adds tmp code for local storage ide values
JWittmeyer Nov 11, 2024
2dab94b
Merge branch 'refinery-reduction' of github.com:code-kern-ai/refinery…
JWittmeyer Nov 11, 2024
32c24a9
Adds tooltip to constant file
JWittmeyer Nov 11, 2024
2362d32
Center overview dropdown to full grid
JWittmeyer Nov 11, 2024
f044660
PR comments
JWittmeyer Nov 11, 2024
b07766a
Removes displayGraphs enum
JWittmeyer Nov 11, 2024
5c2bf02
Submodule merge & remove unused
JWittmeyer Nov 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 2 additions & 36 deletions src/components/projects/projectId/attributes/attributeId/AttributeCalculations.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import { TOOLTIPS_DICT } from "@/src/util/tooltip-constants";
import { selectAllUsers, selectOrganizationId, setComments } from "@/src/reduxStore/states/general";
import { CommentDataManager } from "@/src/util/classes/comments";
import { CommentType } from "@/src/types/shared/comments";
import BricksIntegrator from "@/src/components/shared/bricks-integrator/BricksIntegrator";
import { AttributeCodeLookup } from "@/src/util/classes/attribute-calculation";
import KernDropdown from "@/submodules/react-components/components/KernDropdown";
import { useWebsocket } from "@/submodules/react-components/hooks/web-socket/useWebsocket";
Expand All @@ -35,6 +34,7 @@ import { getLookupListsByProjectId } from "@/src/services/base/lookup-lists";
import { getLabelingTasksByProjectId, getProjectTokenization } from "@/src/services/base/project";
import { getAttributeByAttributeId, updateAttribute } from "@/src/services/base/project-setting";
import { Application, CurrentPage } from "@/submodules/react-components/hooks/web-socket/constants";
import { VisitBricksButton } from "@/src/components/shared/bricks/VisitBricksButton";

const EDITOR_OPTIONS = { theme: 'vs-light', language: 'python', readOnly: false };

Expand All @@ -53,18 +53,13 @@ export default function AttributeCalculation() {
const [isNameOpen, setIsNameOpen] = useState(false);
const [duplicateNameExists, setDuplicateNameExists] = useState(false);
const [tooltipsArray, setTooltipsArray] = useState<string[]>([]);
const [isInitial, setIsInitial] = useState(null); //null as add state to differentiate between initial, not and unchecked
const [editorOptions, setEditorOptions] = useState(EDITOR_OPTIONS);
const [tokenizationProgress, setTokenizationProgress] = useState(0);
const [editorValue, setEditorValue] = useState('');
const [attributeName, setAttributeName] = useState('');
const [checkUnsavedChanges, setCheckUnsavedChanges] = useState(false);
const [enableRunButton, setEnableButton] = useState(false);

useEffect(() => {
if (!currentAttribute) return;
if (isInitial == null) setIsInitial(AttributeCodeLookup.isCodeStillTemplate(currentAttribute.sourceCode, currentAttribute.dataType))
}, [currentAttribute]);

useEffect(() => {
if (!projectId) return;
Expand Down Expand Up @@ -202,9 +197,6 @@ export default function AttributeCalculation() {
}, attributeNew.dataType);
}

function openBricksIntegrator() {
document.getElementById('bricks-integrator-open-button').click();
}

function onScrollEvent(event: any) {
if (!(event.target instanceof HTMLElement)) return;
Expand Down Expand Up @@ -233,13 +225,6 @@ export default function AttributeCalculation() {
});
}

function updateNameAndCodeBricksIntegrator(code: string) {
setEditorValue(code);
const regMatch: any = getPythonFunctionRegExMatch(code);
updateSourceCode(code, regMatch[2]);
setIsInitial(false);
}

function refetchLabelingTasksAndProcess() {
getLabelingTasksByProjectId(projectId, (res) => {
const labelingTasks = postProcessLabelingTasks(res['data']['projectByProjectId']['labelingTasks']['edges']);
Expand Down Expand Up @@ -368,13 +353,7 @@ export default function AttributeCalculation() {
<div className="flex flex-row items-center justify-between my-3">
<div className="text-sm leading-5 font-medium text-gray-700 inline-block mr-2">Editor</div>
<div className="flex flex-row flex-nowrap">
<BricksIntegrator
moduleTypeFilter="generator,classifier" functionType="Attribute"
nameLookups={attributes.map(a => a.name)}
preparedCode={(code: string) => {
if (currentAttribute.state == AttributeState.USABLE) return;
updateNameAndCodeBricksIntegrator(code);
}} />
<VisitBricksButton urlExtension="generators" tooltipPlacement="left" size="small" />
<Tooltip content={TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.AVAILABLE_LIBRARIES} placement="bottom" color="invert">
<a href="https://github.com/code-kern-ai/refinery-ac-exec-env/blob/dev/requirements.txt"
target="_blank"
Expand All @@ -386,19 +365,6 @@ export default function AttributeCalculation() {
</div>

<div className="border mt-1 relative">
{isInitial && <div
className="absolute top-0 bottom-0 left-0 right-0 bg-gray-200 flex items-center justify-center z-10" style={{ opacity: '0.9' }}>
<div className="flex flex-col gap-2">
<button onClick={openBricksIntegrator}
className="bg-white text-gray-900 text font-semibold px-4 py-2 rounded-md border border-gray-300 hover:bg-gray-50 focus:outline-none">
Search in bricks
</button>
<button onClick={() => setIsInitial(false)}
className="bg-white text-gray-900 text font-semibold px-4 py-2 rounded-md border border-gray-300 hover:bg-gray-50 focus:outline-none">
Start from scratch
</button>
</div>
</div>}
<Editor
height="400px"
defaultLanguage={'python'}
Expand Down
4 changes: 1 addition & 3 deletions src/components/projects/projectId/heuristics/HeuristicsOverview.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ import AddActiveLeanerModal from "./modals/AddActiveLearnerModal";
import { postProcessingEmbeddings } from "@/src/util/components/projects/projectId/settings/embeddings-helper";
import { CommentType } from "@/src/types/shared/comments";
import { CommentDataManager } from "@/src/util/classes/comments";
import { selectAllUsers, selectOrganizationId, setBricksIntegrator, setComments } from "@/src/reduxStore/states/general";
import { getEmptyBricksIntegratorConfig } from "@/src/util/shared/bricks-integrator-helper";
import { selectAllUsers, selectOrganizationId, setComments } from "@/src/reduxStore/states/general";
import { useWebsocket } from "@/submodules/react-components/hooks/web-socket/useWebsocket";
import { getAllComments } from "@/src/services/base/comment";
import { getAttributes } from "@/src/services/base/attribute";
Expand Down Expand Up @@ -48,7 +47,6 @@ export function HeuristicsOverview() {
dispatch(setAllAttributes(res.data['attributesByProjectId']));
});
}
dispatch(setBricksIntegrator(getEmptyBricksIntegratorConfig()));
}, [projectId]);

useEffect(() => {
Expand Down
21 changes: 3 additions & 18 deletions src/components/projects/projectId/heuristics/heuristicId/active-learning/ActiveLearning.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,9 @@ import HeuristicStatistics from "../shared/HeuristicStatistics";
import DangerZone from "@/src/components/shared/danger-zone/DangerZone";
import { DangerZoneEnum } from "@/src/types/shared/danger-zone";
import { getPythonClassName, getPythonClassRegExMatch } from "@/submodules/javascript-functions/python-functions-parser";
import { selectAllUsers, setBricksIntegrator, setLabelsBricksIntegrator, setComments, selectOrganizationId } from "@/src/reduxStore/states/general";
import { selectAllUsers, setComments, selectOrganizationId } from "@/src/reduxStore/states/general";
import { CommentType } from "@/src/types/shared/comments";
import { CommentDataManager } from "@/src/util/classes/comments";
import BricksIntegrator from "@/src/components/shared/bricks-integrator/BricksIntegrator";
import { InformationSourceCodeLookup, InformationSourceExamples } from "@/src/util/classes/heuristics";
import { getInformationSourceTemplate } from "@/src/util/components/projects/projectId/heuristics/heuristics-helper";
import KernDropdown from "@/submodules/react-components/components/KernDropdown";
Expand All @@ -35,6 +34,7 @@ import { getLabelingTasksByProjectId } from "@/src/services/base/project";
import { getHeuristicByHeuristicId, getPayloadByPayloadId, updateHeuristicPost } from "@/src/services/base/heuristic";
import { getEmbeddings } from "@/src/services/base/embedding";
import { Application, CurrentPage } from "@/submodules/react-components/hooks/web-socket/constants";
import { VisitBricksButton } from "@/src/components/shared/bricks/VisitBricksButton";

export default function ActiveLearning() {
const dispatch = useDispatch();
Expand All @@ -49,7 +49,6 @@ export default function ActiveLearning() {
const allUsers = useSelector(selectAllUsers);

const [lastTaskLogs, setLastTaskLogs] = useState<string[]>([]);
const [isInitialAL, setIsInitialAL] = useState<boolean>(null); //null as add state to differentiate between initial, not and unchecked
const [checkUnsavedChanges, setCheckUnsavedChanges] = useState(false);

useEffect(() => {
Expand All @@ -71,8 +70,6 @@ export default function ActiveLearning() {
if (!embeddings) return;
dispatch(setFilteredEmbeddings(embeddings.filter(e => embeddingRelevant(e, attributes, labelingTasks, currentHeuristic.labelingTaskId))));
refetchTaskByTaskIdAndProcess();
if (isInitialAL == null) setIsInitialAL(InformationSourceCodeLookup.isCodeStillTemplate(currentHeuristic.sourceCode.replace(embeddingsFiltered[0]?.name, '@@EMBEDDING@@')) != null)

}, [currentHeuristic]);

useEffect(() => {
Expand Down Expand Up @@ -255,23 +252,11 @@ export default function ActiveLearning() {
)}
</div>
<div className="flex flex-row flex-nowrap items-center ml-auto">
<BricksIntegrator
moduleTypeFilter={currentHeuristic.labelingTaskType == 'MULTICLASS_CLASSIFICATION' ? 'classifier' : 'extractor'}
executionTypeFilter="activeLearner"
functionType="Heuristic"
labelingTaskId={currentHeuristic.labelingTaskId}
preparedCode={(code: string) => {
updateSourceCode(code);
setIsInitialAL(false);
}}
newTaskId={(value) => setValueToLabelingTask(value)}
/>
<VisitBricksButton tooltipPlacement="left" size="small" />
</div>
</div>
<HeuristicsEditor
isInitial={isInitialAL}
updatedSourceCode={(code: string) => updateSourceCode(code)}
setIsInitial={(val) => setIsInitialAL(val)}
setCheckUnsavedChanges={(val) => setCheckUnsavedChanges(val)}
/>

Expand Down
52 changes: 11 additions & 41 deletions ...mponents/projects/projectId/heuristics/heuristicId/labeling-function/LabelingFunction.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ import { useRouter } from "next/router";
import HeuristicsLayout from "../shared/HeuristicsLayout";
import { useDispatch, useSelector } from "react-redux";
import { selectProjectId } from "@/src/reduxStore/states/project";
import { useCallback, useEffect, useState } from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { selectHeuristic, setActiveHeuristics, updateHeuristicsState } from "@/src/reduxStore/states/pages/heuristics";
import { postProcessCurrentHeuristic, postProcessLastTaskLogs } from "@/src/util/components/projects/projectId/heuristics/heuristicId/heuristics-details-helper";
import { Tooltip } from "@nextui-org/react";
import { TOOLTIPS_DICT } from "@/src/util/tooltip-constants";
import { postProcessLabelingTasks, postProcessLabelingTasksSchema } from "@/src/util/components/projects/projectId/settings/labeling-tasks-helper";
import { selectVisibleAttributesHeuristics, selectLabelingTasksAll, setLabelingTasksAll, SELECT_LABELING_TASKS_ALL_SNAPSHOT_ACCESS } from "@/src/reduxStore/states/pages/settings";
import { selectVisibleAttributesHeuristics, selectLabelingTasksAll, setLabelingTasksAll } from "@/src/reduxStore/states/pages/settings";
import HeuristicsEditor from "../shared/HeuristicsEditor";
import DangerZone from "@/src/components/shared/danger-zone/DangerZone";
import HeuristicRunButtons from "../shared/HeuristicRunButtons";
Expand All @@ -22,10 +22,9 @@ import { SampleRecord } from "@/src/types/components/projects/projectId/heuristi
import { getPythonFunctionRegExMatch } from "@/submodules/javascript-functions/python-functions-parser";
import CalculationProgress from "./CalculationProgress";
import { copyToClipboard } from "@/submodules/javascript-functions/general";
import { selectAllUsers, selectOrganizationId, setBricksIntegrator, setComments } from "@/src/reduxStore/states/general";
import { selectAllUsers, selectOrganizationId, setComments } from "@/src/reduxStore/states/general";
import { CommentType } from "@/src/types/shared/comments";
import { CommentDataManager } from "@/src/util/classes/comments";
import BricksIntegrator from "@/src/components/shared/bricks-integrator/BricksIntegrator";
import { InformationSourceCodeLookup, InformationSourceExamples } from "@/src/util/classes/heuristics";
import { getInformationSourceTemplate } from "@/src/util/components/projects/projectId/heuristics/heuristics-helper";
import KernDropdown from "@/submodules/react-components/components/KernDropdown";
Expand All @@ -36,8 +35,8 @@ import { useWebsocket } from "@/submodules/react-components/hooks/web-socket/use
import { getAllComments } from "@/src/services/base/comment";
import { getLabelingTasksByProjectId } from "@/src/services/base/project";
import { getHeuristicByHeuristicId, getLabelingFunctionOn10Records, getPayloadByPayloadId, updateHeuristicPost } from "@/src/services/base/heuristic";
import { getStoreSnapshotValue } from "@/src/reduxStore/store";
import { Application, CurrentPage } from "@/submodules/react-components/hooks/web-socket/constants";
import { VisitBricksButton } from "@/src/components/shared/bricks/VisitBricksButton";

export default function LabelingFunction() {
const dispatch = useDispatch();
Expand All @@ -53,7 +52,6 @@ export default function LabelingFunction() {
const [selectedAttribute, setSelectedAttribute] = useState<Attribute>(null);
const [sampleRecords, setSampleRecords] = useState<SampleRecord>(null);
const [displayLogWarning, setDisplayLogWarning] = useState<boolean>(false);
const [isInitialLf, setIsInitialLf] = useState<boolean>(null); //null as add state to differentiate between initial, not and unchecked
const [checkUnsavedChanges, setCheckUnsavedChanges] = useState(false);
const [runOn10IsRunning, setRunOn10IsRunning] = useState(false);
const [justClickedRun, setJustClickedRun] = useState(false);
Expand All @@ -73,9 +71,8 @@ export default function LabelingFunction() {

useEffect(() => {
if (!currentHeuristic) return;
if (isInitialLf == null) setIsInitialLf(InformationSourceCodeLookup.isCodeStillTemplate(currentHeuristic.sourceCode) != null);
refetchTaskByTaskIdAndProcess();
}, [currentHeuristic, isInitialLf]);
}, [currentHeuristic]);

useEffect(() => {
if (!projectId || allUsers.length == 0) return;
Expand Down Expand Up @@ -199,28 +196,14 @@ export default function LabelingFunction() {
}
}, [currentHeuristic]);


const setValueToLabelingTask = useCallback((value: string) => {
const labelingTask = labelingTasks.find(a => a.id == value);
const updateHeuristic = (labelingTasks: any[], maxI: number, task?: any) => {
const labelingTask = task || labelingTasks.find(a => a.id == value);
if (!labelingTask && maxI > 0) {
setTimeout(() => updateHeuristic(getStoreSnapshotValue(SELECT_LABELING_TASKS_ALL_SNAPSHOT_ACCESS), maxI - 1), 100);
} else {
updateHeuristicPost(projectId, currentHeuristic.id, labelingTask.id, currentHeuristic.sourceCode, currentHeuristic.description, currentHeuristic.name, (res) => {
dispatch(updateHeuristicsState(currentHeuristic.id, { labelingTaskId: labelingTask.id, labelingTaskName: labelingTask.name, labels: labelingTask.labels }))
});
}
}
if (!labelingTask) {
//try timeout as this is usually caused by race condition (creating the task+label through the integrator)
setTimeout(() => updateHeuristic(getStoreSnapshotValue(SELECT_LABELING_TASKS_ALL_SNAPSHOT_ACCESS), 5), 100);
} else updateHeuristic(labelingTasks, 0, labelingTask);
}, [projectId, currentHeuristic, labelingTasks])

const orgId = useSelector(selectOrganizationId);
useWebsocket(orgId, Application.REFINERY, CurrentPage.LABELING_FUNCTION, handleWebsocketNotification, projectId);

const bricksUrlExtension = useMemo(() => {
if (currentHeuristic?.labelingTaskType == 'INFORMATION_EXTRACTION') return 'extractors';
return "classifiers"
}, [currentHeuristic?.labelingTaskType]);

return (
<HeuristicsLayout updateSourceCode={(code: string) => updateSourceCodeToDisplay(code)}>
{currentHeuristic && <div>
Expand All @@ -244,18 +227,7 @@ export default function LabelingFunction() {
</div>
<div className="flex items-center justify-center flex-shrink-0">
<div className="flex flex-row flex-nowrap items-center ml-auto">
<BricksIntegrator
moduleTypeFilter={currentHeuristic.labelingTaskType == 'MULTICLASS_CLASSIFICATION' ? 'classifier' : 'extractor'}
executionTypeFilter="pythonFunction,premium"
functionType="Heuristic"
labelingTaskId={currentHeuristic.labelingTaskId}
preparedCode={(code: string) => {
updateSourceCode(code);
setIsInitialLf(false);
}}
newTaskId={(value) => setValueToLabelingTask(value)}
/>

<VisitBricksButton urlExtension={bricksUrlExtension} tooltipPlacement="left" size="small" />
<Tooltip content={TOOLTIPS_DICT.LABELING_FUNCTION.INSTALLED_LIBRARIES} color="invert" placement="left">
<a href="https://github.com/code-kern-ai/refinery-lf-exec-env/blob/dev/requirements.txt"
target="_blank"
Expand All @@ -267,9 +239,7 @@ export default function LabelingFunction() {
</div>
</div>
<HeuristicsEditor
isInitial={isInitialLf}
updatedSourceCode={(code: string) => updateSourceCode(code)}
setIsInitial={(val: boolean) => setIsInitialLf(val)}
setCheckUnsavedChanges={(val: boolean) => setCheckUnsavedChanges(val)} />

<div className="mt-2 flex flex-grow justify-between items-center float-right">
Expand Down
Loading