Skip to content

Commit

Permalink
fix: remember to embed last batch
Browse files Browse the repository at this point in the history
  • Loading branch information
amiller68 committed Apr 15, 2024
1 parent fc759c1 commit 3f69d3a
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion src/knowledge-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ export class KnowledgeStore {
if (item) {
this.documents = item;
}
await this.prune();
return this.documents;
}

Expand All @@ -72,6 +71,7 @@ export class KnowledgeStore {
const doc = createDocument(title, tags);
// Split the document into chunks (which are just Lanhchain documents)
const chunks = await chunkText(title, content);

// Embed each chunk and save the embeddings to localforage
const batch_size = 10;
let batch = [];
Expand All @@ -91,6 +91,12 @@ export class KnowledgeStore {
await this.prune();
throw Error('Error embedding batch: ' + e);
}

// Embed the last batch
if (batch.length > 0) {
await Promise.all(batch.map((c) => this.embedChunk(doc.id, c)));
}

// Add the document to our list of documents
this.documents.set(doc.id, doc);
await this.save();
Expand Down

0 comments on commit 3f69d3a

Please sign in to comment.