propogate file pattern to determine filetype

src/instructlab/sdg/utils/chunking_tmp.py

@@ -534,7 +534,7 @@ def _extract_filetypes_from_docs(documents: List):
     pdf_docs = []
 
     for doc in documents:
-        filetype = doc.rsplit(".")
+        filetype = doc.rsplit(".")[-1]
         if filetype == "md":
             md_docs.append(doc)
         elif filetype == "pdf":
@@ -560,8 +560,8 @@ def chunk_documents(
          List[str]: List of chunked documents.
     """
     print(f"THIS IS KHALED: in chunk_documents {leaf_node=}")
-    leaf_node_path = leaf_node[0]["taxonomy_path"].replace("->", "_")
-    documents = leaf_node[0]["document"]
+    leaf_node_path = leaf_node[0]["pattern"]
+    documents = leaf_node[0]["pattern"]
     print(f"""THIS IS KHALED:
         {leaf_node_path=},
         {documents=},

src/instructlab/sdg/utils/taxonomy.py

@@ -152,7 +152,7 @@ def _get_documents(
                                 file_contents.append(pdf_text)
 
             if file_contents:
-                return file_contents
+                return file_contents, file_patterns
             raise SystemExit("Couldn't find knowledge documents")
 
         except (OSError, git.exc.GitCommandError, FileNotFoundError) as e:
@@ -182,8 +182,9 @@ def _read_taxonomy_file(file_path: str | Path, yamllint_config: str | None = Non
         task_description = contents.get("task_description", None)
         domain = contents.get("domain")
         documents = contents.get("document")
+        print(f"")
         if documents:
-            documents = _get_documents(source=documents)
+            document_contents, file_patterns = _get_documents(source=documents)
             logger.debug("Content from git repo fetched")
 
         for seed_example in contents.get("seed_examples"):
@@ -195,7 +196,8 @@ def _read_taxonomy_file(file_path: str | Path, yamllint_config: str | None = Non
                         "questions_and_answers": question_answer_list,
                         "context": context,
                         "taxonomy_path": tax_path,
-                        "document": documents,
+                        "document": document_contents,
+                        "pattern": file_patterns,
                         "domain": domain,
                         "document_outline": contents.get("document_outline"),
                     }