From 4a4fd67b64e24e67e8fe227b4d39d55e341c31f5 Mon Sep 17 00:00:00 2001 From: Lionello Lunesu Date: Sun, 10 Nov 2024 14:59:56 +0000 Subject: [PATCH] shallow clone to speed up --- app/get_knowledge_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/get_knowledge_base.py b/app/get_knowledge_base.py index 2a9b236..c90b268 100644 --- a/app/get_knowledge_base.py +++ b/app/get_knowledge_base.py @@ -19,7 +19,7 @@ def clone_repository(repo_url, local_dir): """ Clone or pull the repository based on its existence. """ if not os.path.exists(local_dir): print(f"Cloning repository into {local_dir}") - Repo.clone_from(repo_url, local_dir) + Repo.clone_from(repo_url, local_dir, depth=1) else: print(f"Repository already exists at {local_dir}. Pulling latest changes...") repo = Repo(local_dir) @@ -137,7 +137,7 @@ def parse_markdown_file_to_json(file_path): for section in sections: about = ", ".join(section["about"]) text = " ".join(line for line in section["text"] if line) - + if about and text: # Only insert if both 'about' and 'text' are not empty json_output.append({ "id": current_id,