Generate tags around context

Reviewed-on: #16

.gitea/workflows/deploy.yml

@@ -0,0 +1,56 @@
+name: Create Blog Article if new notes exist
+on:
+    schedule:
+        - cron: "15 3 * * *"
+    push:
+        branches:
+            - master
+jobs:
+    prepare_blog_drafts_and_push:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Install dependencies
+              shell: bash
+              run: |
+                  apt update && apt upgrade -y
+                  apt install rustc cargo python-is-python3 pip python3-venv python3-virtualenv libmagic-dev git -y
+                  virtualenv .venv
+                  source .venv/bin/activate
+                  pip install --upgrade pip
+                  pip install -r requirements.txt
+                  git config --global user.name "Blog Creator"
+                  git config --global user.email "ridgway.infrastructure@gmail.com"
+                  git config --global push.autoSetupRemote true
+
+            - name: Create .env
+              shell: bash
+              run: |
+                  echo "TRILIUM_HOST=${{ vars.TRILIUM_HOST }}" > .env
+                  echo "TRILIUM_PORT='${{ vars.TRILIUM_PORT }}'" >> .env
+                  echo "TRILIUM_PROTOCOL='${{ vars.TRILIUM_PROTOCOL }}'" >> .env
+                  echo "TRILIUM_PASS='${{ secrets.TRILIUM_PASS }}'" >> .env
+                  echo "TRILIUM_TOKEN='${{ secrets.TRILIUM_TOKEN }}'" >> .env
+                  echo "OLLAMA_PROTOCOL='${{ vars.OLLAMA_PROTOCOL }}'" >> .env
+                  echo "OLLAMA_HOST='${{ vars.OLLAMA_HOST }}'" >> .env
+                  echo "OLLAMA_PORT='${{ vars.OLLAMA_PORT }}'" >> .env
+                  echo "EMBEDDING_MODEL='${{ vars.EMBEDDING_MODEL }}'" >> .env
+                  echo "EDITOR_MODEL='${{ vars.EDITOR_MODEL }}'" >> .env
+                  export PURE='["${{ vars.CONTENT_CREATOR_MODELS_1 }}", "${{ vars.CONTENT_CREATOR_MODELS_2 }}", "${{ vars.CONTENT_CREATOR_MODELS_3 }}", "${{ vars.CONTENT_CREATOR_MODELS_4 }}"]'
+                  echo "CONTENT_CREATOR_MODELS='$PURE'" >> .env
+                  echo "GIT_PROTOCOL='${{ vars.GIT_PROTOCOL }}'" >> .env
+                  echo "GIT_REMOTE='${{ vars.GIT_REMOTE }}'" >> .env
+                  echo "GIT_USER='${{ vars.GIT_USER }}'" >> .env
+                  echo "GIT_PASS='${{ secrets.GIT_PASS }}'" >> .env
+                  echo "N8N_SECRET='${{ secrets.N8N_SECRET }}'" >> .env
+                  echo "N8N_WEBHOOK_URL='${{ vars.N8N_WEBHOOK_URL }}'" >> .env
+                  echo "CHROMA_HOST='${{ vars.CHROMA_HOST }}'" >> .env
+                  echo "CHROMA_PORT='${{ vars.CHROMA_PORT }}'" >> .env
+
+            - name: Create Blogs
+              shell: bash
+              run: |
+                  source .venv/bin/activate
+                  python src/main.py

requirements.txt

@@ -5,3 +5,4 @@ PyGithub
 chromadb
 langchain-ollama
 PyJWT
+dotenv

src/ai_generators/ollama_md_generator.py

@@ -11,6 +11,8 @@ class OllamaGenerator:
         self.inner_title = inner_title
         self.content = content
         self.response = None
+        print("In Class")
+        print(os.environ["CONTENT_CREATOR_MODELS"])
         try:
             chroma_port = int(os.environ['CHROMA_PORT'])
         except ValueError as e:
@@ -31,7 +33,7 @@ class OllamaGenerator:
             The title for the blog is {self.inner_title}.
             Do not output the title in the markdown.
             The basis for the content of the blog is:
-                {self.content}
+                <blog>{self.content}</blog>
             """
 
     def split_into_chunks(self, text, chunk_size=100):
@@ -123,11 +125,12 @@ class OllamaGenerator:
             writing a 3000 word blog article. You like when they use almost no code examples.
             You are also Australian. The content may have light comedic elements,
             you are more professional and will attempt to tone these down
-            As this person produce and an amalgamtion of this blog as a markdown document.
+            As this person produce the final version of this blog as a markdown document
+            keeping in mind the context provided by the previous drafts.
             The title for the blog is {self.inner_title}.
             Do not output the title in the markdown. Avoid repeated sentences
             The basis for the content of the blog is:
-                {self.content}
+                <blog>{self.content}</blog>
             """
         try:
             query_embed = self.ollama_client.embed(model=self.embed_model, input=prompt_system)['embeddings']
@@ -136,7 +139,9 @@ class OllamaGenerator:
             print("Showing pertinent info from drafts used in final edited edition")
             pertinent_draft_info = '\n\n'.join(collection.query(query_embeddings=query_embed, n_results=100)['documents'][0])
             #print(pertinent_draft_info)
-            prompt_human = f"Generate the final document using this information from the drafts: {pertinent_draft_info} - Only output in markdown, do not wrap in markdown tags"
+            prompt_human = f"""Generate the final, 3000 word, draft of the blog using this information from the drafts: <context>{pertinent_draft_info}</context>
+                            - Only output in markdown, do not wrap in markdown tags, Only provide the draft not a commentary on the drafts in the context
+                            """
             print("Generating final document")
             messages = [("system", prompt_system), ("human", prompt_human),]
             self.response = self.llm.invoke(messages).text()

src/main.py

@@ -4,6 +4,9 @@ import repo_management.repo_manager as git_repo
 from notifications.n8n import N8NWebhookJwt
 import string,os
 from datetime import datetime
+from dotenv import load_dotenv
+load_dotenv()
+print(os.environ["CONTENT_CREATOR_MODELS"])
 
 
 tril = tn.TrilumNotes()
@@ -27,7 +30,7 @@ for note in tril_notes:
     ai_gen = omg.OllamaGenerator(os_friendly_title,
                                  tril_notes[note]['content'],
                                  tril_notes[note]['title'])
-    blog_path = f"/blog_creator/generated_files/{os_friendly_title}.md"
+    blog_path = f"generated_files/{os_friendly_title}.md"
     ai_gen.save_to_file(blog_path)
 
 
@@ -47,9 +50,10 @@ for note in tril_notes:
     git_branch_url = f'https://git.aridgwayweb.com/armistace/blog/src/branch/{os_friendly_title}/src/content/{os_friendly_title}.md'
     n8n_system_prompt = f"You are a blog creator notifiying the final editor of the final creation of blog available at {git_branch_url}"
     n8n_prompt_human = f"""
-        Generate an informal 150 word
+        Generate an informal 100 word
         summary describing {ai_gen.response}.
-        Don't address it or use names. ONLY OUTPUT THE RESPONSE
+        Don't address it or use names. ONLY OUTPUT THE RESPONSE.
+        ONLY OUTPUT IN PLAINTEXT STRIP ALL MARKDOWN
         """
     notification_message = ai_gen.generate_system_message(n8n_system_prompt, n8n_prompt_human)
     secret_key = os.environ['N8N_SECRET']