syntax fixes for retry logic

Reviewed-on: #16

.gitea/workflows/deploy.yml

@@ -0,0 +1,56 @@
+name: Create Blog Article if new notes exist
+on:
+    schedule:
+        - cron: "15 18 * * *"
+    push:
+        branches:
+            - master
+jobs:
+    prepare_blog_drafts_and_push:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Install dependencies
+              shell: bash
+              run: |
+                  apt update && apt upgrade -y
+                  apt install rustc cargo python-is-python3 pip python3-venv python3-virtualenv libmagic-dev git -y
+                  virtualenv .venv
+                  source .venv/bin/activate
+                  pip install --upgrade pip
+                  pip install -r requirements.txt
+                  git config --global user.name "Blog Creator"
+                  git config --global user.email "ridgway.infrastructure@gmail.com"
+                  git config --global push.autoSetupRemote true
+
+            - name: Create .env
+              shell: bash
+              run: |
+                  echo "TRILIUM_HOST=${{ vars.TRILIUM_HOST }}" > .env
+                  echo "TRILIUM_PORT='${{ vars.TRILIUM_PORT }}'" >> .env
+                  echo "TRILIUM_PROTOCOL='${{ vars.TRILIUM_PROTOCOL }}'" >> .env
+                  echo "TRILIUM_PASS='${{ secrets.TRILIUM_PASS }}'" >> .env
+                  echo "TRILIUM_TOKEN='${{ secrets.TRILIUM_TOKEN }}'" >> .env
+                  echo "OLLAMA_PROTOCOL='${{ vars.OLLAMA_PROTOCOL }}'" >> .env
+                  echo "OLLAMA_HOST='${{ vars.OLLAMA_HOST }}'" >> .env
+                  echo "OLLAMA_PORT='${{ vars.OLLAMA_PORT }}'" >> .env
+                  echo "EMBEDDING_MODEL='${{ vars.EMBEDDING_MODEL }}'" >> .env
+                  echo "EDITOR_MODEL='${{ vars.EDITOR_MODEL }}'" >> .env
+                  export PURE='["${{ vars.CONTENT_CREATOR_MODELS_1 }}", "${{ vars.CONTENT_CREATOR_MODELS_2 }}", "${{ vars.CONTENT_CREATOR_MODELS_3 }}", "${{ vars.CONTENT_CREATOR_MODELS_4 }}"]'
+                  echo "CONTENT_CREATOR_MODELS='$PURE'" >> .env
+                  echo "GIT_PROTOCOL='${{ vars.GIT_PROTOCOL }}'" >> .env
+                  echo "GIT_REMOTE='${{ vars.GIT_REMOTE }}'" >> .env
+                  echo "GIT_USER='${{ vars.GIT_USER }}'" >> .env
+                  echo "GIT_PASS='${{ secrets.GIT_PASS }}'" >> .env
+                  echo "N8N_SECRET='${{ secrets.N8N_SECRET }}'" >> .env
+                  echo "N8N_WEBHOOK_URL='${{ vars.N8N_WEBHOOK_URL }}'" >> .env
+                  echo "CHROMA_HOST='${{ vars.CHROMA_HOST }}'" >> .env
+                  echo "CHROMA_PORT='${{ vars.CHROMA_PORT }}'" >> .env
+
+            - name: Create Blogs
+              shell: bash
+              run: |
+                  source .venv/bin/activate
+                  python src/main.py

.gitignore

@@ -3,3 +3,8 @@ __pycache__
 .venv
 .aider*
 .vscode
+.zed
+pyproject.toml
+.ropeproject
+generated_files/*
+pyright*

Dockerfile

@@ -7,8 +7,12 @@ ENV PYTHONUNBUFFERED 1
 
 ADD src/ /blog_creator
 
-RUN apt-get update && apt-get install -y rustc cargo python-is-python3 pip python3.12-venv libmagic-dev 
+RUN apt-get update && apt-get install -y rustc cargo python-is-python3 pip python3-venv libmagic-dev git
-
+# Need to set up git here or we get funky errors
+RUN git config --global user.name "Blog Creator"
+RUN git config --global user.email "ridgway.infrastructure@gmail.com"
+RUN git config --global push.autoSetupRemote true
+#Get a python venv going as well cause safety
 RUN python -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 

README.md

@@ -3,10 +3,19 @@
 This creator requires you to use a working Trilium Instance and create a .env file with the following
 
 ```
-TRILIUM_HOST
+TRILIUM_HOST=
-TRILIUM_PORT
+TRILIUM_PORT=
-TRILIUM_PROTOCOL
+TRILIUM_PROTOCOL=
-TRILIUM_PASS
+TRILIUM_PASS=
+TRILIUM_TOKEN=
+OLLAMA_PROTOCOL=
+OLLAMA_HOST=
+OLLAMA_PORT=11434
+EMBEDDING_MODEL=
+EDITOR_MODEL=
+# This is expected in python list format example `[phi4-mini:latest, qwen3:1.7b, gemma3:latest]`
+CONTENT_CREATOR_MODELS=
+CHROMA_SERVER=<IP_ADDRESS>
 ```
 
 This container is going to be what I use to trigger a blog creation event

docker-compose.yml

@@ -1,3 +1,7 @@
+networks:
+    net:
+        driver: bridge
+
 services:
     blog_creator:
         build:
@@ -8,4 +12,33 @@ services:
             - .env
         volumes:
             - ./generated_files/:/blog_creator/generated_files
+        networks:
+            - net
 
+    chroma:
+        image: chromadb/chroma
+        container_name: chroma
+        volumes:
+            # Be aware that indexed data are located in "/chroma/chroma/"
+            # Default configuration for persist_directory in chromadb/config.py
+            # Read more about deployments: https://docs.trychroma.com/deployment
+            - chroma-data:/chroma/chroma
+        #command: "--host 0.0.0.0 --port 8000 --proxy-headers --log-config chromadb/log_config.yml --timeout-keep-alive 30"
+        environment:
+            - IS_PERSISTENT=TRUE
+        restart: unless-stopped # possible values are: "no", always", "on-failure", "unless-stopped"
+        ports:
+            - "8000:8000"
+        healthcheck:
+            # Adjust below to match your container port
+            test:
+                ["CMD", "curl", "-f", "http://localhost:8000/api/v2/heartbeat"]
+            interval: 30s
+            timeout: 10s
+            retries: 3
+        networks:
+            - net
+
+volumes:
+    chroma-data:
+        driver: local

requirements.txt

@@ -2,3 +2,7 @@ ollama
 trilium-py
 gitpython
 PyGithub
+chromadb
+langchain-ollama
+PyJWT
+dotenv

src/ai_generators/ollama_md_generator.py

@@ -1,44 +1,332 @@
+import json
 import os
-from ollama import Client
+import random
 import re
+import string
+import time
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+
+import chromadb
+from langchain_ollama import ChatOllama
+from ollama import Client
+
 
 class OllamaGenerator:
-
+    def __init__(self, title: str, content: str, inner_title: str):
-    def __init__(self, title: str, content: str, model: str):
         self.title = title
+        self.inner_title = inner_title
         self.content = content
-        ollama_url = f"{os.environ["OLLAMA_PROTOCOL"]}://{os.environ["OLLAMA_HOST"]}:{os.environ["OLLAMA_PORT"]}"
+        self.response = None
+        print("In Class")
+        print(os.environ["CONTENT_CREATOR_MODELS"])
+        try:
+            chroma_port = int(os.environ["CHROMA_PORT"])
+        except ValueError as e:
+            raise Exception(f"CHROMA_PORT is not an integer: {e}")
+        self.chroma = chromadb.HttpClient(
+            host=os.environ["CHROMA_HOST"], port=chroma_port
+        )
+        ollama_url = f"{os.environ['OLLAMA_PROTOCOL']}://{os.environ['OLLAMA_HOST']}:{os.environ['OLLAMA_PORT']}"
         self.ollama_client = Client(host=ollama_url)
-        self.ollama_model = model
+        self.ollama_model = os.environ["EDITOR_MODEL"]
-
+        self.embed_model = os.environ["EMBEDDING_MODEL"]
-    def generate_markdown(self) -> str:
+        self.agent_models = json.loads(os.environ["CONTENT_CREATOR_MODELS"])
-
+        self.llm = ChatOllama(
-        prompt = f"""
+            model=self.ollama_model, temperature=0.6, top_p=0.5
-            You are a Software Developer and DevOps expert
+        )  # This  is the level head in the room
-            who has transistioned in Developer Relations 
+        self.prompt_inject = f"""
-            writing a 1000 word blog for other tech enthusiast.
+            You are a journalist, Software Developer and DevOps expert
+            writing a 5000 word draft blog article for other tech enthusiasts.
             You like to use almost no code examples and prefer to talk
             in a light comedic tone. You are also Australian
             As this person write this blog as a markdown document.
-            The title for the blog is {self.title}.
+            The title for the blog is {self.inner_title}.
             Do not output the title in the markdown.
             The basis for the content of the blog is:
-                {self.content}
+                <blog>{self.content}</blog>
-            Only output markdown DO NOT GENERATE AN EXPLANATION
             """
-        try:
+
-            self.response = self.ollama_client.chat(model=self.ollama_model,
+    def split_into_chunks(self, text, chunk_size=100):
+        """Split text into chunks of size chunk_size"""
+        words = re.findall(r"\S+", text)
+
+        chunks = []
+        current_chunk = []
+        word_count = 0
+
+        for word in words:
+            current_chunk.append(word)
+            word_count += 1
+
+        if word_count >= chunk_size:
+            chunks.append(" ".join(current_chunk))
+            current_chunk = []
+            word_count = 0
+
+        if current_chunk:
+            chunks.append(" ".join(current_chunk))
+
+        return chunks
+
+    def generate_draft(self, model) -> str:
+        """Generate a draft blog post using the specified model"""
+
+        def _generate():
+            # the idea behind this is to make the "creativity" random amongst the content creators
+            # contorlling temperature will allow cause the output to allow more "random" connections in sentences
+            # Controlling top_p will tighten or loosen the embedding connections made
+            # The result should be varied levels of "creativity" in the writing of the drafts
+            # for more see https://python.langchain.com/v0.2/api_reference/ollama/chat_models/langchain_ollama.chat_models.ChatOllama.html
+            temp = random.uniform(0.5, 1.0)
+            top_p = random.uniform(0.4, 0.8)
+            top_k = int(random.uniform(30, 80))
+            agent_llm = ChatOllama(
+                model=model, temperature=temp, top_p=top_p, top_k=top_k
+            )
             messages = [
-                    {
+                (
-                        'role': 'user',
+                    "system",
-                        'content': f'{prompt}',
+                    "You are a creative writer specialising in writing about technology",
-                    },
+                ),
-                ])
+                ("human", self.prompt_inject),
+            ]
+            response = agent_llm.invoke(messages)
+            return (
+                response.text if hasattr(response, "text") else str(response)
+            )  # ['message']['content']
 
-            # the deepseek model returns <think> this removes those tabs from the output
+        # Retry mechanism with 30-minute timeout
-            # return re.sub(r"<think|.\n\r+?|([^;]*)\/think>",'',self.response['message']['content'])
+        timeout_seconds = 30 * 60  # 30 minutes
-            return self.response['message']['content']
+        max_retries = 3
 
+        for attempt in range(max_retries):
+            try:
+                with ThreadPoolExecutor(max_workers=1) as executor:
+                    future = executor.submit(_generate)
+                    result = future.result(timeout=timeout_seconds)
+                    return result
+            except TimeoutError:
+                print(
+                    f"AI call timed out after {timeout_seconds} seconds on attempt {attempt + 1}"
+                )
+                if attempt < max_retries - 1:
+                    print("Retrying...")
+                    time.sleep(5)  # Wait 5 seconds before retrying
+                    continue
+                else:
+                    raise Exception(
+                        f"AI call failed to complete after {max_retries} attempts with {timeout_seconds} second timeouts"
+                    )
+            except Exception as e:
+                if attempt < max_retries - 1:
+                    print(f"Attempt {attempt + 1} failed with error: {e}. Retrying...")
+                    time.sleep(5)  # Wait 5 seconds before retrying
+                    continue
+                else:
+                    raise Exception(
+                        f"Failed to generate blog draft after {max_retries} attempts: {e}"
+                    )
+
+    def get_draft_embeddings(self, draft_chunks):
+        """Get embeddings for the draft chunks"""
+        try:
+            # Handle empty draft chunks
+            if not draft_chunks:
+                print("Warning: No draft chunks to embed")
+                return []
+
+            embeds = self.ollama_client.embed(
+                model=self.embed_model, input=draft_chunks
+            )
+            embeddings = embeds.get("embeddings", [])
+
+            # Check if embeddings were generated successfully
+            if not embeddings:
+                print("Warning: No embeddings generated")
+                return []
+
+            return embeddings
+        except Exception as e:
+            print(f"Error generating embeddings: {e}")
+            return []
+
+    def id_generator(self, size=6, chars=string.ascii_uppercase + string.digits):
+        return "".join(random.choice(chars) for _ in range(size))
+
+    def load_to_vector_db(self):
+        """Load the generated blog drafts into a vector database"""
+        collection_name = (
+            f"blog_{self.title.lower().replace(' ', '_')}_{self.id_generator()}"
+        )
+        collection = self.chroma.get_or_create_collection(
+            name=collection_name
+        )  # , metadata={"hnsw:space": "cosine"})
+        # if any(collection.name == collectionname for collectionname in self.chroma.list_collections()):
+        #    self.chroma.delete_collection("blog_creator")
+        for model in self.agent_models:
+            print(f"Generating draft from {model} for load into vector database")
+            try:
+                draft_content = self.generate_draft(model)
+                draft_chunks = self.split_into_chunks(draft_content)
+
+                # Skip if no content was generated
+                if not draft_chunks or all(
+                    chunk.strip() == "" for chunk in draft_chunks
+                ):
+                    print(f"Skipping {model} - no content generated")
+                    continue
+
+                print(f"generating embeds for {model}")
+                embeds = self.get_draft_embeddings(draft_chunks)
+
+                # Skip if no embeddings were generated
+                if not embeds:
+                    print(f"Skipping {model} - no embeddings generated")
+                    continue
+
+                # Ensure we have the same number of embeddings as chunks
+                if len(embeds) != len(draft_chunks):
+                    print(
+                        f"Warning: Mismatch between chunks ({len(draft_chunks)}) and embeddings ({len(embeds)}) for {model}"
+                    )
+                    # Truncate or pad to match
+                    min_length = min(len(embeds), len(draft_chunks))
+                    draft_chunks = draft_chunks[:min_length]
+                    embeds = embeds[:min_length]
+                    if min_length == 0:
+                        print(f"Skipping {model} - no valid content/embeddings pairs")
+                        continue
+
+                ids = [model + str(i) for i in range(len(draft_chunks))]
+                chunknumber = list(range(len(draft_chunks)))
+                metadata = [{"model_agent": model} for index in chunknumber]
+                print(f"loading into collection for {model}")
+                collection.add(
+                    documents=draft_chunks,
+                    embeddings=embeds,
+                    ids=ids,
+                    metadatas=metadata,
+                )
+            except Exception as e:
+                print(f"Error processing model {model}: {e}")
+                # Continue with other models rather than failing completely
+                continue
+
+        return collection
+
+    def generate_markdown(self) -> str:
+        prompt_human = f"""
+            You are an editor taking information from {len(self.agent_models)} Software
+            Developers and Data experts
+            writing a 5000 word blog article. You like when they use almost no code examples.
+            You are also Australian. The content may have light comedic elements,
+            you are more professional and will attempt to tone these down
+            As this person produce the final version of this blog as a markdown document
+            keeping in mind the context provided by the previous drafts.
+            You are to produce the content not placeholders for further editors
+            The title for the blog is {self.inner_title}.
+            Do not output the title in the markdown. Avoid repeated sentences
+            The basis for the content of the blog is:
+                <blog>{self.content}</blog>
+            """
+
+        def _generate_final_document():
+            try:
+                embed_result = self.ollama_client.embed(
+                    model=self.embed_model, input=prompt_human
+                )
+                query_embed = embed_result.get("embeddings", [])
+                if not query_embed:
+                    print(
+                        "Warning: Failed to generate query embeddings, using empty list"
+                    )
+                    query_embed = [[]]  # Use a single empty embedding as fallback
+            except Exception as e:
+                print(f"Error generating query embeddings: {e}")
+                # Generate empty embeddings as fallback
+                query_embed = [[]]  # Use a single empty embedding as fallback
+
+            collection = self.load_to_vector_db()
+
+            # Try to query the collection, with fallback for empty collections
+            try:
+                collection_query = collection.query(
+                    query_embeddings=query_embed, n_results=100
+                )
+                print("Showing pertinent info from drafts used in final edited edition")
+
+                # Get documents with error handling
+                query_result = collection.query(
+                    query_embeddings=query_embed, n_results=100
+                )
+                documents = query_result.get("documents", [])
+
+                if documents and len(documents) > 0 and len(documents[0]) > 0:
+                    pertinent_draft_info = "\n\n".join(documents[0])
+                else:
+                    print("Warning: No relevant documents found in collection")
+                    pertinent_draft_info = "No relevant information found in drafts."
+
+            except Exception as query_error:
+                print(f"Error querying collection: {query_error}")
+                pertinent_draft_info = (
+                    "No relevant information found in drafts due to query error."
+                )
+            # print(pertinent_draft_info)
+            prompt_system = f"""Generate the final, 5000 word, draft of the blog using this information from the drafts: <context>{pertinent_draft_info}</context>
+                            - Only output in markdown, do not wrap in markdown tags, Only provide the draft not a commentary on the drafts in the context
+                            """
+            print("Generating final document")
+            messages = [
+                ("system", prompt_system),
+                ("human", prompt_human),
+            ]
+            response = self.llm.invoke(messages)
+            return response.text if hasattr(response, "text") else str(response)
+
+        try:
+            # Retry mechanism with 30-minute timeout
+            timeout_seconds = 30 * 60  # 30 minutes
+            max_retries = 3
+
+            for attempt in range(max_retries):
+                try:
+                    with ThreadPoolExecutor(max_workers=1) as executor:
+                        future = executor.submit(_generate_final_document)
+                        self.response = future.result(timeout=timeout_seconds)
+                        break  # Success, exit the retry loop
+                except TimeoutError:
+                    print(
+                        f"AI call timed out after {timeout_seconds} seconds on attempt {attempt + 1}"
+                    )
+                    if attempt < max_retries - 1:
+                        print("Retrying...")
+                        time.sleep(5)  # Wait 5 seconds before retrying
+                        continue
+                    else:
+                        raise Exception(
+                            f"AI call failed to complete after {max_retries} attempts with {timeout_seconds} second timeouts"
+                        )
+                except Exception as e:
+                    if attempt < max_retries - 1:
+                        print(
+                            f"Attempt {attempt + 1} failed with error: {e}. Retrying..."
+                        )
+                        time.sleep(5)  # Wait 5 seconds before retrying
+                        continue
+                    else:
+                        raise Exception(
+                            f"Failed to generate markdown after {max_retries} attempts: {e}"
+                        )
+
+            # self.response = self.ollama_client.chat(model=self.ollama_model,
+            #                                         messages=[
+            #             'content': f'{prompt_enhanced}',
+            #         },
+            #     ])
+            # print ("Markdown Generated")
+            # print (self.response)
+            return self.response  # ['message']['content']
 
         except Exception as e:
             raise Exception(f"Failed to generate markdown: {e}")
@@ -47,3 +335,44 @@ class OllamaGenerator:
         with open(filename, "w") as f:
             f.write(self.generate_markdown())
 
+    def generate_system_message(self, prompt_system, prompt_human):
+        def _generate():
+            messages = [
+                ("system", prompt_system),
+                ("human", prompt_human),
+            ]
+            response = self.llm.invoke(messages)
+            ai_message = response.text if hasattr(response, "text") else str(response)
+            return ai_message
+
+        # Retry mechanism with 30-minute timeout
+        timeout_seconds = 30 * 60  # 30 minutes
+        max_retries = 3
+
+        for attempt in range(max_retries):
+            try:
+                with ThreadPoolExecutor(max_workers=1) as executor:
+                    future = executor.submit(_generate)
+                    result = future.result(timeout=timeout_seconds)
+                    return result
+            except TimeoutError:
+                print(
+                    f"AI call timed out after {timeout_seconds} seconds on attempt {attempt + 1}"
+                )
+                if attempt < max_retries - 1:
+                    print("Retrying...")
+                    time.sleep(5)  # Wait 5 seconds before retrying
+                    continue
+                else:
+                    raise Exception(
+                        f"AI call failed to complete after {max_retries} attempts with {timeout_seconds} second timeouts"
+                    )
+            except Exception as e:
+                if attempt < max_retries - 1:
+                    print(f"Attempt {attempt + 1} failed with error: {e}. Retrying...")
+                    time.sleep(5)  # Wait 5 seconds before retrying
+                    continue
+                else:
+                    raise Exception(
+                        f"Failed to generate system message after {max_retries} attempts: {e}"
+                    )

src/main.py

@@ -1,5 +1,13 @@
 import ai_generators.ollama_md_generator as omg
 import trilium.notes as tn
+import repo_management.repo_manager as git_repo
+from notifications.n8n import N8NWebhookJwt
+import string,os
+from datetime import datetime
+from dotenv import load_dotenv
+load_dotenv()
+print(os.environ["CONTENT_CREATOR_MODELS"])
+
 
 tril = tn.TrilumNotes()
 
@@ -7,16 +15,66 @@ tril.get_new_notes()
 tril_notes = tril.get_notes_content()
 
 
-def convert_to_lowercase_with_underscores(string):
+def convert_to_lowercase_with_underscores(s):
-    return string.lower().replace(" ", "_")
+    allowed = set(string.ascii_letters + string.digits + ' ')
+    filtered_string = ''.join(c for c in s if c in allowed)
+    return filtered_string.lower().replace(" ", "_")
 
 
 for note in tril_notes:
     print(tril_notes[note]['title'])
     # print(tril_notes[note]['content'])
     print("Generating Document")
-    ai_gen = omg.OllamaGenerator(tril_notes[note]['title'],
+
-                                 tril_notes[note]['content'],
-                                 "deepseek-r1:7b")
     os_friendly_title = convert_to_lowercase_with_underscores(tril_notes[note]['title'])
-    ai_gen.save_to_file(f"./generated_files/{os_friendly_title}.md")
+    ai_gen = omg.OllamaGenerator(os_friendly_title,
+                                 tril_notes[note]['content'],
+                                 tril_notes[note]['title'])
+    blog_path = f"generated_files/{os_friendly_title}.md"
+    ai_gen.save_to_file(blog_path)
+
+
+    # Generate commit messages and push to repo
+    print("Generating Commit Message")
+    git_sytem_prompt = "You are a blog creator commiting a piece of content to a central git repo"
+    git_human_prompt = f"Generate a 5 word git commit message describing {ai_gen.response}. ONLY OUTPUT THE RESPONSE"
+    commit_message = ai_gen.generate_system_message(git_sytem_prompt, git_human_prompt)
+    git_user = os.environ["GIT_USER"]
+    git_pass = os.environ["GIT_PASS"]
+    repo_manager = git_repo.GitRepository("blog/", git_user, git_pass)
+    print("Pushing to Repo")
+    repo_manager.create_copy_commit_push(blog_path, os_friendly_title, commit_message)
+
+    # Generate notification for Matrix
+    print("Generating Notification Message")
+    git_branch_url = f'https://git.aridgwayweb.com/armistace/blog/src/branch/{os_friendly_title}/src/content/{os_friendly_title}.md'
+    n8n_system_prompt = f"You are a blog creator notifiying the final editor of the final creation of blog available at {git_branch_url}"
+    n8n_prompt_human = f"""
+        Generate an informal 100 word
+        summary describing {ai_gen.response}.
+        Don't address it or use names. ONLY OUTPUT THE RESPONSE.
+        ONLY OUTPUT IN PLAINTEXT STRIP ALL MARKDOWN
+        """
+    notification_message = ai_gen.generate_system_message(n8n_system_prompt, n8n_prompt_human)
+    secret_key = os.environ['N8N_SECRET']
+    webhook_url = os.environ['N8N_WEBHOOK_URL']
+    notification_string = f"""
+        <h2>{tril_notes[note]['title']}</h2>
+        <h3>Summary</h3>
+        <p>{notification_message}</p>
+        <h3>Branch</h3>
+        <p>{os_friendly_title}</p>
+        <p><a href="{git_branch_url}">Link to Branch</a></p>
+    """
+
+    payload = {
+        "message": f"{notification_string}",
+        "timestamp": datetime.now().isoformat()
+    }
+
+    webhook_client = N8NWebhookJwt(secret_key, webhook_url)
+
+    print("Notifying")
+    n8n_result = webhook_client.send_webhook(payload)
+
+    print(f"N8N response: {n8n_result['status']}")

src/notifications/n8n.py

@@ -0,0 +1,45 @@
+from datetime import datetime, timedelta
+import jwt
+import requests
+from typing import Dict, Optional
+
+class N8NWebhookJwt:
+    def __init__(self, secret_key: str, webhook_url: str):
+        self.secret_key = secret_key
+        self.webhook_url = webhook_url
+        self.token_expiration = datetime.now() + timedelta(hours=1)
+
+    def _generate_jwt_token(self, payload: Dict) -> str:
+        """Generate JWT token with the given payload."""
+        # Include expiration time (optional)
+        payload["exp"] = self.token_expiration.timestamp()
+        encoded_jwt = jwt.encode(
+                    payload,
+                    self.secret_key,
+                    algorithm="HS256",
+                )
+        return encoded_jwt #jwt.decode(encoded_jwt, self.secret_key, algorithms=['HS256'])
+
+    def send_webhook(self, payload: Dict) -> Dict:
+        """Send a webhook request with JWT authentication."""
+        # Generate JWT token
+        token = self._generate_jwt_token(payload)
+
+        # Set headers with JWT token
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json"
+        }
+
+        # Send POST request
+        response = requests.post(
+            self.webhook_url,
+            json=payload,
+            headers=headers
+        )
+
+        # Handle response
+        if response.status_code == 200:
+            return {"status": "success", "response": response.json()}
+        else:
+            return {"status": "error", "response": response.status_code, "message": response.text}

src/repo_management/push_markdown.py

@@ -1,48 +0,0 @@
-import os
-import sys
-from git import Repo
-
-# Set these variables accordingly
-REPO_OWNER = "your_repo_owner"
-REPO_NAME = "your_repo_name"
-
-def clone_repo(repo_url, branch="main"):
-    Repo.clone_from(repo_url, ".", branch=branch)
-
-def create_markdown_file(file_name, content):
-    with open(f"{file_name}.md", "w") as f:
-        f.write(content)
-
-def commit_and_push(file_name, message):
-    repo = Repo(".")
-    repo.index.add([f"{file_name}.md"])
-    repo.index.commit(message)
-    repo.remote().push()
-
-def create_new_branch(branch_name):
-    repo = Repo(".")
-    repo.create_head(branch_name).checkout()
-    repo.head.reference.set_tracking_url(f"https://your_git_server/{REPO_OWNER}/{REPO_NAME}.git/{branch_name}")
-    repo.remote().push()
-
-if __name__ == "__main__":
-    if len(sys.argv) < 3:
-        print("Usage: python push_markdown.py <repo_url> <markdown_file_name>")
-        sys.exit(1)
-
-    repo_url = sys.argv[1]
-    file_name = sys.argv[2]
-
-    # Clone the repository
-    clone_repo(repo_url)
-
-    # Create a new Markdown file with content
-    create_markdown_file(file_name, "Hello, World!\n")
-
-    # Commit and push changes to the main branch
-    commit_and_push(file_name, f"Add {file_name}.md")
-
-    # Create a new branch named after the Markdown file
-    create_new_branch(file_name)
-
-    print(f"Successfully created '{file_name}' branch with '{file_name}.md'.")

src/repo_management/repo_manager.py

@@ -1,35 +1,108 @@
 import os
-from git import Git
+import shutil
-from git.repo import BaseRepository
+from urllib.parse import quote
-from git.exc import InvalidGitRepositoryError
-from git.remote import RemoteAction
 
-# Set the path to your blog repo here
+from git import Repo
-blog_repo = "/path/to/your/blog/repo"
+from git.exc import GitCommandError
 
-# Checkout a new branch and create a new file for our blog post
+
-branch_name = "new-post"
+class GitRepository:
+    # This is designed to be transitory it will desctruvtively create the repo at repo_path
+    # if you have uncommited changes you can kiss them goodbye!
+    # Don't use the repo created by this function for dev -> its a tool!
+    # It is expected that  when used you will add, commit, push, delete
+    def __init__(self, repo_path, username=None, password=None):
+        git_protocol = os.environ["GIT_PROTOCOL"]
+        git_remote = os.environ["GIT_REMOTE"]
+        # if username is not set we don't need parse to the url
+        if username == None or password == None:
+            remote = f"{git_protocol}://{git_remote}"
+        else:
+            # of course if it is we need to parse and escape it so that it
+            # can generate a url
+            git_user = quote(username)
+            git_password = quote(password)
+            remote = f"{git_protocol}://{git_user}:{git_password}@{git_remote}"
+
+        if os.path.exists(repo_path):
+            shutil.rmtree(repo_path)
+        self.repo_path = repo_path
+        print("Cloning Repo")
+        Repo.clone_from(remote, repo_path)
+        self.repo = Repo(repo_path)
+        self.username = username
+        self.password = password
+
+    def clone(self, remote_url, destination_path):
+        """Clone a Git repository with authentication"""
         try:
-    repo = Git(blog_repo)
+            self.repo.clone(remote_url, destination_path)
-    repo.checkout("-b", branch_name, "origin/main")
+            return True
-    with open("my-blog-post.md", "w") as f:
+        except GitCommandError as e:
-        f.write(content)
+            print(f"Cloning failed: {e}")
-except InvalidGitRepositoryError:
+            return False
-    # Handle repository errors gracefully
-    pass
 
-# Add and commit the changes to Git
+    def fetch(self, remote_name="origin", ref_name="main"):
-repo.add("my-blog-post.md")
+        """Fetch updates from a remote repository with authentication"""
-repo.commit("-m", "Added new blog post about DevOps best practices.")
-
-# Push the changes to Git and create a PR
-repo.remote().push("refs/heads/{0}:refs/for/main".format(branch_name), "--set-upstream")
-base_branch = "origin/main"
-target_branch = "main"
-pr_title = "DevOps best practices"
         try:
-    repo.create_head("{0}-{1}", base=base_branch, message="{}".format(pr_title))
+            self.repo.remotes[remote_name].fetch(ref_name=ref_name)
-except RemoteAction.GitExitStatus as e:
+            return True
-    # Handle Git exit status errors gracefully
+        except GitCommandError as e:
-    pass
+            print(f"Fetching failed: {e}")
+            return False
 
+    def pull(self, remote_name="origin", ref_name="main"):
+        """Pull updates from a remote repository with authentication"""
+        print("Pulling Latest Updates (if any)")
+        try:
+            self.repo.remotes[remote_name].pull(ref_name)
+            return True
+        except GitCommandError as e:
+            print(f"Pulling failed: {e}")
+            return False
+
+    def get_branches(self):
+        """List all branches in the repository"""
+        return [branch.name for branch in self.repo.branches]
+
+    def add_and_commit(self, message=None):
+        """Add and commit changes to the repository."""
+        try:
+            print("Commiting latest draft")
+            # Add all changes
+            self.repo.git.add(all=True)
+            # Commit with the provided message or a default
+            if message is None:
+                commit_message = "Added and committed new content"
+            else:
+                commit_message = message
+            self.repo.git.commit(message=commit_message)
+            return True
+        except GitCommandError as e:
+            print(f"Commit failed: {e}")
+            return False
+
+    def create_copy_commit_push(self, file_path, title, commit_message):
+        # Check if branch exists remotely
+        remote_branches = [
+            ref.name.split("/")[-1] for ref in self.repo.remotes.origin.refs
+        ]
+
+        if title in remote_branches:
+            # Branch exists remotely, checkout and pull
+            self.repo.git.checkout(title)
+            self.pull(ref_name=title)
+        else:
+            # New branch, create from main
+            self.repo.git.checkout("-b", title, "origin/main")
+
+        # Ensure destination directory exists
+        dest_dir = f"{self.repo_path}src/content/"
+        os.makedirs(dest_dir, exist_ok=True)
+
+        # Copy file
+        shutil.copy(f"{file_path}", dest_dir)
+
+        # Commit and push
+        self.add_and_commit(commit_message)
+        self.repo.git.push("--set-upstream", "origin", title)

src/trilium/notes.py

@@ -18,9 +18,13 @@ class TrilumNotes:
             print("Please run get_token and set your token")
         else:
             self.ea = ETAPI(self.server_url, self.token)
+        self.new_notes = None
+        self.note_content = None
 
     def get_token(self):
         ea = ETAPI(self.server_url)
+        if self.tril_pass == None:
+            raise ValueError("Trillium password can not be none")
         token = ea.login(self.tril_pass)
         print(token)
         print("I would recomend you update the env file with this tootsweet!")
@@ -40,10 +44,11 @@ class TrilumNotes:
 
     def get_notes_content(self):
         content_dict = {}
+        if self.new_notes is None:
+            raise ValueError("How did you do this? new_notes is None!")
         for note in self.new_notes['results']:
             content_dict[note['noteId']] = {"title" : f"{note['title']}",
                                             "content" : f"{self._get_content(note['noteId'])}"
                                             }
         self.note_content = content_dict
         return content_dict
-