initial
Some checks failed
Build and Push Image / Build and push image (push) Has been cancelled

This commit is contained in:
Andrew Ridgway 2024-09-07 23:24:32 +10:00
commit 1470f0979a
24 changed files with 816 additions and 0 deletions

View File

@ -0,0 +1,53 @@
name: Build and Push Image
on: [push]
jobs:
build:
name: Build and push image
runs-on: ubuntu-latest
container: catthehacker/ubuntu:act-latest
if: gitea.ref == 'refs/heads/master'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Create Kubeconfig
run: |
mkdir $HOME/.kube
echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: kubernetes
driver-opts: |
namespace=gitea-runner
qemu.install=true
- name: Login to Docker Registry
uses: docker/login-action@v3
with:
registry: git.aridgwayweb.com
username: armistace
password: ${{ secrets.REG_PASSWORD }}
- name: Build Base Image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: base.Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
git.aridgwayweb.com/armistace/pool_base_image:latest
- name: Build and Push
uses: docker/build-push-action@v5
with:
context: .
push: true
file: flask.Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
git.aridgwayweb.com/armistace/pool-data:latest

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*__pycache__*
.venv*
.env*
mongo_data/*

11
base.Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM python:3.10 AS beer_base_image
WORKDIR /beer_data
COPY requirements.txt .
RUN apt-get update -y && apt-get upgrade -y && apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev
RUN pip install --upgrade pip
RUN pip --default-timeout=1000 install -r requirements.txt

7
compose_up_cmd.sh Executable file
View File

@ -0,0 +1,7 @@
docker-compose rm -f
docker system prune -f
docker volume prune -f
docker build -t beer_base_image -f base.Dockerfile .
docker-compose up --remove-orphans --build -d
docker logs -f beer_data_web

40
docker-compose.yml Normal file
View File

@ -0,0 +1,40 @@
# Use root/example as user/password credentials
version: '3.1'
services:
beer_data_web:
container_name: beer_data_web
build:
context: .
dockerfile: flask.Dockerfile
# volumes:
#- ./src/flask:/pool_data/src/flask
ports:
- "80:80"
- "5000:5000"
restart: "unless-stopped"
environment:
MONGO_HOST: mongo
MONGO_USER: root
MONGO_PASS: example
mongo:
image: mongo
restart: always
ports:
- 27017:27017
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: example
volumes:
- ./mongo_data:/data/db
mongo-express:
image: mongo-express
restart: always
ports:
- 8081:8081
environment:
ME_CONFIG_MONGODB_ADMINUSERNAME: root
ME_CONFIG_MONGODB_ADMINPASSWORD: example
ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

13
flask.Dockerfile Normal file
View File

@ -0,0 +1,13 @@
FROM git.aridgwayweb.com/armistace/beer_base_image AS flask
COPY requirements.txt .
ADD src /beer_data/src
ENV FLASK_ENV production
ENV FLASK_DEBUG 1
ENTRYPOINT ["flask", "--app", "/pool_data/src/flask/beer_data", "run", "--host=0.0.0.0"]
#ENTRYPOINT ["python", "/pool_data/src/flask/pool_data.py"]

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[tool.pyright]
venvPath = "."
venv = ".venv"

9
requirements.txt Normal file
View File

@ -0,0 +1,9 @@
pymongo[srv]
flask
requests_html
beautifulsoup4
click
Flask-WTF
bootstrap-flask
waitress
bokeh

29
src/flask/add_user.py Normal file
View File

@ -0,0 +1,29 @@
import os
import click
import mongo.user_db as user_db
@click.command()
@click.option('--username', type=str, help="Username to be added or updated")
@click.option('--password', type=str, help="Password to be added or updated")
def main(username, password):
"""
little cli program to update the
user table in mongo
this rightly should eventually be
and admin tool but right now this will dow
"""
user_collection = user_db.user_data()
new_record = {
"username" : f"{username}",
"password" : f"{password}"
}
if user_collection.user_exists(new_record["username"]):
user_collect.update_user(user_collection.existing_record["_id"], new_record["password"])
else:
user_collection.add_user(new_record["username"], new_record["password"])
if __name__ == "__main__":
main()

116
src/flask/beer_data.py Normal file
View File

@ -0,0 +1,116 @@
from bokeh.core.enums import SpatialUnitsType
import mongo.build_db as pool_database
import mongo.query_db as pool_database_query
from flask import Flask, render_template, request, jsonify, redirect, session
from flask_wtf import FlaskForm, CSRFProtect
from flask_bootstrap import Bootstrap5
from wtforms import StringField, SubmitField, DateField, IntegerField, PasswordField, DecimalField, RadioField, TextAreaField
from wtforms.validators import DataRequired, Length, Optional
from waitress import serve
from bokeh.models.layouts import HBox
from bokeh.plotting import column
from charts import BeerCharts
from bokeh.io import output_file, show
from bokeh.layouts import row
app = Flask(__name__)
app.secret_key = 'testsecret' #this value will change
bootstrap = Bootstrap5(app)
csrf = CSRFProtect(app)
# used to configure the bokeh plot for graphs
output_file("/beer_data/src/flask/static/data_plot.html")
def create_graphs():
chart = BeerCharts.BeerCharts()
sg = chart.line_chart("SG", "sg", 30)
show(row(column(sg)))
class userForm(FlaskForm):
username = StringField("User Name?", validators=[DataRequired()])
password = PasswordField("Password?")
submit = SubmitField("Letsa GO!")
class dataForm(FlaskForm):
beer_run_id = IntegerField("Beer Run ID")
beer_run_type = TextAreaField("Beer Type")
Date = DateField("Date:")
sg = IntegerField("SG Reading")
comment = TextAreaField("Any Comments?", validators=[Optional()])
submit = SubmitField("Write it, Write it REAAAAAAL GOOOD")
@app.route("/", methods=["GET","POST"])
def index():
form = userForm()
if form.validate_on_submit():
username = form.username.data
password = form.password.data
db = pool_database_query.pool_query()
if db.user_check(username, password):
session['logged_in'] = True
return redirect("/updater")
else:
return render_template("index.html", try_again=True, form=form)
else:
return render_template("index.html", try_again=False, form=form)
@app.route("/updater", methods=["GET", "POST"])
def updater():
if 'logged_in' not in session:
return redirect("/")
create_graphs()
query_db = pool_database_query.pool_query()
query = query_db.get_top(10, "sg")
form = dataForm()
if form.validate_on_submit():
database = pool_database.pool_data()
new_record = {
"date": f'{form.Date.data}',
"beer_run_id": f'{form.beer_run_id.data}',
"beer_type": f'{form.beer_run_type.data}',
"sg" : f'{form.sg.data}',
"comment": f'{form.comment.data}'
}
if database.record_exists(new_record["date"], new_record["beer_run_id"]):
for field in database.existing_record:
for new_field in new_record:
if field == new_field:
if database.existing_record[field] != new_record[field]:
database.update_re_record(database.existing_record["_id"], field, new_record[field])
else:
database.create_re_record(new_record["beer_run_id"], new_record["beer_type"], new_record["sg"],
new_record["date"], new_record["comment"])
return render_template("updater.html", list=query, form=form, success=True, updater_name = new_record["test_user"])
else:
return render_template("updater.html", list=query, form=form, sucess=False)
@app.route("/update_db", methods=["POST"])
def pool_data_update():
database = pool_database.pool_data()
new_record = request.json
if database.record_exists(new_record["date"], new_record["test_user"]):
for field in database.existing_record:
for new_field in new_record:
if field == new_field:
if database.existing_record[field] != new_record[field]:
database.update_re_record(database.existing_record["_id"], field, new_record[field])
else:
database.create_re_record(new_record["beer_run_id"], new_record["beer_type"], new_record["sg"],
new_record["date"], new_record["comment"])
@app.route("/pool_top/<int:return_number>/<string:field>")
def user_detail(id):
query_db = pool_database_query.pool_query()
query = query_db.get_top(return_number, field)
return jsonify([row.to_json() for row in query])
if __name__ == '__main__':
#app.run(host='0.0.0.0')
serve(app, host='0.0.0.0', port=5000, url_scheme='https')

61
src/flask/chart_test.html Normal file
View File

@ -0,0 +1,61 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Bokeh Plot</title>
<style>
html, body {
box-sizing: border-box;
display: flow-root;
height: 100%;
margin: 0;
padding: 0;
}
</style>
<script type="text/javascript" src="https://cdn.bokeh.org/bokeh/release/bokeh-3.5.0.min.js"></script>
<script type="text/javascript">
Bokeh.set_log_level("info");
</script>
</head>
<body>
<div id="bb5c0e1d-dc95-4d96-9cbc-a29ed2eade5c" data-root-id="p1001" style="display: contents;"></div>
<script type="application/json" id="af8187ed-c6d3-40c3-84a3-bbe58799b466">
{"00bec54e-f52e-4b7f-b2b6-8e6e4eb73c82":{"version":"3.5.0","title":"Bokeh Application","roots":[{"type":"object","name":"Figure","id":"p1001","attributes":{"height":250,"x_range":{"type":"object","name":"FactorRange","id":"p1011","attributes":{"factors":["2024-03-27","2024-03-25","2024-03-15","2024-03-13"]}},"y_range":{"type":"object","name":"DataRange1d","id":"p1003","attributes":{"start":6}},"x_scale":{"type":"object","name":"CategoricalScale","id":"p1012"},"y_scale":{"type":"object","name":"LinearScale","id":"p1013"},"title":{"type":"object","name":"Title","id":"p1004","attributes":{"text":"Pool data"}},"renderers":[{"type":"object","name":"GlyphRenderer","id":"p1030","attributes":{"data_source":{"type":"object","name":"ColumnDataSource","id":"p1024","attributes":{"selected":{"type":"object","name":"Selection","id":"p1025","attributes":{"indices":[],"line_indices":[]}},"selection_policy":{"type":"object","name":"UnionRenderers","id":"p1026"},"data":{"type":"map","entries":[["x",["2024-03-27","2024-03-25","2024-03-15","2024-03-13"]],["y",[7.25,7.5,7.6,7.0]]]}}},"view":{"type":"object","name":"CDSView","id":"p1031","attributes":{"filter":{"type":"object","name":"AllIndices","id":"p1032"}}},"glyph":{"type":"object","name":"Line","id":"p1027","attributes":{"x":{"type":"field","field":"x"},"y":{"type":"field","field":"y"},"line_color":"#1f77b4"}},"nonselection_glyph":{"type":"object","name":"Line","id":"p1028","attributes":{"x":{"type":"field","field":"x"},"y":{"type":"field","field":"y"},"line_color":"#1f77b4","line_alpha":0.1}},"muted_glyph":{"type":"object","name":"Line","id":"p1029","attributes":{"x":{"type":"field","field":"x"},"y":{"type":"field","field":"y"},"line_color":"#1f77b4","line_alpha":0.2}}}}],"toolbar":{"type":"object","name":"Toolbar","id":"p1010"},"toolbar_location":null,"left":[{"type":"object","name":"LinearAxis","id":"p1019","attributes":{"ticker":{"type":"object","name":"BasicTicker","id":"p1020","attributes":{"mantissas":[1,2,5]}},"formatter":{"type":"object","name":"BasicTickFormatter","id":"p1021"},"major_label_policy":{"type":"object","name":"AllLabels","id":"p1022"}}}],"below":[{"type":"object","name":"CategoricalAxis","id":"p1014","attributes":{"ticker":{"type":"object","name":"CategoricalTicker","id":"p1015"},"formatter":{"type":"object","name":"CategoricalTickFormatter","id":"p1016"},"major_label_policy":{"type":"object","name":"AllLabels","id":"p1017"}}}],"center":[{"type":"object","name":"Grid","id":"p1018","attributes":{"axis":{"id":"p1014"},"grid_line_color":null}},{"type":"object","name":"Grid","id":"p1023","attributes":{"dimension":1,"axis":{"id":"p1019"}}}]}}]}}
</script>
<script type="text/javascript">
(function() {
const fn = function() {
Bokeh.safely(function() {
(function(root) {
function embed_document(root) {
const docs_json = document.getElementById('af8187ed-c6d3-40c3-84a3-bbe58799b466').textContent;
const render_items = [{"docid":"00bec54e-f52e-4b7f-b2b6-8e6e4eb73c82","roots":{"p1001":"bb5c0e1d-dc95-4d96-9cbc-a29ed2eade5c"},"root_ids":["p1001"]}];
root.Bokeh.embed.embed_items(docs_json, render_items);
}
if (root.Bokeh !== undefined) {
embed_document(root);
} else {
let attempts = 0;
const timer = setInterval(function(root) {
if (root.Bokeh !== undefined) {
clearInterval(timer);
embed_document(root);
} else {
attempts++;
if (attempts > 100) {
clearInterval(timer);
console.log("Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing");
}
}
}, 10, root)
}
})(window);
});
};
if (document.readyState != "loading") fn();
else document.addEventListener("DOMContentLoaded", fn);
})();
</script>
</body>
</html>

17
src/flask/chart_test.py Normal file
View File

@ -0,0 +1,17 @@
from bokeh.models.layouts import HBox
from bokeh.plotting import column
from charts import PoolCharts
from bokeh.io import output_file, show
from bokeh.layouts import row
output_file("static/data_plot.html")
chart = PoolCharts.PoolCharts()
ph = chart.line_chart("Pool PH", "ph", 50)
total_chlorine = chart.line_chart("Pool Total Chlorine", "total_chlorine", 50)
free_chlorine = chart.line_chart("Pool Free Chlorine", "free_chlorine", 50)
show(column(ph, total_chlorine, free_chlorine))

View File

@ -0,0 +1,49 @@
import sys
from bokeh.models.widgets import DataCube
from bokeh.plotting import figure
from mongo.get_conn import db_conn
class BeerCharts():
def __init__(self):
self.db = db_conn()
def line_chart(self, title, field, limit):
data = self.db.pool_db
data = data.find({}).sort("date", -1).limit(limit)
dates = []
data_list = []
date_count = {}
for record in data:
if field in record and record[field] != "None":
if record["date"] not in dates:
print("new date record")
dates.append(record["date"])
data_list.append(float(record[field]))
print (dates)
print (data_list)
else:
if record["date"] in date_count:
date_count[record["date"]] += 1
else:
date_count[record["date"]] = 2
print(len(data_list))
current_data = data_list[len(data_list) - 1]
data_list[len(data_list) - 1] = (float(record[field]) + current_data) / date_count[record["date"]]
print(dates)
print(data_list)
p = figure(x_range=dates, height=250, title=title,
toolbar_location=None, tools="")
p.line(x=dates, y=data_list)
p.xgrid.grid_line_color = None
if not data_list:
p.y_range.start = 0
else:
p.y_range.start = round(min(data_list) - 1)
return p

View File

View File

View File

@ -0,0 +1,75 @@
import string
import secrets
from mongo.get_conn import db_conn
class beer_data:
"""
This class will allow us to
interact with our data to interact
just create a pool_data var
"""
def __init__(self):
#db_conn has all the the things
#already created from here
#we can get self.db.real_db etc
self.db = db_conn()
def record_exists(self, date, test_user):
"""
This function will accept an address
if it find that address in the database it will return True
and set set the existing_record variable of the class to the
queried record
"""
query = { "date" : f"{date}", "test_user": f"{test_user}"}
record = self.db.beer_db.find_one(query)
if record:
self.existing_record = record
return True
else:
return False
def create_re_record(self, beer_run_id, beer_type, sg, date, comment=""):
"""
create_re_record creates a whole new record
takes the required 7 inputs
1. beer_run_id
2. beer_type
3. sg
4. date
5. comment(optional)
It will autogenerate the id string
this string will be automatically selected
on view in future
"""
key = self.create_id()
insert_record = {
"_id" : f"{key}",
"beer_run_id": f"{beer_run_id}",
"beer_type": f"{beer_type}",
"sg": f"{sg}",
"date": f"{date}",
"comment": f"{comment}"
}
self.db.beer_db.insert_one(insert_record)
def update_re_record(self, id, field, value):
"""
update_re_record
will update the requested field to value for the
selected field
"""
#TODO: do we need to make this take more than 1 id?
query = { "_id": f"{id}"}
update_val = {"$set": {f"{field}": f"{value}" }}
self.db.beer_db.update_one(query, update_val)
def create_id(self):
alphabet = string.ascii_letters + string.digits
key = ''.join(secrets.choice(alphabet) for _ in range(24))
return key

View File

@ -0,0 +1,19 @@
from pymongo import MongoClient
import os
class db_conn:
def __init__(self):
self.db_user = os.getenv('MONGO_USER')
self.db_pass = os.getenv('MONGO_PASS')
self.db_host = os.getenv('MONGO_HOST')
self.client = self.get_client()
self.db = self.client['beer_db']
self.pool_db = self.db['beer_data']
self.users = self.db['users']
self.inspections = self.db['inspections_db']
def get_client(self):
CONNECTION_STRING = f"mongodb://{self.db_user}:{self.db_pass}@{self.db_host}/beer_db?authSource=admin"
return MongoClient(CONNECTION_STRING)

View File

@ -0,0 +1,55 @@
import string
import secrets
from mongo.get_conn import db_conn
class pool_query:
"""
This class will allow us to
interact with our data to interact
just create a realestate_data var
"""
def __init__(self):
#db_conn has all the the things
#already created from here
#we can get self.db.real_db etc
self.db = db_conn()
def record_exists(self, beer_run_id, date):
"""
This function will accept an address
if it find that address in the database it will return True
and set set the existing_record variable of the class to the
queried record
"""
query = {"beer_run_id": f"{beer_run_id}", "date": f"{date}"}
record = self.db.beer_db.find_one(query)
if record:
self.existing_record = record
return True
else:
return False
def get_top(self, num_limit, value_field):
"""
This function will return the
last n records of the chosen value field
It will take the number of records you want to
return as a parameter
"""
records = self.db.beer_db.find({}, {"beer_run_id": 1, "_id": 0, "date": 1, f"{value_field}": 1}).sort("date", -1).limit(num_limit)
return records
def user_check(self, username, password):
"""
function to check username and password
back in db
"""
#TODO: this ueses my own quick hack it likley needs to be rewrittent to follow best practice
query = { "username" : f"{username}", "password" : f"{password}" }
record = self.db.users.find_one(query)
if record:
return True
else:
return False

View File

@ -0,0 +1,63 @@
import string
import secrets
from mongo.get_conn import db_conn
class user_data:
"""
This class will allow us to
interact with our data to interact
just create a user_data var
"""
def __init__(self):
self.db = db_conn()
def create_id(self):
alphabet = string.ascii_letters + string.digits
key = ''.join(secrets.choice(alphabet) for _ in range(24))
return key
def user_exists(self, username):
"""
This function will accept a username
if it find that user in the database it will return True
and set set the existing_record variable of the class to the
queried record
"""
query = { "username" : f"{username}" }
record = self.db.users.find_one(query)
if record:
self.existing_record = record
return True
else:
return False
def add_user(self, username, password):
"""
add to user table
function accepts the username and password
it will generate a uniqe key
"""
key = self.create_id()
insert_record = {
"_id" : f"{key}",
"username" : f"{username}",
"password" : f"{password}"
}
self.db.users.insert_one(insert_record)
def update_user(self, id, password):
"""
update the user record password
requires the id to be updated
"""
query = { "_id": f"{id}"}
update_val = {"$set": {"password": f"{password}" }}
self.db.users.update_one(query, update_val)

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,26 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>{% block title %} Let there Be Pool Data {% endblock %}</title>
{{ bootstrap.load_css() }}
<style>
body {
background: #e8f1f9;
}
</style>
</head>
<body>
<!-- this is a base template using Bootstrap-Flask
https://bootstrap-flask.readthedocs.io/ -->
{% block content %} {% endblock %}
<!-- you can delete the next line if you're not using any Bootstrap JS -->
{{ bootstrap.load_js() }}
</body>
</html>

View File

@ -0,0 +1,37 @@
{% extends 'base.html' %}
{% from 'bootstrap5/form.html' import render_form %}
{% block title %}
Login to Pool Data Tracker
{% endblock %}
{% block content %}
<!--
TIPS about using Bootstrap-Flask:
https://github.com/helloflask/bootstrap-flask
https://bootstrap-flask.readthedocs.io/
-->
<div class="log-form">
{% if try_again %}
<h4>Login Failed Please try Again</h4>
{% endif %}
<div class="container">
<div class="row">
<div class="col-md-10 col-lg-8 mx-lg-auto mx-md-auto">
<h1 class="pt-5 pb-2">WHOOOOOOO ARE YOU!</h1>
<p class="lead">If you know you know</p>
{{ render_form(form) }}
<p class="pt-5"><strong>{{ message }}</strong></p>
</div>
</div>
</div>
</div><!--end log form -->
{% endblock %}

View File

@ -0,0 +1,64 @@
{% extends 'base.html' %}
{% from 'bootstrap5/form.html' import render_form %}
{% block title %}
Data Input
{% endblock %}
{% block content %}
<!--
TIPS about using Bootstrap-Flask:
https://github.com/helloflask/bootstrap-flask
https://bootstrap-flask.readthedocs.io/
-->
<div class="container">
<div class="row">
<div class="col-md-10 col-lg-8 mx-lg-auto mx-md-auto">
<h1 class="pt-5 pb-2">Input Data you want to store</h1>
{% if success %}
<h3 class="pt-3 pb02">Thank you {{ updater_name | safe }}, Data Updated Successfully</h1>
{% endif %}
<div class="row">
<div class="col-md-6">
<table border=0>
<tr>
<td>
{{ render_form(form) }}
</td>
<!-- <td>
<div class="container">
<div class="col-md-10 col-lg-8 mx-lg-auto mx-md-auto">
<table border = 0>
{% for row in list %}
<table border=0>
{% for key, value in row.items() %}
<tr>
<th> {{ key }} </th>
<td> {{ value }} </td>
</tr>
{% endfor %}
</table>
{% endfor %}
</table>
</div>
</div>
</td> -->
<td>
<div class="container">
<div class= "col-xs-12 col-sm-12 col-md-12">
<iframe style="width: 100vw;height: 80vh;position: relative;" src="static/data_plot.html" frameborder="0" allowfullscreen>
</iframe>
</div>
</div>
</td>
</tr>
</table>
</div>
</div>
</div>
</div>
</div>
{% endblock %}

4
start_env Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
source .venv/bin/activate
source .env