initial
Some checks failed
Build and Push Image / Build and push image (push) Has been cancelled

This commit is contained in:
Andrew Ridgway 2024-10-29 14:36:29 +10:00
commit ac1975315c
27 changed files with 881 additions and 0 deletions

View File

@ -0,0 +1,53 @@
name: Build and Push Image
on: [push]
jobs:
build:
name: Build and push image
runs-on: ubuntu-latest
container: catthehacker/ubuntu:act-latest
if: gitea.ref == 'refs/heads/master'
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Create Kubeconfig
run: |
mkdir $HOME/.kube
echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: kubernetes
driver-opts: |
namespace=gitea-runner
qemu.install=true
- name: Login to Docker Registry
uses: docker/login-action@v3
with:
registry: git.aridgwayweb.com
username: armistace
password: ${{ secrets.REG_PASSWORD }}
- name: Build Base Image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: base.Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
git.aridgwayweb.com/armistace/beer_base_image:latest
- name: Build and Push
uses: docker/build-push-action@v5
with:
context: .
push: true
file: flask.Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
git.aridgwayweb.com/armistace/beer-data:latest

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*__pycache__*
.venv*
.env*
mongo_data/*

7
README.md Normal file
View File

@ -0,0 +1,7 @@
# BEER SG data storer
This container will store SG data for beer runs in Mongo and then use that data to calculate Alchohol Content of each beer
It's pretty loose and requires the Data Entry operator to know whats happening
Luck for me... That's me!

11
base.Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM python:3.10 AS beer_base_image
WORKDIR /wine_data
COPY requirements.txt .
RUN apt-get update -y && apt-get upgrade -y && apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev
RUN pip install --upgrade pip
RUN pip --default-timeout=1000 install -r requirements.txt

7
change_db.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
# This little ditty replaces beer_data with wine_Data
# TODO: Update this to take commands so I can basically create a template project for this
# it's the third time i've used this as a basis for a project its becoming habit lol
find . \( ! -regex '.*/\..*' \) -type f | xargs sed -i 's/beer_data/wine_data/g'

7
compose_up_cmd.sh Executable file
View File

@ -0,0 +1,7 @@
docker-compose rm -f
docker system prune -f
docker volume prune -f
docker build -t beer_base_image -f base.Dockerfile .
docker-compose up --remove-orphans --build -d
docker logs -f wine_data_web

40
docker-compose.yml Normal file
View File

@ -0,0 +1,40 @@
# Use root/example as user/password credentials
version: '3.1'
services:
wine_data_web:
container_name: wine_data_web
build:
context: .
dockerfile: flask.Dockerfile
# volumes:
#- ./src/flask:/pool_data/src/flask
ports:
- "80:80"
- "5000:5000"
restart: "unless-stopped"
environment:
MONGO_HOST: mongo
MONGO_USER: root
MONGO_PASS: example
mongo:
image: mongo
restart: always
ports:
- 27017:27017
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: example
volumes:
- ./mongo_data:/data/db
mongo-express:
image: mongo-express
restart: always
ports:
- 8081:8081
environment:
ME_CONFIG_MONGODB_ADMINUSERNAME: root
ME_CONFIG_MONGODB_ADMINPASSWORD: example
ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

14
flask.Dockerfile Normal file
View File

@ -0,0 +1,14 @@
FROM git.aridgwayweb.com/armistace/beer_base_image AS flask
#FROM beer_base_image AS flask
COPY requirements.txt .
ADD src /wine_data/src
ENV FLASK_ENV production
ENV FLASK_DEBUG 1
ENTRYPOINT ["flask", "--app", "/wine_data/src/flask/wine_data", "run", "--host=0.0.0.0"]
#ENTRYPOINT ["python", "/pool_data/src/flask/pool_data.py"]

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[tool.pyright]
venvPath = "."
venv = ".venv"

11
requirements.txt Normal file
View File

@ -0,0 +1,11 @@
pymongo[srv]
flask
requests_html
beautifulsoup4
click
Flask-WTF
bootstrap-flask
waitress
bokeh
pandas
duckdb

29
src/flask/add_user.py Normal file
View File

@ -0,0 +1,29 @@
import os
import click
import mongo.user_db as user_db
@click.command()
@click.option('--username', type=str, help="Username to be added or updated")
@click.option('--password', type=str, help="Password to be added or updated")
def main(username, password):
"""
little cli program to update the
user table in mongo
this rightly should eventually be
and admin tool but right now this will dow
"""
user_collection = user_db.user_data()
new_record = {
"username" : f"{username}",
"password" : f"{password}"
}
if user_collection.user_exists(new_record["username"]):
user_collection.update_user(user_collection.existing_record["_id"], new_record["password"])
else:
user_collection.add_user(new_record["username"], new_record["password"])
if __name__ == "__main__":
main()

View File

@ -0,0 +1,52 @@
import sys
from bokeh.models.widgets import DataCube
from bokeh.plotting import figure
from mongo.get_conn import db_conn
class BeerCharts():
def __init__(self):
self.db = db_conn()
def bar_chart(self, items, data):
return 0
def line_chart(self, title, field, limit):
data = self.db.beer_db
data = data.find({}).sort("date", -1).limit(limit)
dates = []
data_list = []
date_count = {}
for record in data:
if field in record and record[field] != "None":
if record["date"] not in dates:
print("new date record")
dates.append(record["date"])
data_list.append(float(record[field]))
print (dates)
print (data_list)
else:
if record["date"] in date_count:
date_count[record["date"]] += 1
else:
date_count[record["date"]] = 2
print(len(data_list))
current_data = data_list[len(data_list) - 1]
data_list[len(data_list) - 1] = (float(record[field]) + current_data) / date_count[record["date"]]
print(dates)
print(data_list)
p = figure(x_range=dates, height=250, title=title,
toolbar_location=None, tools="")
p.line(x=dates, y=data_list)
p.xgrid.grid_line_color = None
if not data_list:
p.y_range.start = 0
else:
p.y_range.start = round(min(data_list) - 1)
return p

View File

View File

View File

@ -0,0 +1,76 @@
import string
import secrets
from mongo.get_conn import db_conn
class wine_data:
"""
This class will allow us to
interact with our data to interact
just create a pool_data var
"""
def __init__(self):
#db_conn has all the the things
#already created from here
#we can get self.db.real_db etc
self.db = db_conn()
def record_exists(self, date, beer_run_id):
"""
This function will accept an address
if it find that address in the database it will return True
and set set the existing_record variable of the class to the
queried record
"""
query = { "date" : f"{date}", "beer_run_id": f"{beer_run_id}"}
record = self.db.beer_db.find_one(query)
if record:
self.existing_record = record
return True
else:
return False
def create_re_record(self, beer_run_id, beer_type, sg, date, final_reading=False, comment=""):
"""
create_re_record creates a whole new record
takes the required 7 inputs
1. beer_run_id
2. beer_type
3. sg
4. date
5. comment(optional)
It will autogenerate the id string
this string will be automatically selected
on view in future
"""
key = self.create_id()
insert_record = {
"_id" : f"{key}",
"beer_run_id": f"{beer_run_id}",
"beer_type": f"{beer_type}",
"sg": f"{sg}",
"date": f"{date}",
"final_reading": f"{final_reading}",
"comment": f"{comment}"
}
self.db.beer_db.insert_one(insert_record)
def update_re_record(self, id, field, value):
"""
update_re_record
will update the requested field to value for the
selected field
"""
#TODO: do we need to make this take more than 1 id?
query = { "_id": f"{id}"}
update_val = {"$set": {f"{field}": f"{value}" }}
self.db.beer_db.update_one(query, update_val)
def create_id(self):
alphabet = string.ascii_letters + string.digits
key = ''.join(secrets.choice(alphabet) for _ in range(24))
return key

View File

@ -0,0 +1,19 @@
from pymongo import MongoClient
import os
class db_conn:
def __init__(self):
self.db_user = os.getenv('MONGO_USER')
self.db_pass = os.getenv('MONGO_PASS')
self.db_host = os.getenv('MONGO_HOST')
self.client = self.get_client()
self.db = self.client['beer_db']
self.beer_db = self.db['wine_data']
self.users = self.db['users']
self.inspections = self.db['inspections_db']
def get_client(self):
CONNECTION_STRING = f"mongodb://{self.db_user}:{self.db_pass}@{self.db_host}/beer_db?authSource=admin"
return MongoClient(CONNECTION_STRING)

View File

@ -0,0 +1,55 @@
import string
import secrets
from mongo.get_conn import db_conn
class pool_query:
"""
This class will allow us to
interact with our data to interact
just create a realestate_data var
"""
def __init__(self):
#db_conn has all the the things
#already created from here
#we can get self.db.real_db etc
self.db = db_conn()
def record_exists(self, beer_run_id, date):
"""
This function will accept an address
if it find that address in the database it will return True
and set set the existing_record variable of the class to the
queried record
"""
query = {"beer_run_id": f"{beer_run_id}", "date": f"{date}"}
record = self.db.beer_db.find_one(query)
if record:
self.existing_record = record
return True
else:
return False
def get_top(self, num_limit, value_field):
"""
This function will return the
last n records of the chosen value field
It will take the number of records you want to
return as a parameter
"""
records = self.db.beer_db.find({}, {"beer_run_id": 1, "_id": 0, "date": 1, f"{value_field}": 1}).sort("date", -1).limit(num_limit)
return records
def user_check(self, username, password):
"""
function to check username and password
back in db
"""
#TODO: this ueses my own quick hack it likley needs to be rewrittent to follow best practice
query = { "username" : f"{username}", "password" : f"{password}" }
record = self.db.users.find_one(query)
if record:
return True
else:
return False

View File

@ -0,0 +1,63 @@
import string
import secrets
from mongo.get_conn import db_conn
class user_data:
"""
This class will allow us to
interact with our data to interact
just create a user_data var
"""
def __init__(self):
self.db = db_conn()
def create_id(self):
alphabet = string.ascii_letters + string.digits
key = ''.join(secrets.choice(alphabet) for _ in range(24))
return key
def user_exists(self, username):
"""
This function will accept a username
if it find that user in the database it will return True
and set set the existing_record variable of the class to the
queried record
"""
query = { "username" : f"{username}" }
record = self.db.users.find_one(query)
if record:
self.existing_record = record
return True
else:
return False
def add_user(self, username, password):
"""
add to user table
function accepts the username and password
it will generate a uniqe key
"""
key = self.create_id()
insert_record = {
"_id" : f"{key}",
"username" : f"{username}",
"password" : f"{password}"
}
self.db.users.insert_one(insert_record)
def update_user(self, id, password):
"""
update the user record password
requires the id to be updated
"""
query = { "_id": f"{id}"}
update_val = {"$set": {"password": f"{password}" }}
self.db.users.update_one(query, update_val)

File diff suppressed because one or more lines are too long

View File

View File

@ -0,0 +1,90 @@
import sys
from mongo.get_conn import db_conn
import pandas as pd
import duckdb
class TableBuilder():
def __init__(self) -> None:
self.db = db_conn()
data = self.db.beer_db
data = data.find({}).sort("date", -1)
df_dict = {}
df_dict["beer_run_id"]=[]
df_dict["sg"] = []
df_dict["date"] = []
df_dict["final_reading"] = []
for record in data:
df_dict["beer_run_id"].append(record["beer_run_id"])
df_dict["sg"].append(record["sg"])
df_dict["date"].append(record["date"])
df_dict["final_reading"].append(record["final_reading"])
self.df_dict = df_dict
self.df = pd.DataFrame(data=self.df_dict)
def done_runs_build(self, limit=10) -> pd.DataFrame:
df = self.df
sql = f"""
SELECT x.beer_run_id as beer_run_id,
max(sg) as max,
min(sg) as min,
y.date as final_reading_date
FROM df x
JOIN
( SELECT DISTINCT beer_run_id, date
FROM df
WHERE final_reading = 'True'
) y ON x.beer_run_id = y.beer_run_id
GROUP BY x.beer_run_id, y.date
ORDER BY x.beer_run_id desc
LIMIT {limit}
"""
df_sum = duckdb.sql(sql).df()
sql = f"""
SELECT x.beer_run_id as "Beer Run",
x.max as "Max",
x.min as "Min",
ROUND(((CAST (max AS INTEGER) - CAST(min AS INTEGER)) / 7.36) + 0.5, 2) AS "Alcohol Prediction +/- 0.5",
cast(final_reading_date as DATE) + INTERVAL 14 DAY as "Ready Date"
FROM df_sum x
"""
df_calc = duckdb.sql(sql).df()
return df_calc
def current_runs_build(self, limit=10) -> pd.DataFrame:
df = self.df
sql = f"""
SELECT x.beer_run_id as beer_run_id,
max(sg) as max,
min(sg) as min,
min(cast(x.date as DATE)) as first_reading_date
FROM df x
LEFT JOIN
( SELECT DISTINCT beer_run_id
FROM df
WHERE final_reading = 'True'
) y ON x.beer_run_id = y.beer_run_id
WHERE y.beer_run_id is null
GROUP BY x.beer_run_id
ORDER BY x.beer_run_id desc
LIMIT {limit}
"""
df_sum = duckdb.sql(sql).df()
sql = f"""
SELECT x.beer_run_id as "Beer Run",
x.max as "Max",
x.min as "Min",
ROUND(((CAST (max AS INTEGER) - 1012) / 7.36) + 0.5, 2) AS "Alcohol Prediction (1012) +/- 0.5",
first_reading_date + INTERVAL 7 DAY as "Earliest Bottling/Kegging Date"
FROM df_sum x
"""
df_calc = duckdb.sql(sql).df()
return df_calc

View File

@ -0,0 +1,26 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>{% block title %} Let there Be Pool Data {% endblock %}</title>
{{ bootstrap.load_css() }}
<style>
body {
background: #e8f1f9;
}
</style>
</head>
<body>
<!-- this is a base template using Bootstrap-Flask
https://bootstrap-flask.readthedocs.io/ -->
{% block content %} {% endblock %}
<!-- you can delete the next line if you're not using any Bootstrap JS -->
{{ bootstrap.load_js() }}
</body>
</html>

View File

@ -0,0 +1,37 @@
{% extends 'base.html' %}
{% from 'bootstrap5/form.html' import render_form %}
{% block title %}
Login to Pool Data Tracker
{% endblock %}
{% block content %}
<!--
TIPS about using Bootstrap-Flask:
https://github.com/helloflask/bootstrap-flask
https://bootstrap-flask.readthedocs.io/
-->
<div class="log-form">
{% if try_again %}
<h4>Login Failed Please try Again</h4>
{% endif %}
<div class="container">
<div class="row">
<div class="col-md-10 col-lg-8 mx-lg-auto mx-md-auto">
<h1 class="pt-5 pb-2">WHOOOOOOO ARE YOU!</h1>
<p class="lead">If you know you know</p>
{{ render_form(form) }}
<p class="pt-5"><strong>{{ message }}</strong></p>
</div>
</div>
</div>
</div><!--end log form -->
{% endblock %}

View File

@ -0,0 +1,67 @@
{% extends 'base.html' %}
{% from 'bootstrap5/form.html' import render_form %}
{% block title %}
Data Input
{% endblock %}
{% block content %}
<!--
TIPS about using Bootstrap-Flask:
https://github.com/helloflask/bootstrap-flask
https://bootstrap-flask.readthedocs.io/
-->
<div class="container">
<div class="row">
<div class="col-md-10 col-lg-8 mx-lg-auto mx-md-auto">
<h1 class="pt-5 pb-2">Input Data you want to store</h1>
{% if success %}
<h3 class="pt-3 pb02">Thank you {{ updater_name | safe }}, Data Updated Successfully</h1>
{% endif %}
<div class="row">
<div class="col-md-6">
<table border=0>
<tr>
<td>
{{ render_form(form) }}
</td>
<!-- <td>
<div class="container">
<div class="col-md-10 col-lg-8 mx-lg-auto mx-md-auto">
<table border = 0>
{% for row in list %}
<table border=0>
{% for key, value in row.items() %}
<tr>
<th> {{ key }} </th>
<td> {{ value }} </td>
</tr>
{% endfor %}
</table>
{% endfor %}
</table>
</div>
</div>
</td> -->
<!--<iframe style="width: 100vw; height: 40vh;" src="static/data_plot.html" frameborder="0" allowfullscreen>
<!--<iframe style="width: 100vw;height: 80vh;position: relative;" src="static/data_plot.html" frameborder="0" allowfullscreen>-->
</tr>
</table>
</div>
<div class="col-md-6">
<h3>Current Runs</h3>
{{ current_data | safe }}
<h3>Previous Runs</h3>
{{ wine_data | safe }}
</div>
</div>
</div>
</div>
</div>
{% endblock %}

5
src/flask/test.py Normal file
View File

@ -0,0 +1,5 @@
import table.table_builder as table_builder
test = table_builder.TableBuilder()
print(test.table_build())

140
src/flask/wine_data.py Normal file
View File

@ -0,0 +1,140 @@
from bokeh.core.enums import SpatialUnitsType
import mongo.build_db as wine_database
import mongo.query_db as wine_database_query
from table import table_builder
from flask import Flask, render_template, request, jsonify, redirect, session
from flask_wtf import FlaskForm, CSRFProtect
from flask_bootstrap import Bootstrap5
from wtforms import StringField, SubmitField, DateField, IntegerField, PasswordField, DecimalField, RadioField, TextAreaField, BooleanField
from wtforms.validators import DataRequired, Length, Optional
from waitress import serve
from bokeh.models.layouts import HBox
from bokeh.plotting import column
from charts import BeerCharts
from bokeh.io import output_file, show
from bokeh.layouts import row
app = Flask(__name__)
app.secret_key = 'testsecret' #this value will change
bootstrap = Bootstrap5(app)
csrf = CSRFProtect(app)
# used to configure the bokeh plot for graphs
output_file("/wine_data/src/flask/static/data_plot.html")
def create_graphs():
chart = BeerCharts.BeerCharts()
sg = chart.line_chart("SG", "sg", 30)
show(column(sg))
def data_frame_to_table(df) -> str:
tr_replace_string = '<tr align="center" style="border-bottom:1pt solid black;">'
return_html = df.to_html(col_space='75px', index=False,
justify='center', border=3).replace('<tr>', tr_replace_string)
return return_html
class userForm(FlaskForm):
username = StringField("User Name?", validators=[DataRequired()])
password = PasswordField("Password?")
submit = SubmitField("Letsa GO!")
class dataForm(FlaskForm):
beer_run_id = IntegerField("Beer Run ID")
beer_run_type = TextAreaField("Beer Type")
Date = DateField("Date:")
sg = IntegerField("SG Reading")
final_run = BooleanField("Final Reading?")
comment = TextAreaField("Any Comments?", validators=[Optional()])
submit = SubmitField("Write it, Write it REAAAAAAL GOOOD")
@app.route("/", methods=["GET","POST"])
def index():
form = userForm()
if form.validate_on_submit():
username = form.username.data
password = form.password.data
db = wine_database_query.pool_query()
if db.user_check(username, password):
session['logged_in'] = True
return redirect("/updater")
else:
return render_template("index.html", try_again=True, form=form)
else:
return render_template("index.html", try_again=False, form=form)
@app.route("/updater", methods=["GET", "POST"])
def updater():
if 'logged_in' not in session:
return redirect("/")
table_Data = table_builder.TableBuilder()
predicted_alc_table = table_Data.done_runs_build()
current_runs_table = table_Data.current_runs_build()
beer_html = data_frame_to_table(predicted_alc_table)
current_html = data_frame_to_table(current_runs_table)
query_db = wine_database_query.pool_query()
query = query_db.get_top(10, "sg")
form = dataForm()
if form.validate_on_submit():
database = wine_database.wine_data()
new_record = {
"date": f'{form.Date.data}',
"beer_run_id": f'{form.beer_run_id.data}',
"beer_type": f'{form.beer_run_type.data}',
"sg" : f'{form.sg.data}',
"final_reading" : F'{form.final_run.data}',
"comment": f'{form.comment.data}'
}
if database.record_exists(new_record["date"], new_record["beer_run_id"]):
for field in database.existing_record:
for new_field in new_record:
if field == new_field:
if database.existing_record[field] != new_record[field]:
database.update_re_record(database.existing_record["_id"], field, new_record[field])
else:
if new_record["final_reading"] == "True":
final_run_value = True
else:
final_run_value = False
database.create_re_record(new_record["beer_run_id"], new_record["beer_type"], new_record["sg"],
new_record["date"], final_run_value, new_record["comment"])
return render_template("updater.html", wine_data = beer_html, current_data = current_html
, list=query, form=form, success=True, updater_name = "Saucy Beer Maker")
else:
return render_template("updater.html", wine_data = beer_html, current_data = current_html
, list=query, form=form, sucess=False)
# @app.route("/update_db", methods=["POST"])
# def wine_data_update():
# database = wine_database.wine_data()
# new_record = request.json
# if database.record_exists(new_record["date"], new_record["test_user"]):
# for field in database.existing_record:
# for new_field in new_record:
# if field == new_field:
# if database.existing_record[field] != new_record[field]:
# database.update_re_record(database.existing_record["_id"], field, new_record[field])
# else:
# database.create_re_record(new_record["beer_run_id"], new_record["beer_type"], new_record["sg"],
# new_record["date"], new_record["comment"])
# @app.route("/pool_top/<int:return_number>/<string:field>")
# def user_detail(id):
# query_db = wine_database_query.pool_query()
# query = query_db.get_top(return_number, field)
# return jsonify([row.to_json() for row in query])
if __name__ == '__main__':
#app.run(host='0.0.0.0')
serve(app, host='0.0.0.0', port=5000, url_scheme='https')

4
start_env Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
source .venv/bin/activate
source .env