From 1470f0979a9f0609045c8f56e23383238f0c7e1f Mon Sep 17 00:00:00 2001 From: Andrew Ridgway Date: Sat, 7 Sep 2024 23:24:32 +1000 Subject: [PATCH] initial --- .gitea/workflows/build_push.yml | 53 ++++++++++++++ .gitignore | 4 ++ base.Dockerfile | 11 +++ compose_up_cmd.sh | 7 ++ docker-compose.yml | 40 +++++++++++ flask.Dockerfile | 13 ++++ pyproject.toml | 3 + requirements.txt | 9 +++ src/flask/add_user.py | 29 ++++++++ src/flask/beer_data.py | 116 +++++++++++++++++++++++++++++++ src/flask/chart_test.html | 61 ++++++++++++++++ src/flask/chart_test.py | 17 +++++ src/flask/charts/BeerCharts.py | 49 +++++++++++++ src/flask/charts/__init__.py | 0 src/flask/mongo/__init__.py | 0 src/flask/mongo/build_db.py | 75 ++++++++++++++++++++ src/flask/mongo/get_conn.py | 19 +++++ src/flask/mongo/query_db.py | 55 +++++++++++++++ src/flask/mongo/user_db.py | 63 +++++++++++++++++ src/flask/static/data_plot.html | 61 ++++++++++++++++ src/flask/templates/base.html | 26 +++++++ src/flask/templates/index.html | 37 ++++++++++ src/flask/templates/updater.html | 64 +++++++++++++++++ start_env | 4 ++ 24 files changed, 816 insertions(+) create mode 100644 .gitea/workflows/build_push.yml create mode 100644 .gitignore create mode 100644 base.Dockerfile create mode 100755 compose_up_cmd.sh create mode 100644 docker-compose.yml create mode 100644 flask.Dockerfile create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 src/flask/add_user.py create mode 100644 src/flask/beer_data.py create mode 100644 src/flask/chart_test.html create mode 100644 src/flask/chart_test.py create mode 100644 src/flask/charts/BeerCharts.py create mode 100644 src/flask/charts/__init__.py create mode 100644 src/flask/mongo/__init__.py create mode 100644 src/flask/mongo/build_db.py create mode 100644 src/flask/mongo/get_conn.py create mode 100644 src/flask/mongo/query_db.py create mode 100644 src/flask/mongo/user_db.py create mode 100644 src/flask/static/data_plot.html create mode 100644 src/flask/templates/base.html create mode 100644 src/flask/templates/index.html create mode 100644 src/flask/templates/updater.html create mode 100755 start_env diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml new file mode 100644 index 0000000..38182cf --- /dev/null +++ b/.gitea/workflows/build_push.yml @@ -0,0 +1,53 @@ +name: Build and Push Image +on: [push] + +jobs: + build: + name: Build and push image + runs-on: ubuntu-latest + container: catthehacker/ubuntu:act-latest + if: gitea.ref == 'refs/heads/master' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create Kubeconfig + run: | + mkdir $HOME/.kube + echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: kubernetes + driver-opts: | + namespace=gitea-runner + qemu.install=true + + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: git.aridgwayweb.com + username: armistace + password: ${{ secrets.REG_PASSWORD }} + + - name: Build Base Image + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: base.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + git.aridgwayweb.com/armistace/pool_base_image:latest + + - name: Build and Push + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: flask.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + git.aridgwayweb.com/armistace/pool-data:latest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c770f86 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*__pycache__* +.venv* +.env* +mongo_data/* diff --git a/base.Dockerfile b/base.Dockerfile new file mode 100644 index 0000000..3a367e1 --- /dev/null +++ b/base.Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.10 AS beer_base_image + +WORKDIR /beer_data + +COPY requirements.txt . + +RUN apt-get update -y && apt-get upgrade -y && apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev + +RUN pip install --upgrade pip + +RUN pip --default-timeout=1000 install -r requirements.txt diff --git a/compose_up_cmd.sh b/compose_up_cmd.sh new file mode 100755 index 0000000..0c8cbaa --- /dev/null +++ b/compose_up_cmd.sh @@ -0,0 +1,7 @@ + +docker-compose rm -f +docker system prune -f +docker volume prune -f +docker build -t beer_base_image -f base.Dockerfile . +docker-compose up --remove-orphans --build -d +docker logs -f beer_data_web diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..def3740 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +# Use root/example as user/password credentials +version: '3.1' + +services: + beer_data_web: + container_name: beer_data_web + build: + context: . + dockerfile: flask.Dockerfile + # volumes: + #- ./src/flask:/pool_data/src/flask + ports: + - "80:80" + - "5000:5000" + restart: "unless-stopped" + environment: + MONGO_HOST: mongo + MONGO_USER: root + MONGO_PASS: example + + mongo: + image: mongo + restart: always + ports: + - 27017:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + volumes: + - ./mongo_data:/data/db + + mongo-express: + image: mongo-express + restart: always + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: root + ME_CONFIG_MONGODB_ADMINPASSWORD: example + ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/ diff --git a/flask.Dockerfile b/flask.Dockerfile new file mode 100644 index 0000000..4cdfe2d --- /dev/null +++ b/flask.Dockerfile @@ -0,0 +1,13 @@ +FROM git.aridgwayweb.com/armistace/beer_base_image AS flask + +COPY requirements.txt . + +ADD src /beer_data/src + +ENV FLASK_ENV production +ENV FLASK_DEBUG 1 + + +ENTRYPOINT ["flask", "--app", "/pool_data/src/flask/beer_data", "run", "--host=0.0.0.0"] + +#ENTRYPOINT ["python", "/pool_data/src/flask/pool_data.py"] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a578942 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pyright] +venvPath = "." +venv = ".venv" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aaf30fa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +pymongo[srv] +flask +requests_html +beautifulsoup4 +click +Flask-WTF +bootstrap-flask +waitress +bokeh diff --git a/src/flask/add_user.py b/src/flask/add_user.py new file mode 100644 index 0000000..76fb24d --- /dev/null +++ b/src/flask/add_user.py @@ -0,0 +1,29 @@ +import os +import click + +import mongo.user_db as user_db + +@click.command() +@click.option('--username', type=str, help="Username to be added or updated") +@click.option('--password', type=str, help="Password to be added or updated") + +def main(username, password): + """ + little cli program to update the + user table in mongo + this rightly should eventually be + and admin tool but right now this will dow + """ + user_collection = user_db.user_data() + new_record = { + "username" : f"{username}", + "password" : f"{password}" + } + if user_collection.user_exists(new_record["username"]): + user_collect.update_user(user_collection.existing_record["_id"], new_record["password"]) + else: + user_collection.add_user(new_record["username"], new_record["password"]) + + +if __name__ == "__main__": + main() diff --git a/src/flask/beer_data.py b/src/flask/beer_data.py new file mode 100644 index 0000000..4664592 --- /dev/null +++ b/src/flask/beer_data.py @@ -0,0 +1,116 @@ +from bokeh.core.enums import SpatialUnitsType +import mongo.build_db as pool_database +import mongo.query_db as pool_database_query + +from flask import Flask, render_template, request, jsonify, redirect, session +from flask_wtf import FlaskForm, CSRFProtect +from flask_bootstrap import Bootstrap5 +from wtforms import StringField, SubmitField, DateField, IntegerField, PasswordField, DecimalField, RadioField, TextAreaField +from wtforms.validators import DataRequired, Length, Optional +from waitress import serve +from bokeh.models.layouts import HBox +from bokeh.plotting import column +from charts import BeerCharts +from bokeh.io import output_file, show +from bokeh.layouts import row + +app = Flask(__name__) +app.secret_key = 'testsecret' #this value will change + +bootstrap = Bootstrap5(app) + +csrf = CSRFProtect(app) +# used to configure the bokeh plot for graphs +output_file("/beer_data/src/flask/static/data_plot.html") + +def create_graphs(): + chart = BeerCharts.BeerCharts() + sg = chart.line_chart("SG", "sg", 30) + show(row(column(sg))) + + +class userForm(FlaskForm): + username = StringField("User Name?", validators=[DataRequired()]) + password = PasswordField("Password?") + submit = SubmitField("Letsa GO!") + +class dataForm(FlaskForm): + beer_run_id = IntegerField("Beer Run ID") + beer_run_type = TextAreaField("Beer Type") + Date = DateField("Date:") + sg = IntegerField("SG Reading") + comment = TextAreaField("Any Comments?", validators=[Optional()]) + submit = SubmitField("Write it, Write it REAAAAAAL GOOOD") + +@app.route("/", methods=["GET","POST"]) +def index(): + form = userForm() + if form.validate_on_submit(): + username = form.username.data + password = form.password.data + db = pool_database_query.pool_query() + if db.user_check(username, password): + session['logged_in'] = True + return redirect("/updater") + else: + return render_template("index.html", try_again=True, form=form) + else: + return render_template("index.html", try_again=False, form=form) + +@app.route("/updater", methods=["GET", "POST"]) +def updater(): + if 'logged_in' not in session: + return redirect("/") + create_graphs() + query_db = pool_database_query.pool_query() + query = query_db.get_top(10, "sg") + form = dataForm() + + if form.validate_on_submit(): + database = pool_database.pool_data() + new_record = { + "date": f'{form.Date.data}', + "beer_run_id": f'{form.beer_run_id.data}', + "beer_type": f'{form.beer_run_type.data}', + "sg" : f'{form.sg.data}', + "comment": f'{form.comment.data}' + } + if database.record_exists(new_record["date"], new_record["beer_run_id"]): + for field in database.existing_record: + for new_field in new_record: + if field == new_field: + if database.existing_record[field] != new_record[field]: + database.update_re_record(database.existing_record["_id"], field, new_record[field]) + else: + database.create_re_record(new_record["beer_run_id"], new_record["beer_type"], new_record["sg"], + new_record["date"], new_record["comment"]) + + return render_template("updater.html", list=query, form=form, success=True, updater_name = new_record["test_user"]) + else: + return render_template("updater.html", list=query, form=form, sucess=False) + +@app.route("/update_db", methods=["POST"]) +def pool_data_update(): + database = pool_database.pool_data() + new_record = request.json + if database.record_exists(new_record["date"], new_record["test_user"]): + for field in database.existing_record: + for new_field in new_record: + if field == new_field: + if database.existing_record[field] != new_record[field]: + database.update_re_record(database.existing_record["_id"], field, new_record[field]) + else: + database.create_re_record(new_record["beer_run_id"], new_record["beer_type"], new_record["sg"], + new_record["date"], new_record["comment"]) + + + +@app.route("/pool_top//") +def user_detail(id): + query_db = pool_database_query.pool_query() + query = query_db.get_top(return_number, field) + return jsonify([row.to_json() for row in query]) + +if __name__ == '__main__': + #app.run(host='0.0.0.0') + serve(app, host='0.0.0.0', port=5000, url_scheme='https') diff --git a/src/flask/chart_test.html b/src/flask/chart_test.html new file mode 100644 index 0000000..0ce9862 --- /dev/null +++ b/src/flask/chart_test.html @@ -0,0 +1,61 @@ + + + + + Bokeh Plot + + + + + +
+ + + + + \ No newline at end of file diff --git a/src/flask/chart_test.py b/src/flask/chart_test.py new file mode 100644 index 0000000..d30feaa --- /dev/null +++ b/src/flask/chart_test.py @@ -0,0 +1,17 @@ +from bokeh.models.layouts import HBox +from bokeh.plotting import column +from charts import PoolCharts +from bokeh.io import output_file, show +from bokeh.layouts import row + +output_file("static/data_plot.html") + +chart = PoolCharts.PoolCharts() + +ph = chart.line_chart("Pool PH", "ph", 50) + +total_chlorine = chart.line_chart("Pool Total Chlorine", "total_chlorine", 50) + +free_chlorine = chart.line_chart("Pool Free Chlorine", "free_chlorine", 50) + +show(column(ph, total_chlorine, free_chlorine)) diff --git a/src/flask/charts/BeerCharts.py b/src/flask/charts/BeerCharts.py new file mode 100644 index 0000000..451b927 --- /dev/null +++ b/src/flask/charts/BeerCharts.py @@ -0,0 +1,49 @@ +import sys + +from bokeh.models.widgets import DataCube +from bokeh.plotting import figure + +from mongo.get_conn import db_conn + +class BeerCharts(): + def __init__(self): + self.db = db_conn() + + + def line_chart(self, title, field, limit): + data = self.db.pool_db + data = data.find({}).sort("date", -1).limit(limit) + dates = [] + data_list = [] + date_count = {} + for record in data: + if field in record and record[field] != "None": + if record["date"] not in dates: + print("new date record") + dates.append(record["date"]) + data_list.append(float(record[field])) + print (dates) + print (data_list) + else: + if record["date"] in date_count: + date_count[record["date"]] += 1 + else: + date_count[record["date"]] = 2 + print(len(data_list)) + current_data = data_list[len(data_list) - 1] + data_list[len(data_list) - 1] = (float(record[field]) + current_data) / date_count[record["date"]] + + print(dates) + print(data_list) + + p = figure(x_range=dates, height=250, title=title, + toolbar_location=None, tools="") + + p.line(x=dates, y=data_list) + + p.xgrid.grid_line_color = None + if not data_list: + p.y_range.start = 0 + else: + p.y_range.start = round(min(data_list) - 1) + return p diff --git a/src/flask/charts/__init__.py b/src/flask/charts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/flask/mongo/__init__.py b/src/flask/mongo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/flask/mongo/build_db.py b/src/flask/mongo/build_db.py new file mode 100644 index 0000000..1e70c46 --- /dev/null +++ b/src/flask/mongo/build_db.py @@ -0,0 +1,75 @@ +import string +import secrets + +from mongo.get_conn import db_conn + +class beer_data: + """ + This class will allow us to + interact with our data to interact + just create a pool_data var + """ + def __init__(self): + #db_conn has all the the things + #already created from here + #we can get self.db.real_db etc + self.db = db_conn() + + def record_exists(self, date, test_user): + """ + This function will accept an address + if it find that address in the database it will return True + and set set the existing_record variable of the class to the + queried record + """ + query = { "date" : f"{date}", "test_user": f"{test_user}"} + record = self.db.beer_db.find_one(query) + if record: + self.existing_record = record + return True + else: + return False + + + def create_re_record(self, beer_run_id, beer_type, sg, date, comment=""): + """ + create_re_record creates a whole new record + takes the required 7 inputs + 1. beer_run_id + 2. beer_type + 3. sg + 4. date + 5. comment(optional) + It will autogenerate the id string + this string will be automatically selected + on view in future + """ + key = self.create_id() + insert_record = { + "_id" : f"{key}", + "beer_run_id": f"{beer_run_id}", + "beer_type": f"{beer_type}", + "sg": f"{sg}", + "date": f"{date}", + "comment": f"{comment}" + + } + self.db.beer_db.insert_one(insert_record) + + def update_re_record(self, id, field, value): + """ + update_re_record + will update the requested field to value for the + selected field + """ + #TODO: do we need to make this take more than 1 id? + + query = { "_id": f"{id}"} + update_val = {"$set": {f"{field}": f"{value}" }} + self.db.beer_db.update_one(query, update_val) + + def create_id(self): + alphabet = string.ascii_letters + string.digits + key = ''.join(secrets.choice(alphabet) for _ in range(24)) + return key + diff --git a/src/flask/mongo/get_conn.py b/src/flask/mongo/get_conn.py new file mode 100644 index 0000000..20ad979 --- /dev/null +++ b/src/flask/mongo/get_conn.py @@ -0,0 +1,19 @@ +from pymongo import MongoClient +import os + +class db_conn: + def __init__(self): + self.db_user = os.getenv('MONGO_USER') + self.db_pass = os.getenv('MONGO_PASS') + self.db_host = os.getenv('MONGO_HOST') + self.client = self.get_client() + self.db = self.client['beer_db'] + self.pool_db = self.db['beer_data'] + self.users = self.db['users'] + self.inspections = self.db['inspections_db'] + + def get_client(self): + CONNECTION_STRING = f"mongodb://{self.db_user}:{self.db_pass}@{self.db_host}/beer_db?authSource=admin" + return MongoClient(CONNECTION_STRING) + + diff --git a/src/flask/mongo/query_db.py b/src/flask/mongo/query_db.py new file mode 100644 index 0000000..7537894 --- /dev/null +++ b/src/flask/mongo/query_db.py @@ -0,0 +1,55 @@ +import string +import secrets + +from mongo.get_conn import db_conn + +class pool_query: + """ + This class will allow us to + interact with our data to interact + just create a realestate_data var + """ + def __init__(self): + #db_conn has all the the things + #already created from here + #we can get self.db.real_db etc + self.db = db_conn() + + def record_exists(self, beer_run_id, date): + """ + This function will accept an address + if it find that address in the database it will return True + and set set the existing_record variable of the class to the + queried record + """ + query = {"beer_run_id": f"{beer_run_id}", "date": f"{date}"} + record = self.db.beer_db.find_one(query) + if record: + self.existing_record = record + return True + else: + return False + + def get_top(self, num_limit, value_field): + """ + This function will return the + last n records of the chosen value field + It will take the number of records you want to + return as a parameter + """ + records = self.db.beer_db.find({}, {"beer_run_id": 1, "_id": 0, "date": 1, f"{value_field}": 1}).sort("date", -1).limit(num_limit) + return records + + def user_check(self, username, password): + """ + function to check username and password + back in db + """ + #TODO: this ueses my own quick hack it likley needs to be rewrittent to follow best practice + + query = { "username" : f"{username}", "password" : f"{password}" } + record = self.db.users.find_one(query) + if record: + return True + else: + return False diff --git a/src/flask/mongo/user_db.py b/src/flask/mongo/user_db.py new file mode 100644 index 0000000..8e25970 --- /dev/null +++ b/src/flask/mongo/user_db.py @@ -0,0 +1,63 @@ +import string +import secrets + +from mongo.get_conn import db_conn + + +class user_data: + """ + This class will allow us to + interact with our data to interact + just create a user_data var + """ + def __init__(self): + self.db = db_conn() + + def create_id(self): + alphabet = string.ascii_letters + string.digits + key = ''.join(secrets.choice(alphabet) for _ in range(24)) + return key + + def user_exists(self, username): + """ + This function will accept a username + if it find that user in the database it will return True + and set set the existing_record variable of the class to the + queried record + """ + query = { "username" : f"{username}" } + record = self.db.users.find_one(query) + if record: + self.existing_record = record + return True + else: + return False + + + + def add_user(self, username, password): + """ + add to user table + function accepts the username and password + it will generate a uniqe key + """ + + key = self.create_id() + insert_record = { + "_id" : f"{key}", + "username" : f"{username}", + "password" : f"{password}" + } + self.db.users.insert_one(insert_record) + + def update_user(self, id, password): + """ + update the user record password + requires the id to be updated + """ + + query = { "_id": f"{id}"} + update_val = {"$set": {"password": f"{password}" }} + self.db.users.update_one(query, update_val) + + diff --git a/src/flask/static/data_plot.html b/src/flask/static/data_plot.html new file mode 100644 index 0000000..d5d1fd7 --- /dev/null +++ b/src/flask/static/data_plot.html @@ -0,0 +1,61 @@ + + + + + Bokeh Plot + + + + + +
+ + + + + \ No newline at end of file diff --git a/src/flask/templates/base.html b/src/flask/templates/base.html new file mode 100644 index 0000000..f619e1c --- /dev/null +++ b/src/flask/templates/base.html @@ -0,0 +1,26 @@ + + + + + + + {% block title %} Let there Be Pool Data {% endblock %} + + {{ bootstrap.load_css() }} + + + + + + + {% block content %} {% endblock %} + + + {{ bootstrap.load_js() }} + + diff --git a/src/flask/templates/index.html b/src/flask/templates/index.html new file mode 100644 index 0000000..244597f --- /dev/null +++ b/src/flask/templates/index.html @@ -0,0 +1,37 @@ +{% extends 'base.html' %} +{% from 'bootstrap5/form.html' import render_form %} + +{% block title %} +Login to Pool Data Tracker +{% endblock %} +{% block content %} + + + +
+ {% if try_again %} +

Login Failed Please try Again

+ {% endif %} + +
+
+
+ +

WHOOOOOOO ARE YOU!

+ +

If you know you know

+ + {{ render_form(form) }} + +

{{ message }}

+ +
+
+
+
+ +{% endblock %} diff --git a/src/flask/templates/updater.html b/src/flask/templates/updater.html new file mode 100644 index 0000000..8c8c088 --- /dev/null +++ b/src/flask/templates/updater.html @@ -0,0 +1,64 @@ +{% extends 'base.html' %} +{% from 'bootstrap5/form.html' import render_form %} + +{% block title %} + Data Input +{% endblock %} +{% block content %} + + +
+
+
+ +

Input Data you want to store

+ {% if success %} +

Thank you {{ updater_name | safe }}, Data Updated Successfully

+ {% endif %} + +
+
+ + + + + + +
+ {{ render_form(form) }} + +
+
+ +
+
+
+
+
+
+
+
+{% endblock %} diff --git a/start_env b/start_env new file mode 100755 index 0000000..142b3a6 --- /dev/null +++ b/start_env @@ -0,0 +1,4 @@ +#!/bin/bash + +source .venv/bin/activate +source .env