From a0c8b0eeab638f10e2ee58a9f2bf89f7acf5fd3d Mon Sep 17 00:00:00 2001 From: Andrew Ridgway Date: Tue, 12 Mar 2024 08:44:13 +1000 Subject: [PATCH] initial transfer of realestate code to pool to start a web service to capture the data --- .gitignore | 4 ++ base.Dockerfile | 12 +++++ compose_up_cmd.sh | 7 +++ docker-compose.yml | 40 +++++++++++++++ flask.Dockerfile | 8 +++ requirements.txt | 5 ++ src/flask/add_user.py | 29 +++++++++++ src/flask/mongo/__init__.py | 0 src/flask/mongo/build_db.py | 83 ++++++++++++++++++++++++++++++++ src/flask/mongo/get_conn.py | 19 ++++++++ src/flask/mongo/query_db.py | 55 +++++++++++++++++++++ src/flask/mongo/user_db.py | 63 ++++++++++++++++++++++++ src/flask/pool_data.py | 77 +++++++++++++++++++++++++++++ src/flask/templates/index.html | 11 +++++ src/flask/templates/updater.html | 64 ++++++++++++++++++++++++ start_env | 4 ++ 16 files changed, 481 insertions(+) create mode 100644 .gitignore create mode 100644 base.Dockerfile create mode 100755 compose_up_cmd.sh create mode 100644 docker-compose.yml create mode 100644 flask.Dockerfile create mode 100644 requirements.txt create mode 100644 src/flask/add_user.py create mode 100644 src/flask/mongo/__init__.py create mode 100644 src/flask/mongo/build_db.py create mode 100644 src/flask/mongo/get_conn.py create mode 100644 src/flask/mongo/query_db.py create mode 100644 src/flask/mongo/user_db.py create mode 100644 src/flask/pool_data.py create mode 100644 src/flask/templates/index.html create mode 100644 src/flask/templates/updater.html create mode 100755 start_env diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56c1acd --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*__pycache__* +.venv* +.env +mongo_data/* diff --git a/base.Dockerfile b/base.Dockerfile new file mode 100644 index 0000000..0569307 --- /dev/null +++ b/base.Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.10 as real_base_image + +WORKDIR /pool_data + +COPY requirements.txt . + +RUN apt-get update -y && apt-get upgrade -y && apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev + +RUN pip install --upgrade pip + +RUN pip --default-timeout=1000 install -r requirements.txt + diff --git a/compose_up_cmd.sh b/compose_up_cmd.sh new file mode 100755 index 0000000..8748503 --- /dev/null +++ b/compose_up_cmd.sh @@ -0,0 +1,7 @@ + +docker-compose rm -f +docker system prune -f +docker volume prune -f +docker build -t real_base_image -f base.Dockerfile . +docker-compose up --remove-orphans --build -d +docker logs -f realtracker_web diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cb4ee19 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +# Use root/example as user/password credentials +version: '3.1' + +services: + realtracker_web: + container_name: realtracker_web + build: + context: . + dockerfile: flask.Dockerfile + volumes: + - ./src/flask:/realestate_tracker/src/flask + ports: + - "80:80" + - "5000:5000" + restart: "unless-stopped" + environment: + MONGO_HOST: mongo + MONGO_USER: root + MONGO_PASS: example + + mongo: + image: mongo + restart: always + ports: + - 27017:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + volumes: + - ./mongo_data:/data/db + + mongo-express: + image: mongo-express + restart: always + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: root + ME_CONFIG_MONGODB_ADMINPASSWORD: example + ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/ diff --git a/flask.Dockerfile b/flask.Dockerfile new file mode 100644 index 0000000..652d5b0 --- /dev/null +++ b/flask.Dockerfile @@ -0,0 +1,8 @@ +FROM real_base_image as flask + +COPY requirements.txt . + +ENV FLASK_ENV development +ENV FLASK_DEBUG 1 + +#ENTRYPOINT ["python", "src/flask/realtracker_web.py"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..030807e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pymongo[srv] +flask +requests_html +beautifulsoup4 +click diff --git a/src/flask/add_user.py b/src/flask/add_user.py new file mode 100644 index 0000000..76fb24d --- /dev/null +++ b/src/flask/add_user.py @@ -0,0 +1,29 @@ +import os +import click + +import mongo.user_db as user_db + +@click.command() +@click.option('--username', type=str, help="Username to be added or updated") +@click.option('--password', type=str, help="Password to be added or updated") + +def main(username, password): + """ + little cli program to update the + user table in mongo + this rightly should eventually be + and admin tool but right now this will dow + """ + user_collection = user_db.user_data() + new_record = { + "username" : f"{username}", + "password" : f"{password}" + } + if user_collection.user_exists(new_record["username"]): + user_collect.update_user(user_collection.existing_record["_id"], new_record["password"]) + else: + user_collection.add_user(new_record["username"], new_record["password"]) + + +if __name__ == "__main__": + main() diff --git a/src/flask/mongo/__init__.py b/src/flask/mongo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/flask/mongo/build_db.py b/src/flask/mongo/build_db.py new file mode 100644 index 0000000..0ce7620 --- /dev/null +++ b/src/flask/mongo/build_db.py @@ -0,0 +1,83 @@ +import string +import secrets + +from mongo.get_conn import db_conn + +class pool_data: + """ + This class will allow us to + interact with our data to interact + just create a pool_data var + """ + def __init__(self): + #db_conn has all the the things + #already created from here + #we can get self.db.real_db etc + self.db = db_conn() + + def record_exists(self, address): + """ + This function will accept an address + if it find that address in the database it will return True + and set set the existing_record variable of the class to the + queried record + """ + query = { "address" : f"{address}" } + record = self.db.real_db.find_one(query) + if record: + self.existing_record = record + return True + else: + return False + + + def create_re_record(self, address, rooms, pool_link, like_out_of_five, + pool = "", requested_price="", user_price="", + suburb="", actual_price=""): + """ + create_re_record creates a whole new record + takes the required 8 inputs + 1. address + 2. rooms + 3. pool + 4. realeastate link + 5. like out of 5 + 6. requested_price - THe price the current owner wants if known + 7. user_price - what does the user think the price of hte house should be/what offer they've put down + 8. actual_price - after sold what price the house went for + It will autogenerate the id string + this string will be automatically selected + on view in future + """ + key = self.create_id() + insert_record = { + "_id" : f"{key}", + "address": f"{address}", + "rooms": f"{rooms}", + "pool": f"{pool}", + "pool_link": f"{pool_link}", + "like_out_of_five": f"{like_out_of_five}", + "requested_price": f"{requested_price}", + "user_price": f"{user_price}", + "actual_price": f"{actual_price}", + "suburb" : f"{suburb}" + } + self.db.real_db.insert_one(insert_record) + + def update_re_record(self, id, field, value): + """ + update_re_record + will update the requested field to value for the + selected field + """ + #TODO: do we need to make this take more than 1 id? + + query = { "_id": f"{id}"} + update_val = {"$set": {f"{field}": f"{value}" }} + self.db.real_db.update_one(query, update_val) + + def create_id(self): + alphabet = string.ascii_letters + string.digits + key = ''.join(secrets.choice(alphabet) for _ in range(24)) + return key + diff --git a/src/flask/mongo/get_conn.py b/src/flask/mongo/get_conn.py new file mode 100644 index 0000000..5766f99 --- /dev/null +++ b/src/flask/mongo/get_conn.py @@ -0,0 +1,19 @@ +from pymongo import MongoClient +import os + +class db_conn: + def __init__(self): + self.db_user = os.getenv('MONGO_USER') + self.db_pass = os.getenv('MONGO_PASS') + self.db_host = os.getenv('MONGO_HOST') + self.client = self.get_client() + self.db = self.client['realestate_db'] + self.real_db = self.db['realestate_data'] + self.users = self.db['users'] + self.inspections = self.db['inspections_db'] + + def get_client(self): + CONNECTION_STRING = f"mongodb://{self.db_user}:{self.db_pass}@{self.db_host}/realestate_db?authSource=admin" + return MongoClient(CONNECTION_STRING) + + diff --git a/src/flask/mongo/query_db.py b/src/flask/mongo/query_db.py new file mode 100644 index 0000000..4030e8b --- /dev/null +++ b/src/flask/mongo/query_db.py @@ -0,0 +1,55 @@ +import string +import secrets + +from mongo.get_conn import db_conn + +class realestate_query: + """ + This class will allow us to + interact with our data to interact + just create a realestate_data var + """ + def __init__(self): + #db_conn has all the the things + #already created from here + #we can get self.db.real_db etc + self.db = db_conn() + + def record_exists(self, address): + """ + This function will accept an address + if it find that address in the database it will return True + and set set the existing_record variable of the class to the + queried record + """ + query = { "address" : f"{address}" } + record = self.db.real_db.find_one(query) + if record: + self.existing_record = record + return True + else: + return False + + def get_top(self, num_limit): + """ + This function will return the + top n records based on like_out_of_five + It will take the number of records you want to + return as a parameter + """ + records = self.db.real_db.find({}, {"address": 1, "_id": 0, "realestate_link": 1, "user_price": 1}).sort("like_out_of_five", -1).limit(num_limit) + return records + + def user_check(self, username, password): + """ + function to check username and password + back in db + """ + #TODO: this ueses my own quick hack it likley needs to be rewrittent to follow best practice + + query = { "username" : f"{username}", "password" : f"{password}" } + record = self.db.users.find_one(query) + if record: + return True + else: + return False diff --git a/src/flask/mongo/user_db.py b/src/flask/mongo/user_db.py new file mode 100644 index 0000000..8e25970 --- /dev/null +++ b/src/flask/mongo/user_db.py @@ -0,0 +1,63 @@ +import string +import secrets + +from mongo.get_conn import db_conn + + +class user_data: + """ + This class will allow us to + interact with our data to interact + just create a user_data var + """ + def __init__(self): + self.db = db_conn() + + def create_id(self): + alphabet = string.ascii_letters + string.digits + key = ''.join(secrets.choice(alphabet) for _ in range(24)) + return key + + def user_exists(self, username): + """ + This function will accept a username + if it find that user in the database it will return True + and set set the existing_record variable of the class to the + queried record + """ + query = { "username" : f"{username}" } + record = self.db.users.find_one(query) + if record: + self.existing_record = record + return True + else: + return False + + + + def add_user(self, username, password): + """ + add to user table + function accepts the username and password + it will generate a uniqe key + """ + + key = self.create_id() + insert_record = { + "_id" : f"{key}", + "username" : f"{username}", + "password" : f"{password}" + } + self.db.users.insert_one(insert_record) + + def update_user(self, id, password): + """ + update the user record password + requires the id to be updated + """ + + query = { "_id": f"{id}"} + update_val = {"$set": {"password": f"{password}" }} + self.db.users.update_one(query, update_val) + + diff --git a/src/flask/pool_data.py b/src/flask/pool_data.py new file mode 100644 index 0000000..7c26b9d --- /dev/null +++ b/src/flask/pool_data.py @@ -0,0 +1,77 @@ +import mongo.build_db as pool_database +import mongo.query_db as pool_database_query + +from flask import Flask, render_template, request, jsonify, redirect + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def index(): + if request.method == "POST": + username = request.form["username"] + password = request.form["password"] + db = pool_database_query.pool_query() + if db.user_check(username, password): + return redirect("/updater") + else: + return render_template("index.html", try_again=True) + else: + return render_template("index.html", try_again=False) + +@app.route("/updater", methods=["GET", "POST"]) +def updater(): + query_db = pool_database_query.pool_query() + query = query_db.get_top(10) + + if request.method == "POST": + database = pool_database.pool_data() + new_record = { + "address": f'{request.form["address"]}', + "rooms" : f'{request.form["rooms"]}', + "real_estate_link": f'{request.form["real_estate_link"]}', + "like_out_of_five": f'{request.form["like_out_of_five"]}', + "pool": f'{request.form["pool"]}', + "requested_price": f'{request.form["requested_price"]}', + "user_price": f'{request.form["user_price"]}', + "actual_price": f'{request.form["actual_price"]}', + "suburb" : f'{request.form["suburb"]}' + } + if database.record_exists(new_record["address"]): + for field in database.existing_record: + for new_field in new_record: + if field == new_field: + if database.existing_record[field] != new_record[field]: + database.update_re_record(database.existing_record["_id"], field, new_record[field]) + else: + database.create_re_record(new_record["address"], new_record["rooms"], new_record["real_estate_link"], + new_record["like_out_of_five"], new_record["pool"], new_record["requested_price"], + new_record["user_price"], new_record["actual_price"], new_record["suburb"]) + + + return render_template("updater.html", list=query) + else: + return render_template("updater.html", list=query) + +@app.route("/update_db", methods=["POST"]) +def pool_data_update(): + database = pool_database.pool_data() + new_record = request.json + if database.record_exists(new_record["address"]): + for field in database.existing_record: + for new_field in new_record: + if field == new_field: + if database.existing_record[field] != new_record[field]: + database.update_re_record(database.existing_record["_id"], field, new_record[field]) + else: + database.create_re_record(new_record["address"], new_record["rooms"], new_record["real_estate_link"], + new_record["like_out_of_five"], new_record["pool"], new_record["requested_price"], + new_record["user_price"], new_record["actual_price"]) + +@app.route("/pool_top/") +def user_detail(id): + query_db = pool_database_query.pool_query() + query = query_db.get_top(return_number) + return jsonify([row.to_json() for row in query]) + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=80) diff --git a/src/flask/templates/index.html b/src/flask/templates/index.html new file mode 100644 index 0000000..ab0ef99 --- /dev/null +++ b/src/flask/templates/index.html @@ -0,0 +1,11 @@ +
+

Login to Realestate Tracker

+ {% if try_again %} +

Login Failed Please try Again

+ {% endif %} +
+ + + +
+
diff --git a/src/flask/templates/updater.html b/src/flask/templates/updater.html new file mode 100644 index 0000000..5106825 --- /dev/null +++ b/src/flask/templates/updater.html @@ -0,0 +1,64 @@ +

Data Input

+
Input Data you want to store
+
+
+ + + +
+
+
+ +

+
+
+ +

+
+
+ +

+
+
+ +

+
+
+ +

+
+
+ +

+
+
+ +

+
+
+ +

+
+
+ +

+
+ +
+ 		+ + {% for row in list %} +
+ {% for key, value in row.items() %} + + + + + {% endfor %} +
{{ key }} {{ value }}
+ {% endfor %} +
+ + + +
diff --git a/start_env b/start_env new file mode 100755 index 0000000..4657523 --- /dev/null +++ b/start_env @@ -0,0 +1,4 @@ +# /bin/bash + +source .venv/bin/activate +source .env