initial transfer of realestate code to pool to start a web service to capture the data

.gitignore

@@ -0,0 +1,4 @@
+*__pycache__*
+.venv*
+.env
+mongo_data/*

base.Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.10 as real_base_image
+
+WORKDIR /pool_data
+
+COPY requirements.txt .
+
+RUN apt-get update -y && apt-get upgrade -y && apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev
+
+RUN pip install --upgrade pip
+
+RUN pip --default-timeout=1000 install -r requirements.txt
+

compose_up_cmd.sh

@@ -0,0 +1,7 @@
+
+docker-compose rm -f 
+docker system prune -f
+docker volume prune -f
+docker build -t real_base_image -f base.Dockerfile .
+docker-compose up --remove-orphans --build -d
+docker logs -f realtracker_web

docker-compose.yml

@@ -0,0 +1,40 @@
+# Use root/example as user/password credentials
+version: '3.1'
+
+services:
+  realtracker_web:
+    container_name: realtracker_web
+    build:
+      context: .
+      dockerfile: flask.Dockerfile
+    volumes:
+      - ./src/flask:/realestate_tracker/src/flask
+    ports:
+      - "80:80"
+      - "5000:5000"
+    restart: "unless-stopped"
+    environment:
+      MONGO_HOST: mongo
+      MONGO_USER: root
+      MONGO_PASS: example
+
+  mongo:
+    image: mongo
+    restart: always
+    ports:
+      - 27017:27017
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+    volumes:
+      - ./mongo_data:/data/db
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: example
+      ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

flask.Dockerfile

@@ -0,0 +1,8 @@
+FROM real_base_image as flask
+
+COPY requirements.txt .
+
+ENV FLASK_ENV development
+ENV FLASK_DEBUG 1
+
+#ENTRYPOINT ["python", "src/flask/realtracker_web.py"]

requirements.txt

@@ -0,0 +1,5 @@
+pymongo[srv]
+flask
+requests_html
+beautifulsoup4
+click

src/flask/add_user.py

@@ -0,0 +1,29 @@
+import os
+import click
+
+import mongo.user_db as user_db
+
+@click.command()
+@click.option('--username', type=str, help="Username to be added or updated")
+@click.option('--password', type=str, help="Password to be added or updated")
+
+def main(username, password):
+    """
+        little cli program to update the 
+        user table in mongo
+        this rightly should eventually be 
+        and admin tool but right now this will dow
+    """
+    user_collection = user_db.user_data()
+    new_record = {
+            "username" : f"{username}",
+            "password" : f"{password}"
+            }
+    if user_collection.user_exists(new_record["username"]):
+        user_collect.update_user(user_collection.existing_record["_id"], new_record["password"])
+    else:
+        user_collection.add_user(new_record["username"], new_record["password"])
+
+
+if __name__ == "__main__":
+    main()

src/flask/mongo/build_db.py

@@ -0,0 +1,83 @@
+import string
+import secrets
+
+from mongo.get_conn import db_conn
+
+class pool_data:
+    """
+    This class will allow us to
+    interact with our data to interact
+    just create a pool_data var
+    """
+    def __init__(self):
+        #db_conn has all the the things
+        #already created from here 
+        #we can get self.db.real_db etc 
+        self.db = db_conn()
+
+    def record_exists(self, address):
+        """
+            This function will accept an address
+            if it find that address in the database it will return True
+            and set set the existing_record variable of the class to the 
+            queried record
+        """
+        query = { "address" : f"{address}" }
+        record = self.db.real_db.find_one(query)
+        if record:
+            self.existing_record = record
+            return True
+        else:
+            return False
+
+
+    def create_re_record(self, address, rooms, pool_link, like_out_of_five,
+                         pool = "", requested_price="", user_price="", 
+                         suburb="", actual_price=""):
+        """
+            create_re_record creates a whole new record
+            takes the required 8 inputs
+            1. address
+            2. rooms
+            3. pool
+            4. realeastate link
+            5. like out of 5
+            6. requested_price - THe price the current owner wants if known
+            7. user_price - what does the user think the price of hte house should be/what offer they've put down
+            8. actual_price - after sold what price the house went for
+            It will autogenerate the id string
+            this string will be automatically selected
+            on view in future
+        """
+        key = self.create_id()
+        insert_record = {
+                "_id" : f"{key}",
+                "address": f"{address}",
+                "rooms": f"{rooms}",
+                "pool": f"{pool}",
+                "pool_link": f"{pool_link}",
+                "like_out_of_five": f"{like_out_of_five}",
+                "requested_price": f"{requested_price}",
+                "user_price": f"{user_price}",
+                "actual_price": f"{actual_price}",
+                "suburb" : f"{suburb}"
+                }
+        self.db.real_db.insert_one(insert_record)
+
+    def update_re_record(self, id, field, value):
+        """
+            update_re_record
+            will update the requested field to value for the
+            selected field 
+        """
+        #TODO: do we need to make this take more than 1 id?
+
+        query = { "_id": f"{id}"}
+        update_val = {"$set": {f"{field}": f"{value}" }}
+        self.db.real_db.update_one(query, update_val)
+
+    def create_id(self):
+        alphabet = string.ascii_letters + string.digits
+        key = ''.join(secrets.choice(alphabet) for _ in range(24))
+        return key
+

src/flask/mongo/get_conn.py

@@ -0,0 +1,19 @@
+from pymongo import MongoClient
+import os
+
+class db_conn:
+    def __init__(self):
+        self.db_user = os.getenv('MONGO_USER')
+        self.db_pass = os.getenv('MONGO_PASS')
+        self.db_host = os.getenv('MONGO_HOST')
+        self.client = self.get_client()
+        self.db = self.client['realestate_db']
+        self.real_db = self.db['realestate_data']
+        self.users  = self.db['users']
+        self.inspections = self.db['inspections_db']
+
+    def get_client(self):
+        CONNECTION_STRING = f"mongodb://{self.db_user}:{self.db_pass}@{self.db_host}/realestate_db?authSource=admin"
+        return MongoClient(CONNECTION_STRING)
+
+

src/flask/mongo/query_db.py

@@ -0,0 +1,55 @@
+import string
+import secrets
+
+from mongo.get_conn import db_conn
+
+class realestate_query:
+    """
+    This class will allow us to
+    interact with our data to interact
+    just create a realestate_data var
+    """
+    def __init__(self):
+        #db_conn has all the the things
+        #already created from here 
+        #we can get self.db.real_db etc 
+        self.db = db_conn()
+
+    def record_exists(self, address):
+        """
+            This function will accept an address
+            if it find that address in the database it will return True
+            and set set the existing_record variable of the class to the 
+            queried record
+        """
+        query = { "address" : f"{address}" }
+        record = self.db.real_db.find_one(query)
+        if record:
+            self.existing_record = record
+            return True
+        else:
+            return False
+
+    def get_top(self, num_limit):
+        """
+            This function will return the
+            top n records based on like_out_of_five
+            It will take the number of records you want to 
+            return as a parameter
+        """
+        records = self.db.real_db.find({}, {"address": 1, "_id": 0, "realestate_link": 1, "user_price": 1}).sort("like_out_of_five", -1).limit(num_limit)
+        return records
+
+    def user_check(self, username, password):
+        """
+            function to check username and password
+            back in db
+        """
+        #TODO: this ueses my own quick hack it likley needs to be rewrittent to follow best practice
+
+        query = { "username" : f"{username}", "password" : f"{password}" }
+        record = self.db.users.find_one(query)
+        if record:
+            return True
+        else:
+            return False

src/flask/mongo/user_db.py

@@ -0,0 +1,63 @@
+import string
+import secrets
+
+from mongo.get_conn import db_conn
+
+
+class user_data:
+    """
+        This class will allow us to
+        interact with our data to interact
+        just create a user_data var
+    """
+    def __init__(self):
+        self.db = db_conn()
+
+    def create_id(self):
+        alphabet = string.ascii_letters + string.digits
+        key = ''.join(secrets.choice(alphabet) for _ in range(24))
+        return key
+
+    def user_exists(self, username):
+        """
+            This function will accept a username
+            if it find that user in the database it will return True
+            and set set the existing_record variable of the class to the 
+            queried record
+        """
+        query = { "username" : f"{username}" }
+        record = self.db.users.find_one(query)
+        if record:
+            self.existing_record = record
+            return True
+        else:
+            return False
+
+
+
+    def add_user(self, username, password):
+        """
+            add to user table
+            function accepts the username and password
+            it will generate a uniqe key
+        """
+
+        key = self.create_id()
+        insert_record = {
+                         "_id" : f"{key}",
+                         "username" : f"{username}",
+                         "password" : f"{password}"
+                         }
+        self.db.users.insert_one(insert_record)
+
+    def update_user(self, id, password):
+        """
+            update the user record password
+            requires the id to be updated 
+        """
+
+        query = { "_id": f"{id}"}
+        update_val = {"$set": {"password": f"{password}" }}
+        self.db.users.update_one(query, update_val)
+
+

src/flask/pool_data.py

@@ -0,0 +1,77 @@
+import mongo.build_db as pool_database
+import mongo.query_db as pool_database_query
+
+from flask import Flask, render_template, request, jsonify, redirect
+
+app = Flask(__name__)
+
+@app.route("/", methods=["GET","POST"])
+def index():
+    if request.method == "POST":
+        username = request.form["username"]
+        password = request.form["password"]
+        db = pool_database_query.pool_query()
+        if db.user_check(username, password):
+            return redirect("/updater")
+        else:
+            return render_template("index.html", try_again=True)
+    else:
+        return render_template("index.html", try_again=False)
+
+@app.route("/updater", methods=["GET", "POST"])
+def updater():
+    query_db = pool_database_query.pool_query()
+    query = query_db.get_top(10)
+
+    if request.method == "POST":
+        database = pool_database.pool_data()
+        new_record = {
+                "address": f'{request.form["address"]}',
+                "rooms" : f'{request.form["rooms"]}',
+                "real_estate_link": f'{request.form["real_estate_link"]}',
+                "like_out_of_five": f'{request.form["like_out_of_five"]}',
+                "pool": f'{request.form["pool"]}',
+                "requested_price": f'{request.form["requested_price"]}',
+                "user_price": f'{request.form["user_price"]}',
+                "actual_price": f'{request.form["actual_price"]}',
+                "suburb" : f'{request.form["suburb"]}'
+            }
+        if database.record_exists(new_record["address"]):
+            for field in database.existing_record:
+                for new_field in new_record:
+                    if field == new_field:
+                        if database.existing_record[field] != new_record[field]:
+                            database.update_re_record(database.existing_record["_id"], field, new_record[field])
+        else:
+            database.create_re_record(new_record["address"], new_record["rooms"], new_record["real_estate_link"], 
+                                      new_record["like_out_of_five"], new_record["pool"], new_record["requested_price"],
+                                      new_record["user_price"], new_record["actual_price"], new_record["suburb"])
+
+
+        return render_template("updater.html", list=query)
+    else:
+        return render_template("updater.html", list=query)
+
+@app.route("/update_db", methods=["POST"])
+def pool_data_update():
+    database = pool_database.pool_data()
+    new_record = request.json
+    if database.record_exists(new_record["address"]):
+        for field in database.existing_record:
+            for new_field in new_record:
+                if field == new_field:
+                    if database.existing_record[field] != new_record[field]:
+                        database.update_re_record(database.existing_record["_id"], field, new_record[field])
+            else:
+                database.create_re_record(new_record["address"], new_record["rooms"], new_record["real_estate_link"], 
+                                      new_record["like_out_of_five"], new_record["pool"], new_record["requested_price"],
+                                      new_record["user_price"], new_record["actual_price"])
+
+@app.route("/pool_top/<int:return_number>")
+def user_detail(id):
+    query_db = pool_database_query.pool_query()
+    query = query_db.get_top(return_number)
+    return jsonify([row.to_json() for row in query])
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=80)

src/flask/templates/index.html

@@ -0,0 +1,11 @@
+<div class="log-form">
+  <h2>Login to Realestate Tracker</h2>
+  {% if try_again %}
+  <h4>Login Failed Please try Again</h4>
+  {% endif %}
+  <form method="POST" actions="/">
+    <input type="text" id="username" name="username" title="username" placeholder="username" />
+    <input type="password" id="password" name="password" title="username" placeholder="password" />
+    <button type="submit" class="btn">Login</button>
+  </form>
+</div><!--end log form -->

src/flask/templates/updater.html

@@ -0,0 +1,64 @@
+<h2>Data Input</h2>
+<h5>Input Data you want to store</h5>
+<div class="row">
+    <div class="col-md-6">
+    <table border=6>
+        <tr>
+            <td>
+                <form method="POST" action="/updater">
+                <div class="form-group">
+                    <label for="address">Address:</label>
+                    <input type="text" id="address" class="form-control" name="address" required><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="rooms">Rooms:</label>
+                    <input type="text" id="rooms" class="form-control" name="rooms" required><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="pool">Pool(True, False):</label>
+                    <input type="text" class="form-control" id="pool" name="pool" /><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="real_estate_link">Realestate.com Link:</label>
+                    <input type="text" id="real_estate_link" class="form-control" name="real_estate_link" required><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="like_out_of_five">Like Out of 5:</label>
+                    <input type="text" id="like_out_of_five" class="form-control" name="like_out_of_five" pattern="[1-5]{1}" required><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="requested_price">Request Price:</label>
+                    <input type="text" id="requested_price" class="form-control" name="requested_price" /><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="user_price">What do you think its worth?:</label>
+                    <input type="text" id="user_price" class="form-control" name="user_price" /><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="actual_price">What Price did it go for in the end?:</label>
+                    <input type="text" id="actual_price" class="form-control" name="actual_price" /><br><br>
+                </div>
+                <div class="form-group">
+                    <label for="suburb">What Suburb is it in?:</label>
+                    <input type="text" id="suburb" class="form-control" name="suburb" /><br><br>
+                </div>
+                <input type="submit" class="btn btn-primary" name="action"/>
+            </form>
+        </td>
+        <td>
+            <table border = 1>
+            {% for row in list %}
+                <table border=2>
+                {% for key, value in row.items() %}
+                <tr>
+                    <th> {{ key }} </th>
+                    <td> {{ value }} </td>
+                </tr>
+                {% endfor %}
+                </table>
+            {% endfor %}
+            </table>
+        </td>
+    </tr>
+
+</div>

start_env

@@ -0,0 +1,4 @@
+# /bin/bash
+
+source .venv/bin/activate
+source .env