tidying db functions

src/aliexpress.py

@@ -78,10 +78,16 @@ def get_attributes(attributes_raw):
 
     return attributes
 
-def fill_db(db_settings, items_dict):
+def fill_db(items_dict):
     '''add new history entries in database with data extracted in item_dict'''
     for key,value in items_dict.items():
-        add_history_entry(db_settings, key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["image"], value["price"], value["currency"], value["quantity"], value["discount_percentage"])
+        add_history_entry(key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["image"], value["price"], value["currency"], value["quantity"], value["discount_percentage"])
+
+def update_items():
+    '''add new history entries for items in database'''
+    in_db = get_item_keys()
+    new_entries = check_items(in_db)
+    fill_db(new_entries)
 
 
 

src/app.py

@@ -7,33 +7,16 @@ from aliexpress import *
 app = Flask(__name__)
 CORS(app)
 
-def get_conf():
-    '''return settings in settings.yaml file'''
-    settings = dict()
-    with open(os.path.dirname(os.path.realpath(__file__))+"/settings.yaml", 'r') as conf_file:
-        settings = yaml.safe_load(conf_file)
-    settings["db"] = {
-            "host": os.environ.get('POSTGRES_HOST'),
-            "port": os.environ.get('POSTGRES_PORT'),
-            "database": os.environ.get('POSTGRES_DB'),
-            "user": os.environ.get('POSTGRES_USER'),
-            "password": os.environ.get('POSTGRES_PASSWORD')
-            }
-
-    return settings
-
 @app.route('/init')
 def init_db():
     print("init")
-    settings = get_conf()
+    initialize()
-    initialize(settings["db"])
     return 'Hello, World!'
 
 @app.route('/update')
 def update_hist():
     print("update")
-    settings = get_conf()
+    fill_db(check_items())
-    fill_db(settings["db"], check_items(settings["item"]))
     return 'Hello, World!'
 
 @app.route('/add', methods=['POST'])
@@ -49,14 +32,13 @@ def add_item():
     extr = check_items(new_item)
 
     if len(extr) > 0:
-        settings = get_conf()
         skuid = list(extr.values())[0]["skuid"]
-        if check_exist(settings["db"], itemid, skuid):
+        if check_exist(itemid, skuid):
             # item already exists
             return jsonify({'status': 3}), 400
         else:
             # item is valid
-            fill_db(settings["db"], extr)
+            fill_db(extr)
             return jsonify({'status': 0}), 200
     else:
         # item not valid or can't be parsed
@@ -66,15 +48,13 @@ def add_item():
 def data_history_request():
     if request.method == 'GET':
         print("fetching data history")
-        settings = get_conf()
+        return jsonify(get_history())
-        return jsonify(get_history(settings["db"]))
 
 @app.route('/dataitem',methods = ['POST', 'GET'])
 def data_item_request():
     if request.method == 'GET':
         print("fetching data item")
-        settings = get_conf()
+        return jsonify(get_item())
-        return jsonify(get_item(settings["db"]))
 
 if __name__ == '__main__':
    app.run(debug = True)

src/background.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python
+from aliexpress import *
+
+if __name__ == '__main__':
+    print("ok")

src/db.py

@@ -3,9 +3,21 @@ import psycopg2
 import csv
 import os
 
-def connect_db(db_settings):
+def get_conf():
+    '''return db connection settings'''
+    settings = {
+            "host": os.environ.get('POSTGRES_HOST'),
+            "port": os.environ.get('POSTGRES_PORT'),
+            "database": os.environ.get('POSTGRES_DB'),
+            "user": os.environ.get('POSTGRES_USER'),
+            "password": os.environ.get('POSTGRES_PASSWORD')
+            }
+    return settings
+
+def connect_db():
     '''open and return a connection to the database'''
     conn = None
+    db_settings = get_conf()
     print("Connecting to the PostgreSQL database...")
     try:
         conn = psycopg2.connect(**db_settings)
@@ -14,9 +26,9 @@ def connect_db(db_settings):
         print(error)
     return conn
 
-def add_item(db_settings, itemid, skuid, choice, attributes, image):
+def add_item(itemid, skuid, choice, attributes, image):
     '''insert a new item in the database'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
 
     cursor.execute("""
@@ -26,13 +38,13 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image):
     connection.commit()
     connection.close()
 
-def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, price, currency, quantity, discount_percentage):
+def add_history_entry(itemid, skuid, choice, attributes, image, price, currency, quantity, discount_percentage):
     '''Add a new history entry for an item in the database. If item isn't in database yet, add it.'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
 
-    if not check_exist(db_settings, itemid, skuid):
+    if not check_exist(itemid, skuid):
-        add_item(db_settings, itemid, skuid, choice, attributes, image)
+        add_item(itemid, skuid, choice, attributes, image)
         cursor.execute("""
         SELECT uuid
         FROM item
@@ -49,9 +61,9 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri
     connection.commit()
     connection.close()
 
-def get_history(db_settings):
+def get_history():
     '''return history data from database'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
     cursor.execute("""
     SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp
@@ -62,9 +74,9 @@ def get_history(db_settings):
     connection.close()
     return results
 
-def get_item(db_settings):
+def get_item():
     '''return items data from database'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
     cursor.execute("""
     SELECT uuid, itemid, skuid, choice, attributes, image
@@ -75,9 +87,22 @@ def get_item(db_settings):
     connection.close()
     return results
 
-def check_exist(db_settings, itemid, skuid):
+def get_item_keys():
+    '''return items id and attributes from database'''
+    connection = connect_db()
+    cursor = connection.cursor()
+    cursor.execute("""
+    SELECT itemid, attributes
+    FROM item
+    """)
+    results = cursor.fetchall()
+    cursor.close()
+    connection.close()
+    return results
+
+def check_exist(itemid, skuid):
     '''check if an item is already in the database'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
 
     cursor.execute("""
@@ -96,9 +121,10 @@ def check_exist(db_settings, itemid, skuid):
     else:
         return True
 
-def export_csv(db_settings):
+
+def export_csv():
     '''join item and history data from database and export it in ./output.csv'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
 
     cursor.execute("""
@@ -118,9 +144,9 @@ def export_csv(db_settings):
     connection.close()
 
 
-def initialize(db_settings):
+def initialize():
     '''Create tables and sequence in database. Drop them first if they already exist.'''
-    connection = connect_db(db_settings)
+    connection = connect_db()
     cursor = connection.cursor()
 
     cursor.execute("""

web/app.js

@@ -24,7 +24,6 @@ async function fetch_history() {
     const response = await fetch(`${currentUrl}app/datahistory`);
     const rawData = await response.json();
     var dateFormat = d3.timeParse("%a, %d %b %Y %H:%M:%S GMT");
-    console.log(rawData)
     // SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp
     let historyData = rawData.map(d => ({
         uuid: d[0],
@@ -32,7 +31,6 @@ async function fetch_history() {
         currency: d[4],
         date: dateFormat(d[5]),
     }));
-    console.log(historyData)
     return historyData;
   } catch (error) {
     console.error('Error fetching data history: ', error);