diff --git a/src/aliexpress.py b/src/aliexpress.py index a06ec12..6774ca8 100644 --- a/src/aliexpress.py +++ b/src/aliexpress.py @@ -78,10 +78,16 @@ def get_attributes(attributes_raw): return attributes -def fill_db(db_settings, items_dict): +def fill_db(items_dict): '''add new history entries in database with data extracted in item_dict''' for key,value in items_dict.items(): - add_history_entry(db_settings, key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["image"], value["price"], value["currency"], value["quantity"], value["discount_percentage"]) + add_history_entry(key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["image"], value["price"], value["currency"], value["quantity"], value["discount_percentage"]) + +def update_items(): + '''add new history entries for items in database''' + in_db = get_item_keys() + new_entries = check_items(in_db) + fill_db(new_entries) diff --git a/src/app.py b/src/app.py index a9a20f0..e9c6240 100644 --- a/src/app.py +++ b/src/app.py @@ -7,33 +7,16 @@ from aliexpress import * app = Flask(__name__) CORS(app) -def get_conf(): - '''return settings in settings.yaml file''' - settings = dict() - with open(os.path.dirname(os.path.realpath(__file__))+"/settings.yaml", 'r') as conf_file: - settings = yaml.safe_load(conf_file) - settings["db"] = { - "host": os.environ.get('POSTGRES_HOST'), - "port": os.environ.get('POSTGRES_PORT'), - "database": os.environ.get('POSTGRES_DB'), - "user": os.environ.get('POSTGRES_USER'), - "password": os.environ.get('POSTGRES_PASSWORD') - } - - return settings - @app.route('/init') def init_db(): print("init") - settings = get_conf() - initialize(settings["db"]) + initialize() return 'Hello, World!' @app.route('/update') def update_hist(): print("update") - settings = get_conf() - fill_db(settings["db"], check_items(settings["item"])) + fill_db(check_items()) return 'Hello, World!' @app.route('/add', methods=['POST']) @@ -49,14 +32,13 @@ def add_item(): extr = check_items(new_item) if len(extr) > 0: - settings = get_conf() skuid = list(extr.values())[0]["skuid"] - if check_exist(settings["db"], itemid, skuid): + if check_exist(itemid, skuid): # item already exists return jsonify({'status': 3}), 400 else: # item is valid - fill_db(settings["db"], extr) + fill_db(extr) return jsonify({'status': 0}), 200 else: # item not valid or can't be parsed @@ -66,15 +48,13 @@ def add_item(): def data_history_request(): if request.method == 'GET': print("fetching data history") - settings = get_conf() - return jsonify(get_history(settings["db"])) + return jsonify(get_history()) @app.route('/dataitem',methods = ['POST', 'GET']) def data_item_request(): if request.method == 'GET': print("fetching data item") - settings = get_conf() - return jsonify(get_item(settings["db"])) + return jsonify(get_item()) if __name__ == '__main__': app.run(debug = True) diff --git a/src/background.py b/src/background.py new file mode 100644 index 0000000..12650a2 --- /dev/null +++ b/src/background.py @@ -0,0 +1,5 @@ +#!/usr/bin/python +from aliexpress import * + +if __name__ == '__main__': + print("ok") diff --git a/src/db.py b/src/db.py index f24e345..d73a8d5 100644 --- a/src/db.py +++ b/src/db.py @@ -3,9 +3,21 @@ import psycopg2 import csv import os -def connect_db(db_settings): +def get_conf(): + '''return db connection settings''' + settings = { + "host": os.environ.get('POSTGRES_HOST'), + "port": os.environ.get('POSTGRES_PORT'), + "database": os.environ.get('POSTGRES_DB'), + "user": os.environ.get('POSTGRES_USER'), + "password": os.environ.get('POSTGRES_PASSWORD') + } + return settings + +def connect_db(): '''open and return a connection to the database''' conn = None + db_settings = get_conf() print("Connecting to the PostgreSQL database...") try: conn = psycopg2.connect(**db_settings) @@ -14,9 +26,9 @@ def connect_db(db_settings): print(error) return conn -def add_item(db_settings, itemid, skuid, choice, attributes, image): +def add_item(itemid, skuid, choice, attributes, image): '''insert a new item in the database''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() cursor.execute(""" @@ -26,13 +38,13 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image): connection.commit() connection.close() -def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, price, currency, quantity, discount_percentage): +def add_history_entry(itemid, skuid, choice, attributes, image, price, currency, quantity, discount_percentage): '''Add a new history entry for an item in the database. If item isn't in database yet, add it.''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() - if not check_exist(db_settings, itemid, skuid): - add_item(db_settings, itemid, skuid, choice, attributes, image) + if not check_exist(itemid, skuid): + add_item(itemid, skuid, choice, attributes, image) cursor.execute(""" SELECT uuid FROM item @@ -49,9 +61,9 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri connection.commit() connection.close() -def get_history(db_settings): +def get_history(): '''return history data from database''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() cursor.execute(""" SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp @@ -62,9 +74,9 @@ def get_history(db_settings): connection.close() return results -def get_item(db_settings): +def get_item(): '''return items data from database''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() cursor.execute(""" SELECT uuid, itemid, skuid, choice, attributes, image @@ -75,9 +87,22 @@ def get_item(db_settings): connection.close() return results -def check_exist(db_settings, itemid, skuid): +def get_item_keys(): + '''return items id and attributes from database''' + connection = connect_db() + cursor = connection.cursor() + cursor.execute(""" + SELECT itemid, attributes + FROM item + """) + results = cursor.fetchall() + cursor.close() + connection.close() + return results + +def check_exist(itemid, skuid): '''check if an item is already in the database''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() cursor.execute(""" @@ -96,9 +121,10 @@ def check_exist(db_settings, itemid, skuid): else: return True -def export_csv(db_settings): + +def export_csv(): '''join item and history data from database and export it in ./output.csv''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() cursor.execute(""" @@ -118,9 +144,9 @@ def export_csv(db_settings): connection.close() -def initialize(db_settings): +def initialize(): '''Create tables and sequence in database. Drop them first if they already exist.''' - connection = connect_db(db_settings) + connection = connect_db() cursor = connection.cursor() cursor.execute(""" diff --git a/web/app.js b/web/app.js index cbd4d12..685326b 100644 --- a/web/app.js +++ b/web/app.js @@ -24,7 +24,6 @@ async function fetch_history() { const response = await fetch(`${currentUrl}app/datahistory`); const rawData = await response.json(); var dateFormat = d3.timeParse("%a, %d %b %Y %H:%M:%S GMT"); - console.log(rawData) // SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp let historyData = rawData.map(d => ({ uuid: d[0], @@ -32,7 +31,6 @@ async function fetch_history() { currency: d[4], date: dateFormat(d[5]), })); - console.log(historyData) return historyData; } catch (error) { console.error('Error fetching data history: ', error);