From b5ad33f1385f16f0c9f44404cbc22d008e4de5ea Mon Sep 17 00:00:00 2001 From: Sam Hadow Date: Thu, 25 Jan 2024 23:52:23 +0100 Subject: [PATCH] flask server, gunicorn, fetch data in JS --- app.py | 33 ++++++++++++++++++++ db.py | 65 ++++++++++++++++++++++++++++++-------- gunicorn_config.py | 17 ++++++++++ main.py | 2 +- web/app.js | 78 ++++++++++++++++++++++++++++++++++++---------- 5 files changed, 165 insertions(+), 30 deletions(-) create mode 100644 app.py create mode 100644 gunicorn_config.py diff --git a/app.py b/app.py new file mode 100644 index 0000000..3218165 --- /dev/null +++ b/app.py @@ -0,0 +1,33 @@ +from flask import Flask, request, jsonify +import requests, re, json, os, yaml +from db import * +from aliexpress import * + +app = Flask(__name__) + +def get_conf(): + '''return settings in settings.yaml file''' + with open(os.path.dirname(os.path.realpath(__file__))+"/settings.yaml", 'r') as conf_file: + settings = yaml.safe_load(conf_file) + return settings + +@app.route('/') +def hello_world(): + return 'Hello, World!' + +@app.route('/datahistory',methods = ['POST', 'GET']) +def data_history_request(): + if request.method == 'GET': + print("fetching data history") + settings = get_conf() + return jsonify(get_history(settings["db"])) + +@app.route('/dataitem',methods = ['POST', 'GET']) +def data_item_request(): + if request.method == 'GET': + print("fetching data item") + settings = get_conf() + return jsonify(get_item(settings["db"])) + +if __name__ == '__main__': + app.run(debug = True) diff --git a/db.py b/db.py index 19b9ee7..f29ba54 100644 --- a/db.py +++ b/db.py @@ -18,8 +18,8 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image): cursor = connection.cursor() cursor.execute(""" - INSERT INTO item (itemid, skuid, choice, attributes, image) - VALUES (%s, %s, %s, %s, %s) + INSERT INTO item (uuid, itemid, skuid, choice, attributes, image) + VALUES (nextval('uuid_sequence'), %s, %s, %s, %s, %s) """, (itemid, skuid, choice, attributes, image)) connection.commit() connection.close() @@ -29,7 +29,7 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri cursor = connection.cursor() cursor.execute(""" - SELECT * + SELECT uuid FROM item WHERE itemid = %s AND skuid = %s @@ -37,23 +37,54 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri if cursor.rowcount == 0: add_item(db_settings, itemid, skuid, choice, attributes, image) + cursor.execute(""" + SELECT uuid + FROM item + WHERE itemid = %s + AND skuid = %s + """, (itemid, skuid)) + + uuid = cursor.fetchall()[0] cursor.execute(""" - INSERT INTO history (itemid, skuid, price, currency, quantity, discount_percentage, h_timestamp) - VALUES (%s, %s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP)) - """, (itemid, skuid, price, currency, quantity, discount_percentage)) + INSERT INTO history (uuid, price, currency, quantity, discount_percentage, h_timestamp) + VALUES (%s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP)) + """, (uuid, price, currency, quantity, discount_percentage)) connection.commit() connection.close() +def get_history(db_settings): + connection = connect_db(db_settings) + cursor = connection.cursor() + cursor.execute(""" + SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp + FROM history + """) + results = cursor.fetchall() + cursor.close() + connection.close() + return results + +def get_item(db_settings): + connection = connect_db(db_settings) + cursor = connection.cursor() + cursor.execute(""" + SELECT uuid, itemid, skuid, choice, attributes, image + FROM item + """) + results = cursor.fetchall() + cursor.close() + connection.close() + return results def export_csv(db_settings): connection = connect_db(db_settings) cursor = connection.cursor() cursor.execute(""" - SELECT i.itemid, i.skuid, i.choice, i.attributes, i.image, h.quantity, h.discount_percentage, h.price, h.currency, h.h_timestamp + SELECT i.uuid, i.itemid, i.skuid, i.choice, i.attributes, i.image, h.quantity, h.discount_percentage, h.price, h.currency, h.h_timestamp FROM item i, history h - WHERE i.itemid = h.itemid and i.skuid = h.skuid + WHERE i.uuid = h.uuid """) results = cursor.fetchall() with open(os.path.dirname(os.path.realpath(__file__))+"/output.csv", 'w') as csv_file: @@ -78,28 +109,36 @@ def initialize(db_settings): DROP TABLE IF EXISTS item """) cursor.execute(""" + DROP SEQUENCE IF EXISTS uuid_sequence + """) + cursor.execute(""" + CREATE SEQUENCE uuid_sequence + INCREMENT BY 1 + START WITH 1 + """) + cursor.execute(""" CREATE TABLE item ( + uuid int, itemid bigint, skuid bigint, choice boolean, attributes text[], image text, - primary key (itemid,skuid) + primary key (uuid) ) """) cursor.execute(""" CREATE TABLE history ( - itemid bigint, - skuid bigint, + uuid int, quantity integer, discount_percentage numeric(2), price money, currency varchar(4), h_timestamp timestamp, - foreign key (itemid,skuid) references item(itemid,skuid), - primary key (itemid,skuid,h_timestamp) + foreign key (uuid) references item(uuid), + primary key (uuid, h_timestamp) ) """) diff --git a/gunicorn_config.py b/gunicorn_config.py new file mode 100644 index 0000000..76eca1a --- /dev/null +++ b/gunicorn_config.py @@ -0,0 +1,17 @@ +import os + + + +workers = int(os.environ.get('GUNICORN_PROCESSES', '2')) + +threads = int(os.environ.get('GUNICORN_THREADS', '4')) + +# timeout = int(os.environ.get('GUNICORN_TIMEOUT', '120')) + +bind = os.environ.get('GUNICORN_BIND', '0.0.0.0:8080') + + + +forwarded_allow_ips = '*' + +secure_scheme_headers = { 'X-Forwarded-Proto': 'https' } diff --git a/main.py b/main.py index efb0884..ef4384c 100644 --- a/main.py +++ b/main.py @@ -14,7 +14,7 @@ if __name__ == '__main__': settings = get_conf() - #initialize(settings["db"]) + # initialize(settings["db"]) fill_db(settings["db"], check_items(settings["item"])) export_csv(settings["db"]) diff --git a/web/app.js b/web/app.js index 53d65a0..1122098 100644 --- a/web/app.js +++ b/web/app.js @@ -16,24 +16,69 @@ const width = window.innerWidth - margin.right - margin.left -10; // (resizes every elements) window.onresize = function(){ location.reload(); } -d3.csv('http://127.0.0.1/output.csv', d => { - return { - date: new Date(d.h_timestamp.replace(' ', 'T')), - value: parseFloat(d.price.replace('$', '')), - skuid: d.skuid, - itemid: d.itemid, - image: d.image, - currency: d.currency - } -}).then(function(data) { +async function fetch_history() { + try { + const response = await fetch('http://127.0.0.1:8080/datahistory'); + const rawData = await response.json(); + // SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp + let historyData = rawData.map(d => ({ + uuid: d[0], + value: parseFloat(d[3].replace('$', '')), + currency: d[4], + date: new Date(d[5].replace(' ', 'T')), + })); + return historyData; + } catch (error) { + console.error('Error fetching data history: ', error); + throw error; + } +} + +async function fetch_item() { + try { + // SELECT uuid, itemid, skuid, choice, attributes, image + const response = await fetch('http://127.0.0.1:8080/dataitem'); + const rawData = await response.json(); + const items = rawData.reduce((item, row) => { + const uuid = row[0]; + + const itemid = row[1]; + const skuid = row[2]; + const choice = row[3]; + const attributes = row[4]; + const image = row[5]; + + const values = { + itemid: itemid, + skuid: skuid, + choice: choice, + attributes: attributes, + image: image + }; + + item[uuid] = values; + + return item; + }, {}); + + return items; + } catch (error) { + console.error('Error fetching data item: ', error); + throw error; + } +} + + +fetch_history().then(async function(data) { + items = await fetch_item(); // Date domain (width) const x = d3.scaleTime() .domain(d3.extent(data, d => d.date)) .range([3, width-3]); - // Group the data by (itemid, skuid) - const nestedData = d3.group(data, d => d.skuid); + // Group the data by (uuid) + const nestedData = d3.group(data, d => d.uuid); // Create a div for each graph const graphDivs = d3.select("#graphs") .selectAll(".graph-container") @@ -58,16 +103,17 @@ d3.csv('http://127.0.0.1/output.csv', d => { const svg = d3.select(this); // context on right side - // text - const link = `https://fr.aliexpress.com/item/${dataSubset[0].itemid}.html`; + const link = `https://fr.aliexpress.com/item/${items[key].itemid}.html`; // image svg.append("image") .attr("x", width + margin.right*0.1) .attr("y", height*0.1) .attr("width", height*0.8) .attr("height", height*0.8) - .attr("xlink:href", dataSubset[0].image) // placeholder picture for now, should be item picture - .on("click", function() { window.open(link); }); + .attr("xlink:href", items[key].image) + .on("click", function() { + window.open(link, '_blank', 'noopener,noreferrer'); + }); // Price domain (height) const y = d3.scaleLinear()