flask server, gunicorn, fetch data in JS

app.py

@@ -0,0 +1,33 @@
+from flask import Flask, request, jsonify
+import requests, re, json, os, yaml
+from db import *
+from aliexpress import *
+
+app = Flask(__name__)
+
+def get_conf():
+    '''return settings in settings.yaml file'''
+    with open(os.path.dirname(os.path.realpath(__file__))+"/settings.yaml", 'r') as conf_file:
+        settings = yaml.safe_load(conf_file)
+    return settings
+
+@app.route('/')
+def hello_world():
+    return 'Hello, World!'
+
+@app.route('/datahistory',methods = ['POST', 'GET'])
+def data_history_request():
+    if request.method == 'GET':
+        print("fetching data history")
+        settings = get_conf()
+        return jsonify(get_history(settings["db"]))
+
+@app.route('/dataitem',methods = ['POST', 'GET'])
+def data_item_request():
+    if request.method == 'GET':
+        print("fetching data item")
+        settings = get_conf()
+        return jsonify(get_item(settings["db"]))
+
+if __name__ == '__main__':
+   app.run(debug = True)

db.py

@@ -18,8 +18,8 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image):
     cursor = connection.cursor()
 
     cursor.execute("""
-    INSERT INTO item (itemid, skuid, choice, attributes, image)
+    INSERT INTO item (uuid, itemid, skuid, choice, attributes, image)
-    VALUES (%s, %s, %s, %s, %s)
+    VALUES (nextval('uuid_sequence'), %s, %s, %s, %s, %s)
     """, (itemid, skuid, choice, attributes, image))
     connection.commit()
     connection.close()
@@ -29,7 +29,7 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri
     cursor = connection.cursor()
 
     cursor.execute("""
-    SELECT *
+    SELECT uuid
     FROM item
     WHERE itemid = %s
     AND skuid = %s
@@ -37,23 +37,54 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri
 
     if cursor.rowcount == 0:
         add_item(db_settings, itemid, skuid, choice, attributes, image)
+        cursor.execute("""
+        SELECT uuid
+        FROM item
+        WHERE itemid = %s
+        AND skuid = %s
+        """, (itemid, skuid))
+
+    uuid = cursor.fetchall()[0]
 
     cursor.execute("""
-    INSERT INTO history (itemid, skuid, price, currency, quantity, discount_percentage, h_timestamp)
+    INSERT INTO history (uuid, price, currency, quantity, discount_percentage, h_timestamp)
-    VALUES (%s, %s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP))
+    VALUES (%s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP))
-    """, (itemid, skuid, price, currency, quantity, discount_percentage))
+    """, (uuid, price, currency, quantity, discount_percentage))
     connection.commit()
     connection.close()
 
+def get_history(db_settings):
+    connection = connect_db(db_settings)
+    cursor = connection.cursor()
+    cursor.execute("""
+    SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp
+    FROM history
+    """)
+    results = cursor.fetchall()
+    cursor.close()
+    connection.close()
+    return results
+
+def get_item(db_settings):
+    connection = connect_db(db_settings)
+    cursor = connection.cursor()
+    cursor.execute("""
+    SELECT uuid, itemid, skuid, choice, attributes, image
+    FROM item
+    """)
+    results = cursor.fetchall()
+    cursor.close()
+    connection.close()
+    return results
 
 def export_csv(db_settings):
     connection = connect_db(db_settings)
     cursor = connection.cursor()
 
     cursor.execute("""
-    SELECT i.itemid, i.skuid, i.choice, i.attributes, i.image, h.quantity, h.discount_percentage, h.price, h.currency, h.h_timestamp
+    SELECT i.uuid, i.itemid, i.skuid, i.choice, i.attributes, i.image, h.quantity, h.discount_percentage, h.price, h.currency, h.h_timestamp
     FROM item i, history h
-    WHERE i.itemid = h.itemid and i.skuid = h.skuid
+    WHERE i.uuid = h.uuid
     """)
     results = cursor.fetchall()
     with open(os.path.dirname(os.path.realpath(__file__))+"/output.csv", 'w') as csv_file:
@@ -78,28 +109,36 @@ def initialize(db_settings):
     DROP TABLE IF EXISTS item
     """)
     cursor.execute("""
+    DROP SEQUENCE IF EXISTS uuid_sequence
+    """)
+    cursor.execute("""
+    CREATE SEQUENCE uuid_sequence
+    INCREMENT BY 1
+    START WITH 1
+    """)
+    cursor.execute("""
     CREATE TABLE item
     (
+        uuid    int,
         itemid  bigint,
         skuid   bigint,
         choice  boolean,
         attributes  text[],
         image   text,
-        primary key (itemid,skuid)
+        primary key (uuid)
     )
     """)
     cursor.execute("""
     CREATE TABLE history
     (
-        itemid  bigint,
+        uuid    int,
-        skuid   bigint,
         quantity  integer,
         discount_percentage numeric(2),
         price   money,
         currency    varchar(4),
         h_timestamp timestamp,
-        foreign key (itemid,skuid) references item(itemid,skuid),
+        foreign key (uuid) references item(uuid),
-        primary key (itemid,skuid,h_timestamp)
+        primary key (uuid, h_timestamp)
     )
     """)
 

gunicorn_config.py

@@ -0,0 +1,17 @@
+import os
+
+
+
+workers = int(os.environ.get('GUNICORN_PROCESSES', '2'))
+
+threads = int(os.environ.get('GUNICORN_THREADS', '4'))
+
+# timeout = int(os.environ.get('GUNICORN_TIMEOUT', '120'))
+
+bind = os.environ.get('GUNICORN_BIND', '0.0.0.0:8080')
+
+
+
+forwarded_allow_ips = '*'
+
+secure_scheme_headers = { 'X-Forwarded-Proto': 'https' }

web/app.js

@@ -16,24 +16,69 @@ const width = window.innerWidth - margin.right - margin.left -10;
 // (resizes every elements)
 window.onresize = function(){ location.reload(); }
 
-d3.csv('http://127.0.0.1/output.csv', d => {
+async function fetch_history() {
-    return {
+  try {
-        date: new Date(d.h_timestamp.replace(' ', 'T')),
+    const response = await fetch('http://127.0.0.1:8080/datahistory');
-        value: parseFloat(d.price.replace('$', '')),
+    const rawData = await response.json();
-        skuid: d.skuid,
+    // SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp
-        itemid: d.itemid,
+    let historyData = rawData.map(d => ({
-        image: d.image,
+        uuid: d[0],
-        currency: d.currency
+        value: parseFloat(d[3].replace('$', '')),
+        currency: d[4],
+        date: new Date(d[5].replace(' ', 'T')),
+    }));
+    return historyData;
+  } catch (error) {
+    console.error('Error fetching data history: ', error);
+    throw error;
   }
-}).then(function(data) {
+}
+
+async function fetch_item() {
+  try {
+    // SELECT uuid, itemid, skuid, choice, attributes, image
+    const response = await fetch('http://127.0.0.1:8080/dataitem');
+    const rawData = await response.json();
+    const items = rawData.reduce((item, row) => {
+      const uuid = row[0];
+
+      const itemid = row[1];
+      const skuid = row[2];
+      const choice = row[3];
+      const attributes = row[4];
+      const image = row[5];
+
+      const values = {
+        itemid: itemid,
+        skuid: skuid,
+        choice: choice,
+        attributes: attributes,
+        image: image
+      };
+
+      item[uuid] = values;
+
+      return item;
+    }, {});
+
+    return items;
+  } catch (error) {
+    console.error('Error fetching data item: ', error);
+    throw error;
+  }
+}
+
+
+fetch_history().then(async function(data) {
+    items = await fetch_item();
 
     // Date domain (width)
     const x = d3.scaleTime()
         .domain(d3.extent(data, d => d.date))
         .range([3, width-3]);
 
-    // Group the data by (itemid, skuid)
+    // Group the data by (uuid)
-    const nestedData = d3.group(data, d => d.skuid);
+    const nestedData = d3.group(data, d => d.uuid);
     // Create a div for each graph
     const graphDivs = d3.select("#graphs")
         .selectAll(".graph-container")
@@ -58,16 +103,17 @@ d3.csv('http://127.0.0.1/output.csv', d => {
         const svg = d3.select(this);
 
         // context on right side
-        // text
+        const link = `https://fr.aliexpress.com/item/${items[key].itemid}.html`;
-        const link = `https://fr.aliexpress.com/item/${dataSubset[0].itemid}.html`;
         // image
         svg.append("image")
             .attr("x", width + margin.right*0.1)
             .attr("y", height*0.1)
             .attr("width", height*0.8)
             .attr("height", height*0.8)
-            .attr("xlink:href", dataSubset[0].image) // placeholder picture for now, should be item picture
+            .attr("xlink:href", items[key].image)
-            .on("click", function() { window.open(link); });
+            .on("click", function() {
+                window.open(link, '_blank', 'noopener,noreferrer');
+            });
 
         // Price domain (height)
         const y = d3.scaleLinear()