sam.hadow/aliexpress-price-tracker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

flask server, gunicorn, fetch data in JS

Browse Source
This commit is contained in:
Sam Hadow 2024-01-25 23:52:23 +01:00
parent 2ed92e5412
commit b5ad33f138
5 changed files with 165 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
app.py Normal file
View File

@ -0,0 +1,33 @@
from flask import Flask, request, jsonify
import requests, re, json, os, yaml
from db import *
from aliexpress import *
app = Flask(__name__)
def get_conf():
'''return settings in settings.yaml file'''
with open(os.path.dirname(os.path.realpath(__file__))+"/settings.yaml", 'r') as conf_file:
settings = yaml.safe_load(conf_file)
return settings
@app.route('/')
def hello_world():
return 'Hello, World!'
@app.route('/datahistory',methods = ['POST', 'GET'])
def data_history_request():
if request.method == 'GET':
print("fetching data history")
settings = get_conf()
return jsonify(get_history(settings["db"]))
@app.route('/dataitem',methods = ['POST', 'GET'])
def data_item_request():
if request.method == 'GET':
print("fetching data item")
settings = get_conf()
return jsonify(get_item(settings["db"]))
if __name__ == '__main__':
app.run(debug = True)

65
db.py
View File

@ -18,8 +18,8 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image):
cursor = connection.cursor()
cursor.execute("""
INSERT INTO item (itemid, skuid, choice, attributes, image)
VALUES (%s, %s, %s, %s, %s)
INSERT INTO item (uuid, itemid, skuid, choice, attributes, image)
VALUES (nextval('uuid_sequence'), %s, %s, %s, %s, %s)
""", (itemid, skuid, choice, attributes, image))
connection.commit()
connection.close()
@ -29,7 +29,7 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri
cursor = connection.cursor()
cursor.execute("""
SELECT *
SELECT uuid
FROM item
WHERE itemid = %s
AND skuid = %s
@ -37,23 +37,54 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri
if cursor.rowcount == 0:
add_item(db_settings, itemid, skuid, choice, attributes, image)
cursor.execute("""
SELECT uuid
FROM item
WHERE itemid = %s
AND skuid = %s
""", (itemid, skuid))
uuid = cursor.fetchall()[0]
cursor.execute("""
INSERT INTO history (itemid, skuid, price, currency, quantity, discount_percentage, h_timestamp)
VALUES (%s, %s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP))
""", (itemid, skuid, price, currency, quantity, discount_percentage))
INSERT INTO history (uuid, price, currency, quantity, discount_percentage, h_timestamp)
VALUES (%s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP))
""", (uuid, price, currency, quantity, discount_percentage))
connection.commit()
connection.close()
def get_history(db_settings):
connection = connect_db(db_settings)
cursor = connection.cursor()
cursor.execute("""
SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp
FROM history
""")
results = cursor.fetchall()
cursor.close()
connection.close()
return results
def get_item(db_settings):
connection = connect_db(db_settings)
cursor = connection.cursor()
cursor.execute("""
SELECT uuid, itemid, skuid, choice, attributes, image
FROM item
""")
results = cursor.fetchall()
cursor.close()
connection.close()
return results
def export_csv(db_settings):
connection = connect_db(db_settings)
cursor = connection.cursor()
cursor.execute("""
SELECT i.itemid, i.skuid, i.choice, i.attributes, i.image, h.quantity, h.discount_percentage, h.price, h.currency, h.h_timestamp
SELECT i.uuid, i.itemid, i.skuid, i.choice, i.attributes, i.image, h.quantity, h.discount_percentage, h.price, h.currency, h.h_timestamp
FROM item i, history h
WHERE i.itemid = h.itemid and i.skuid = h.skuid
WHERE i.uuid = h.uuid
""")
results = cursor.fetchall()
with open(os.path.dirname(os.path.realpath(__file__))+"/output.csv", 'w') as csv_file:
@ -78,28 +109,36 @@ def initialize(db_settings):
DROP TABLE IF EXISTS item
""")
cursor.execute("""
DROP SEQUENCE IF EXISTS uuid_sequence
""")
cursor.execute("""
CREATE SEQUENCE uuid_sequence
INCREMENT BY 1
START WITH 1
""")
cursor.execute("""
CREATE TABLE item
(
uuid int,
itemid bigint,
skuid bigint,
choice boolean,
attributes text[],
image text,
primary key (itemid,skuid)
primary key (uuid)
)
""")
cursor.execute("""
CREATE TABLE history
(
itemid bigint,
skuid bigint,
uuid int,
quantity integer,
discount_percentage numeric(2),
price money,
currency varchar(4),
h_timestamp timestamp,
foreign key (itemid,skuid) references item(itemid,skuid),
primary key (itemid,skuid,h_timestamp)
foreign key (uuid) references item(uuid),
primary key (uuid, h_timestamp)
)
""")

17
gunicorn_config.py Normal file
View File

@ -0,0 +1,17 @@
import os
workers = int(os.environ.get('GUNICORN_PROCESSES', '2'))
threads = int(os.environ.get('GUNICORN_THREADS', '4'))
# timeout = int(os.environ.get('GUNICORN_TIMEOUT', '120'))
bind = os.environ.get('GUNICORN_BIND', '0.0.0.0:8080')
forwarded_allow_ips = '*'
secure_scheme_headers = { 'X-Forwarded-Proto': 'https' }

2
main.py
View File

@ -14,7 +14,7 @@ if __name__ == '__main__':
settings = get_conf()
#initialize(settings["db"])
# initialize(settings["db"])
fill_db(settings["db"], check_items(settings["item"]))
export_csv(settings["db"])

76
web/app.js
View File

@ -16,24 +16,69 @@ const width = window.innerWidth - margin.right - margin.left -10;
// (resizes every elements)
window.onresize = function(){ location.reload(); }
d3.csv('http://127.0.0.1/output.csv', d => {
return {
date: new Date(d.h_timestamp.replace(' ', 'T')),
value: parseFloat(d.price.replace('$', '')),
skuid: d.skuid,
itemid: d.itemid,
image: d.image,
currency: d.currency
async function fetch_history() {
try {
const response = await fetch('http://127.0.0.1:8080/datahistory');
const rawData = await response.json();
// SELECT uuid, quantity, discount_percentage, price, currency, h_timestamp
let historyData = rawData.map(d => ({
uuid: d[0],
value: parseFloat(d[3].replace('$', '')),
currency: d[4],
date: new Date(d[5].replace(' ', 'T')),
}));
return historyData;
} catch (error) {
console.error('Error fetching data history: ', error);
throw error;
}
}).then(function(data) {
}
async function fetch_item() {
try {
// SELECT uuid, itemid, skuid, choice, attributes, image
const response = await fetch('http://127.0.0.1:8080/dataitem');
const rawData = await response.json();
const items = rawData.reduce((item, row) => {
const uuid = row[0];
const itemid = row[1];
const skuid = row[2];
const choice = row[3];
const attributes = row[4];
const image = row[5];
const values = {
itemid: itemid,
skuid: skuid,
choice: choice,
attributes: attributes,
image: image
};
item[uuid] = values;
return item;
}, {});
return items;
} catch (error) {
console.error('Error fetching data item: ', error);
throw error;
}
}
fetch_history().then(async function(data) {
items = await fetch_item();
// Date domain (width)
const x = d3.scaleTime()
.domain(d3.extent(data, d => d.date))
.range([3, width-3]);
// Group the data by (itemid, skuid)
const nestedData = d3.group(data, d => d.skuid);
// Group the data by (uuid)
const nestedData = d3.group(data, d => d.uuid);
// Create a div for each graph
const graphDivs = d3.select("#graphs")
.selectAll(".graph-container")
@ -58,16 +103,17 @@ d3.csv('http://127.0.0.1/output.csv', d => {
const svg = d3.select(this);
// context on right side
// text
const link = `https://fr.aliexpress.com/item/${dataSubset[0].itemid}.html`;
const link = `https://fr.aliexpress.com/item/${items[key].itemid}.html`;
// image
svg.append("image")
.attr("x", width + margin.right*0.1)
.attr("y", height*0.1)
.attr("width", height*0.8)
.attr("height", height*0.8)
.attr("xlink:href", dataSubset[0].image) // placeholder picture for now, should be item picture
.on("click", function() { window.open(link); });
.attr("xlink:href", items[key].image)
.on("click", function() {
window.open(link, '_blank', 'noopener,noreferrer');
});
// Price domain (height)
const y = d3.scaleLinear()