From f298da7ac5ee6199880ffc811810b3030eb11b2f Mon Sep 17 00:00:00 2001 From: Sam Hadow Date: Thu, 28 Dec 2023 17:39:36 +0100 Subject: [PATCH] fill database with items in settings.yaml --- .gitignore | 1 + aliexpress.py | 37 ++++++++++++++++++------- db.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++--- main.py | 5 ++-- settings.yaml | 3 -- 5 files changed, 103 insertions(+), 20 deletions(-) delete mode 100644 settings.yaml diff --git a/.gitignore b/.gitignore index 8f9adc0..179512d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cookies.json test*.txt settings.yaml +__pycache__/* diff --git a/aliexpress.py b/aliexpress.py index 014d5f9..c712fb2 100644 --- a/aliexpress.py +++ b/aliexpress.py @@ -1,5 +1,6 @@ #!/usr/bin/python import requests, re, json, os, yaml +from db import * def load_cookies_from_file(file_path): with open(file_path, 'r') as file: @@ -17,7 +18,7 @@ def load_cookies_from_file(file_path): return cookies_dict -def check_items(settings_id): +def check_items(settings_items): item_regex = re.compile(r'skuAttr\\\":\\\"(([0-9]*:[0-9]*#[a-zA-Z0-9 ]*;?)*)\\\",\\\"skuId\\\":([0-9]*),\\\"skuIdStr\\\":\\\"[0-9]*\\\",\\\"skuPropIds\\\":\\\"[0-9,]*\\\",\\\"skuVal\\\":{\\\"availQuantity\\\":([0-9]*),(\\\"discount\\\":\\\"([0-9]*)\\\",\\\"discountTips\\\":\\\"-[0-9]*%\\\",)?\\\"hideOriPrice\\\":(false|true),\\\"inventory\\\":([0-9]*),\\\"isActivity\\\":(true|false),\\\"optionalWarrantyPrice\\\":\[\],(\\\"skuActivityAmount\\\":{\\\"currency\\\":\\\"([A-Z]*)\\\",\\\"formatedAmount\\\":\\\"[0-9]*,[0-9]*.\\\",\\\"value\\\":([0-9]*\.[0-9]*)},\\\"skuActivityAmountLocal\\\":\\\"[0-9]*,[0-9]*.\|[0-9]*\|[0-9]*\\\",)?\\\"skuAmount\\\":{\\\"currency\\\":\\\"([A-Z]*)\\\",\\\"formatedAmount\\\":\\\"[0-9]*,[0-9]*.\\\",\\\"value\\\":([0-9]*\.[0-9]*)}') choice_regex = re.compile(r'businessModel\\\":\\\"CHOICE\\\"') @@ -30,21 +31,37 @@ def check_items(settings_id): extract = dict() - for item in settings_id: + for (item,filter_attributes) in settings_items: url = 'https://aliexpress.com/item/'+item+'.html' target_page_response = session.get(url) - if target_page_response.status_code == 200: content = re.findall(item_regex, target_page_response.text) is_choice = bool(re.search(choice_regex, target_page_response.text)) for elem in content: - key = item+':'+elem[0] - discount = 0 if len(elem[5]) == 0 else int(elem[5]) - price = float(elem[13]) if len(elem[11]) == 0 else float(elem[11]) + if set(get_attributes(elem[0])) == set(filter_attributes): + key = (item,tuple(filter_attributes)) + discount = 0 if len(elem[5]) == 0 else int(elem[5]) + price = float(elem[13]) if len(elem[11]) == 0 else float(elem[11]) - # skuId, quantity, discount_percentage, price, currency, choice_delivery - extract[key] = (elem[2], elem[3], discount, price, elem[12], is_choice) - return extract + # skuId, quantity, discount_percentage, price, currency, choice_delivery + extract[key] = {"skuid": elem[2], "quantity": elem[3], "discount_percentage": discount, "price": price, "currency": elem[12], "choice_delivery": is_choice} else: print(f'Failed to fetch target page. Status code: {target_page_response.status_code}') - return None + return extract + +def get_attributes(attributes_raw): + # id_regex = re.compile(r'([0-9]*)=') + attr_regex = re.compile(r'#([0-9a-zA-Z ]*)') + + # item_id = re.search(id_regex, attributes_raw).group(1) + attributes = re.findall(attr_regex, attributes_raw) + + return attributes + +def fill_db(db_settings, items_dict): + for key,value in items_dict.items(): + add_history_entry(db_settings, key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["price"], value["currency"], value["quantity"], value["discount_percentage"]) + + + + diff --git a/db.py b/db.py index ec5dd17..424cae4 100644 --- a/db.py +++ b/db.py @@ -11,7 +11,76 @@ def connect_db(db_settings): print(error) return conn -def close_db(conn): - if conn is not None: - conn.close() - print("Connection closed") +def add_item(db_settings, itemid, skuid, choice, attributes): + connection = connect_db(db_settings) + cursor = connection.cursor() + + cursor.execute(""" + INSERT INTO item (itemid, skuid, choice, attributes) + VALUES (%s, %s, %s, %s) + """, (itemid, skuid, choice, attributes)) + connection.commit() + connection.close() + +def add_history_entry(db_settings, itemid, skuid, choice, attributes, price, currency, quantity, discount_percentage): + connection = connect_db(db_settings) + cursor = connection.cursor() + + cursor.execute(""" + SELECT * + FROM item + WHERE itemid = %s + AND skuid = %s + """, (itemid, skuid)) + + if cursor.rowcount == 0: + add_item(db_settings, itemid, skuid, choice, attributes) + + cursor.execute(""" + INSERT INTO history (itemid, skuid, price, currency, quantity, discount_percentage, h_timestamp) + VALUES (%s, %s, %s, %s, %s, %s, (SELECT LOCALTIMESTAMP)) + """, (itemid, skuid, price, currency, quantity, discount_percentage)) + connection.commit() + connection.close() + + + + +def initialize(db_settings): + connection = connect_db(db_settings) + cursor = connection.cursor() + + cursor.execute(""" + DROP TABLE IF EXISTS history + """) + cursor.execute(""" + DROP TABLE IF EXISTS item + """) + cursor.execute(""" + CREATE TABLE item + ( + itemid bigint, + skuid bigint, + choice boolean, + attributes text[], + primary key (itemid,skuid) + ) + """) + cursor.execute(""" + CREATE TABLE history + ( + itemid bigint, + skuid bigint, + quantity integer, + discount_percentage numeric(2), + price money, + currency varchar(4), + h_timestamp timestamp, + foreign key (itemid,skuid) references item(itemid,skuid), + primary key (itemid,skuid,h_timestamp) + ) + """) + + connection.commit() + connection.close() + print("Database initialized") diff --git a/main.py b/main.py index 0e91809..1d17b3f 100644 --- a/main.py +++ b/main.py @@ -14,8 +14,7 @@ if __name__ == '__main__': settings = get_conf() - print(check_items(settings["id"])) + #initialize(settings["db"]) + fill_db(settings["db"], check_items(settings["item"])) - connection = connect_db(settings["db"]) - close_db(connection) diff --git a/settings.yaml b/settings.yaml deleted file mode 100644 index 2754e5a..0000000 --- a/settings.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: - - "1005005769229528" - - "4001259224639"