diff --git a/aliexpress.py b/aliexpress.py index 96ab9cc..8d17620 100644 --- a/aliexpress.py +++ b/aliexpress.py @@ -3,6 +3,7 @@ import requests, re, json, os, yaml from db import * def load_cookies_from_file(file_path): + '''load cookies from a file and return a dict usable in a request session''' with open(file_path, 'r') as file: cookies_data = json.load(file) @@ -19,6 +20,14 @@ def load_cookies_from_file(file_path): return cookies_dict def check_items(settings_items): + ''' + return a dict with items data extracted from aliexpress. + a file containing aliexpress login token cookies has to be provided in ./cookies.json (obtained with cookie-quick-manager https://github.com/ysard/cookie-quick-manager/) for accurate prices (no "welcome discount") + extracted data: + skuId, quantity, discount_percentage, price, currency, choice_delivery, image + parameter settings_item is a list of tables (string(itemid), attributes) + itemid is in aliexpress link to item page. attributes is a list of string. Each string is a choice value (for example which length, or which colour) if multiple items are on the same page, only one by category, order doesn't matter. + ''' item_regex = re.compile(r'skuAttr\\\":\\\"(([0-9]*:[0-9]*#[a-zA-Z0-9 ]*;?)*)\\\",\\\"skuId\\\":([0-9]*),\\\"skuIdStr\\\":\\\"[0-9]*\\\",\\\"skuPropIds\\\":\\\"[0-9,]*\\\",\\\"skuVal\\\":{\\\"availQuantity\\\":([0-9]*),(\\\"discount\\\":\\\"([0-9]*)\\\",\\\"discountTips\\\":\\\"-[0-9]*%\\\",)?\\\"hideOriPrice\\\":(false|true),\\\"inventory\\\":([0-9]*),\\\"isActivity\\\":(true|false),\\\"optionalWarrantyPrice\\\":\[\],(\\\"skuActivityAmount\\\":{\\\"currency\\\":\\\"([A-Z]*)\\\",\\\"formatedAmount\\\":\\\"[0-9]*,[0-9]*.\\\",\\\"value\\\":([0-9]*\.[0-9]*)},\\\"skuActivityAmountLocal\\\":\\\"[0-9]*,[0-9]*.\|[0-9]*\|[0-9]*\\\",)?\\\"skuAmount\\\":{\\\"currency\\\":\\\"([A-Z]*)\\\",\\\"formatedAmount\\\":\\\"[0-9]*,[0-9]*.\\\",\\\"value\\\":([0-9]*\.[0-9]*)}') choice_regex = re.compile(r'businessModel\\\":\\\"CHOICE\\\"') @@ -60,6 +69,7 @@ def check_items(settings_items): return extract def get_attributes(attributes_raw): + '''return a list of attributes from attributes raw string''' # id_regex = re.compile(r'([0-9]*)=') attr_regex = re.compile(r'#([0-9a-zA-Z ]*)') @@ -69,6 +79,7 @@ def get_attributes(attributes_raw): return attributes def fill_db(db_settings, items_dict): + '''add new history entries in database with data extracted in item_dict''' for key,value in items_dict.items(): add_history_entry(db_settings, key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["image"], value["price"], value["currency"], value["quantity"], value["discount_percentage"]) diff --git a/db.py b/db.py index f29ba54..61bc0c8 100644 --- a/db.py +++ b/db.py @@ -4,6 +4,7 @@ import csv import os def connect_db(db_settings): + '''open and return a connection to the database''' conn = None print("Connecting to the PostgreSQL database...") try: @@ -14,6 +15,7 @@ def connect_db(db_settings): return conn def add_item(db_settings, itemid, skuid, choice, attributes, image): + '''insert a new item in the database''' connection = connect_db(db_settings) cursor = connection.cursor() @@ -25,6 +27,7 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image): connection.close() def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, price, currency, quantity, discount_percentage): + '''Add a new history entry for an item in the database. If item isn't in database yet, add it.''' connection = connect_db(db_settings) cursor = connection.cursor() @@ -54,6 +57,7 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri connection.close() def get_history(db_settings): + '''return history data from database''' connection = connect_db(db_settings) cursor = connection.cursor() cursor.execute(""" @@ -66,6 +70,7 @@ def get_history(db_settings): return results def get_item(db_settings): + '''return items data from database''' connection = connect_db(db_settings) cursor = connection.cursor() cursor.execute(""" @@ -78,6 +83,7 @@ def get_item(db_settings): return results def export_csv(db_settings): + '''join item and history data from database and export it in ./output.csv''' connection = connect_db(db_settings) cursor = connection.cursor() @@ -99,6 +105,7 @@ def export_csv(db_settings): def initialize(db_settings): + '''Create tables and sequence in database. Drop them first if they already exist.''' connection = connect_db(db_settings) cursor = connection.cursor()