comments

aliexpress.py

@@ -3,6 +3,7 @@ import requests, re, json, os, yaml
 from db import *
 
 def load_cookies_from_file(file_path):
+    '''load cookies from a file and return a dict usable in a request session'''
     with open(file_path, 'r') as file:
         cookies_data = json.load(file)
 
@@ -19,6 +20,14 @@ def load_cookies_from_file(file_path):
     return cookies_dict
 
 def check_items(settings_items):
+    '''
+    return a dict with items data extracted from aliexpress.
+    a file containing aliexpress login token cookies has to be provided in ./cookies.json (obtained with cookie-quick-manager https://github.com/ysard/cookie-quick-manager/) for accurate prices (no "welcome discount")
+    extracted data:
+    skuId, quantity, discount_percentage, price, currency, choice_delivery, image
+    parameter settings_item is a list of tables (string(itemid), attributes)
+    itemid is in aliexpress link to item page. attributes is a list of string. Each string is a choice value (for example which length, or which colour) if multiple items are on the same page, only one by category, order doesn't matter.
+    '''
 
     item_regex = re.compile(r'skuAttr\\\":\\\"(([0-9]*:[0-9]*#[a-zA-Z0-9 ]*;?)*)\\\",\\\"skuId\\\":([0-9]*),\\\"skuIdStr\\\":\\\"[0-9]*\\\",\\\"skuPropIds\\\":\\\"[0-9,]*\\\",\\\"skuVal\\\":{\\\"availQuantity\\\":([0-9]*),(\\\"discount\\\":\\\"([0-9]*)\\\",\\\"discountTips\\\":\\\"-[0-9]*%\\\",)?\\\"hideOriPrice\\\":(false|true),\\\"inventory\\\":([0-9]*),\\\"isActivity\\\":(true|false),\\\"optionalWarrantyPrice\\\":\[\],(\\\"skuActivityAmount\\\":{\\\"currency\\\":\\\"([A-Z]*)\\\",\\\"formatedAmount\\\":\\\"[0-9]*,[0-9]*.\\\",\\\"value\\\":([0-9]*\.[0-9]*)},\\\"skuActivityAmountLocal\\\":\\\"[0-9]*,[0-9]*.\|[0-9]*\|[0-9]*\\\",)?\\\"skuAmount\\\":{\\\"currency\\\":\\\"([A-Z]*)\\\",\\\"formatedAmount\\\":\\\"[0-9]*,[0-9]*.\\\",\\\"value\\\":([0-9]*\.[0-9]*)}')
     choice_regex = re.compile(r'businessModel\\\":\\\"CHOICE\\\"')
@@ -60,6 +69,7 @@ def check_items(settings_items):
     return extract
 
 def get_attributes(attributes_raw):
+    '''return a list of attributes from attributes raw string'''
     # id_regex = re.compile(r'([0-9]*)=')
     attr_regex = re.compile(r'#([0-9a-zA-Z ]*)')
 
@@ -69,6 +79,7 @@ def get_attributes(attributes_raw):
     return attributes
 
 def fill_db(db_settings, items_dict):
+    '''add new history entries in database with data extracted in item_dict'''
     for key,value in items_dict.items():
         add_history_entry(db_settings, key[0], value["skuid"], value["choice_delivery"], list(key[1]), value["image"], value["price"], value["currency"], value["quantity"], value["discount_percentage"])
 

db.py

@@ -4,6 +4,7 @@ import csv
 import os
 
 def connect_db(db_settings):
+    '''open and return a connection to the database'''
     conn = None
     print("Connecting to the PostgreSQL database...")
     try:
@@ -14,6 +15,7 @@ def connect_db(db_settings):
     return conn
 
 def add_item(db_settings, itemid, skuid, choice, attributes, image):
+    '''insert a new item in the database'''
     connection = connect_db(db_settings)
     cursor = connection.cursor()
 
@@ -25,6 +27,7 @@ def add_item(db_settings, itemid, skuid, choice, attributes, image):
     connection.close()
 
 def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, price, currency, quantity, discount_percentage):
+    '''Add a new history entry for an item in the database. If item isn't in database yet, add it.'''
     connection = connect_db(db_settings)
     cursor = connection.cursor()
 
@@ -54,6 +57,7 @@ def add_history_entry(db_settings, itemid, skuid, choice, attributes, image, pri
     connection.close()
 
 def get_history(db_settings):
+    '''return history data from database'''
     connection = connect_db(db_settings)
     cursor = connection.cursor()
     cursor.execute("""
@@ -66,6 +70,7 @@ def get_history(db_settings):
     return results
 
 def get_item(db_settings):
+    '''return items data from database'''
     connection = connect_db(db_settings)
     cursor = connection.cursor()
     cursor.execute("""
@@ -78,6 +83,7 @@ def get_item(db_settings):
     return results
 
 def export_csv(db_settings):
+    '''join item and history data from database and export it in ./output.csv'''
     connection = connect_db(db_settings)
     cursor = connection.cursor()
 
@@ -99,6 +105,7 @@ def export_csv(db_settings):
 
 
 def initialize(db_settings):
+    '''Create tables and sequence in database. Drop them first if they already exist.'''
     connection = connect_db(db_settings)
     cursor = connection.cursor()