main.py

import datetime
import multiprocessing

from collections import OrderedDict

from scraper import Scraper, DigitecScraper, MicrospotScraper, ConradScraper, PCOstschweizScraper
from datastructures import Product, ProductCompany, Price, Company, PriceChanges, PriceChangesSimple,Base, BaseSimple

from sqlalchemy import create_engine, asc, Index
from sqlalchemy.orm import sessionmaker

import matplotlib.pyplot as plt
import matplotlib.dates as mdates

from preispirat import Preispirat

def binary_search(prices, value):
    """
    :param prices: list of prices
    :param value: value to retrieve range
    :return: index range of value
    """
    l_leftmost = 0
    r = len(prices)
    while l_leftmost < r:
        mid = int((l_leftmost + r) / 2)
        if prices[mid].product_company_id < value:
            l_leftmost = mid+1
        else:
            r = mid

    l_rightmost = 0
    r = len(prices)
    while l_rightmost < r:
        mid = int((l_rightmost + r) / 2)
        if prices[mid].product_company_id > value:
            r = mid
        else:
            l_rightmost = mid+1
    return l_leftmost, l_rightmost - 1

def price_analyser_biggest_change():
    biggest_changes = []
    counter = 0
    started = datetime.datetime.now()
    prices = session.query(Price).order_by(Price.product_company_id.asc()).order_by(Price.date.asc()).all()
    for product in session.query(Product):
        print(datetime.datetime.now()-started)
        counter += 1
        for product_company in product.product_offered:
            l, r = binary_search(prices, product_company.id)
            print(l, r)
            if r-l <= 0:
                continue

            if prices[r].price != prices[r-1].price:
                biggest_changes.append({"percent_change": 1-prices[r].price/prices[r-1].price, "price": prices[r].price, "last_price": prices[r-1].price, "product_name": product.name, "company": session.query(Company).get(product_company.company_id).name, "id": product_company.tag, "date": prices[r].date,"product": product, "product_company": product_company, "price_today": prices[r].id, "price_yesterday": prices[r-1].id})
                print("Found price change")
    biggest_changes.sort(key=lambda x: x["percent_change"])
    [print(i) for i in biggest_changes]
    print("Took %s seconds to query all prices" % (datetime.datetime.now()-started))
    return biggest_changes

def price_analyser_biggest_change_overall(mode = 1, show_graph = False):
    biggest_changes = []
    counter = 0
    started = datetime.datetime.now()
    for product in session.query(Product):
        print(counter)
        counter+=1
        for product_company in product.product_offered:
            biggest_changes.append([0, 0, product.name, session.query(Company).get(product_company.company_id).name, product_company.tag, product])
            prices = product_company.prices

            if len(prices) == 0:
                continue
            last_price = prices[0]
            for price in prices:
                if last_price.price != price.price:
                    print("Product: %s \tCompany: %s \tID: %s"%(product.name, session.query(Company).get(product_company.company_id).name, product_company.tag))
                    print("Price before: %f \tnow: %f \tdate: %s"%(last_price.price, price.price, price.date))
                    biggest_changes[-1][0] += abs(price.price-last_price.price)
                    biggest_changes[-1][1] += abs(1-price.price/last_price.price)
                last_price = price
    biggest_changes.sort(key=lambda x:x[mode])
    [print(i) for i in biggest_changes]

    for i in range(20):
        if not show_graph: break
        product = session.query(Product).get(biggest_changes[-1-i][-1])
        plt.figure(product.id)
        plt.suptitle(product.name)
        for product_company in product.product_offered:
            prices = product_company.prices
            if len(prices) == 0:
                continue
            x = []
            y = []
            for price in prices:
                #x.append(mdates.date2num(datetime.date(price.date.year, price.date.month, price.date.day)))
                x.append(mdates.date2num(price.date))
                y.append(price.price)
            plt.plot(x, y, label=session.query(Company).get(product_company.company_id).name)
            plt.gcf().autofmt_xdate()
            myFmt = mdates.DateFormatter('%D')
            plt.gca().xaxis.set_major_formatter(myFmt)
            plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2, mode="expand", borderaxespad=0.)
    plt.show()
    print("Took %s seconds to query all prices"%(datetime.datetime.now()-started))
    return biggest_changes

def scrape_images():
    for product in session.query(Product).all():
        product.url_image = digitec_scraper.scrape_image_product(product)
        print(product.url_image)

def get_pricegraph(product):
    plt.figure(product.id)
    plt.suptitle(product.name)
    for product_company in product.product_offered:
        prices = session.query(Price).filter(Price.product_company_id == product_company.id).order_by(asc(Price.date)).all()

        if len(prices) == 0:
            continue
        x = []
        y = []
        for price in prices:
            x.append(mdates.date2num(price.date))
            y.append(price.price)
        plt.plot(x, y, label=session.query(Company).get(product_company.company_id).name)
        plt.gcf().autofmt_xdate()
        my_fmt = mdates.DateFormatter('%D')
        plt.gca().xaxis.set_major_formatter(my_fmt)
        plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2, mode="expand", borderaxespad=0.)
    plt.show()

def zero_sum(timeframe = [0,0]):
    #parser.parse('', dayfirst=True)
    start_day = session.query(Price).order_by(Price.date.asc()).first()
    end_day = session.query(Price).order_by(Price.date.desc()).first()

    date_dict = OrderedDict()
    for day in range((end_day.date - start_day.date + datetime.timedelta(days=1)).days):
        day = start_day.date + day*datetime.timedelta(days=1)
        sum_dict = {}
        for company in session.query(Company):
            sum_dict[company.name] = 0
        date_dict[datetime.datetime(day=day.day, year=day.year, month=day.month)] = sum_dict

    for company in session.query(Company):
        sum = 0
        for product_company in session.query(ProductCompany).filter(ProductCompany.company == company):
            last_price = None
            for price in session.query(Price).filter(Price.product_company_id == product_company.id):
                if last_price == None: last_price = price.price
                if price.price / last_price != 1:
                    sum += price.price - last_price
                    last_price = price.price

                try:
                    date_dict[datetime.datetime(day=price.date.day, year=price.date.year, month=price.date.month)][company.name] = sum
                except:
                    pass
        x = []
        y = []
        for date, sum_dict in date_dict.items():
            x.append(mdates.date2num(date))
            y.append(sum_dict[company.name])

        plt.plot(x, y, label=company.name)
        plt.gcf().autofmt_xdate()
        my_fmt = mdates.DateFormatter('%D')
        plt.gca().xaxis.set_major_formatter(my_fmt)
        plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2, mode="expand", borderaxespad=0.)

        plt.show()

        print("Sum of company %s : %f"%(company.name, sum))
        print(date_dict)
    plt.show()

def delete_prices_of_day():
    print(datetime.date.today())
    for price in session.query(Price):
        if (price.date.date() == datetime.date.today()):
            print(price.id, price.date, price.date.date() == datetime.date.today())
            session.delete(price)

def preispiratTest(testProduct):
    lol = datetime.datetime.now()
    #testProduct = session.query(ProductCompany).join(Product).join(Company).filter(and_(Product.manufacturer_id == 'DELL-U2718Q', Company.name == 'Digitec')).first()
    #print(digitec_scraper.get_latest_price(testProduct))
    #testProduct.url = 'https://www.digitec.ch/de/s1/product/dell-ultrasharp-u2718q-27-3840-x-2160-pixels-monitor-6412351'
    test = Preispirat()
    test.uploadProduct(testProduct)

def url_to_product():
    for product_company in digitec.stock:
        product_company.url = digitec_scraper.url_product(product_company)
    for product_company in microspot.stock:
        product_company.url = 'https://www.microspot.ch/msp/products/' + product_company.tag
    for product_company in conrad.stock:
        product_company.url = conrad_scraper.url_product(product_company)

def scrape_products():
    for product in session.query(Product).all():
        pcostschweiz_scraper.scrape_by_manufacturer_id(product, True)

def add_day_price_changes(price_change_dict):
    product_companies = session.query(ProductCompany)\
                        .join(Product)\
                        .distinct(ProductCompany.id)\
                        .filter(Product.id == price_change_dict['product_company'].product_id).all()

    other_best_price = -1

    for pc in product_companies:
        if pc.id is not price_change_dict['product_company'].id:
            pc_price = session.query(Price) \
                .filter(Price.product_company_id == pc.id) \
                .order_by(Price.date.desc()) \
                .first()

            if isinstance(other_best_price, int) or pc_price.price < other_best_price.price:
                other_best_price = pc_price


    if other_best_price is not None and (isinstance(other_best_price, int) or other_best_price.price > price_change_dict['price']):
        if isinstance(other_best_price, int) or other_best_price.price > price_change_dict['last_price']:
            other_best_price = price_change_dict['price_yesterday']

        pc = PriceChanges(date=datetime.datetime.today(),
                            price_today_id=price_change_dict['price_today'],
                            price_yesterday_id=(other_best_price if isinstance(other_best_price, int) else other_best_price.id),
                            percent_change=(2 - price_change_dict['price']/other_best_price.price if not isinstance(other_best_price, int) else price_change_dict['percent_change'] + 1),
                            product_company_id=price_change_dict['product_company'].id)
        return pc
    else:
        return None

def PriceChangeToSimple(price_change):
    if not isinstance(price_change, PriceChanges):
        raise TypeError()
    price_change_simple = PriceChangesSimple(product_company=session.query(ProductCompany).get(price_change.product_company_id),
                                                price_today=session.query(Price).get(price_change.price_today_id),
                                                price_yesterday=session.query(Price).get(price_change.price_yesterday_id),
                                                product=session.query(ProductCompany).get(price_change.product_company_id).product
                                                )
    return price_change_simple

if __name__ == "__main__":
    try:
        engine = create_engine('postgresql://postgres:admin@localhost:5432/pricetracker_database')
        Base.metadata.create_all(engine)
        Session = sessionmaker()
        session = Session(bind=engine)

        engine_simple = create_engine('postgres://hdlubjhvpibagw:68fde9cba5c79eeca8eba0c52528f095ca1ffb912c12095104ce03809b8f2939@ec2-54-247-178-166.eu-west-1.compute.amazonaws.com:5432/d3p7ql5olgpc0j')
        BaseSimple.metadata.create_all(engine_simple)
        session_simple = Session(bind=engine_simple)

        digitec = session.query(Company).filter(Company.name == 'Digitec').first()
        microspot = session.query(Company).filter(Company.name == 'Microspot').first()
        conrad = session.query(Company).filter(Company.name == 'Conrad').first()
        pcostschweiz = session.query(Company).filter(Company.name == 'PCOstschweiz').first()

        digitec_scraper = DigitecScraper(digitec.url, digitec.scrape_url, digitec.id)
        microspot_scraper = MicrospotScraper(microspot.url, microspot.scrape_url, microspot.id)
        conrad_scraper = ConradScraper(conrad.url, conrad.scrape_url, conrad.id)
        pcostschweiz_scraper = PCOstschweizScraper(pcostschweiz.url, pcostschweiz.scrape_url, pcostschweiz.id)

        #scrape_products()

        if True:
            pcostschweiz_thread = multiprocessing.Process(target=pcostschweiz_scraper.scrape_for_day, name="PCOstschweiz Process")
            digitec_thread = multiprocessing.Process(target=digitec_scraper.scrape_for_day, name="Digitec Process")
            microspot_thread = multiprocessing.Process(target=microspot_scraper.scrape_for_day, name="Microspot Process")
            conrad_thread = multiprocessing.Process(target=conrad_scraper.scrape_for_day, name="Conrad Process")

            processes = [pcostschweiz_thread, digitec_thread, microspot_thread, conrad_thread]

            [process.start() for process in processes]
            [process.join() for process in processes]

            #results = session_simple.query(PriceChangesSimple).all()
            #[session_simple.delete(x) for x in results]

        #delete_prices_of_day()

        #preispiratTest()

        #digitec_scraper.scrape_tag_category_products(591, 1000, 0)

        #scrape_images()


        #result = price_analyser_biggest_change_overall(0) # 0 sort for absolute change, 1 sort for percent change
        #get_pricegraph(res)
        #zero_sum()
        #print(digitec_scraper.get_toppreise(session.query(Product).filter(Product.manufacturer_id=='DELL-U2718Q').first()))

        #url_to_product()


        result = price_analyser_biggest_change()

        new_changes = list()
        for i in reversed(result):
            if i['percent_change'] < 0:
                break
            if i['date'].date() != datetime.date.today():
                continue
            if True:
                change = add_day_price_changes(i)
                if change is not None:
                    print(i)
                    new_changes.append(change)

        counter = 0
        for change in sorted(new_changes, key=lambda x: x.percent_change, reverse=True):
            if counter < 100:
                print(change.percent_change)
                session_simple.add(PriceChangeToSimple(change))
                session.add(change)
                #get_pricegraph(i['product'])
                #preispiratTest(i['product_company'])
            else:
                break
            counter+=1

        #zero_sum()

    finally:
        session.commit()
        session_simple.commit()
        session.close()
        session_simple.close()
        Scraper.driver.quit()