From a927ab92f2687c5f73d35936eb80fb91caf3f3e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?King=20K=C3=A9vin?= <kingkevin@cuvoodoo.info>
Date: Thu, 19 Dec 2019 16:18:00 +0100
Subject: [PATCH] Rakefile: add cost target to generate cost estimate

---
 hardware/Rakefile | 200 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)

diff --git a/hardware/Rakefile b/hardware/Rakefile
index f6471a8..092cc2b 100644
--- a/hardware/Rakefile
+++ b/hardware/Rakefile
@@ -8,6 +8,11 @@ Rakefile instead of Makefile for better text file parsing capabilities.
 =end
 require 'rake/clean'
 require 'csv' # to export BOM and costs
+require 'open-uri' # to parse URLs
+require 'open_uri_redirections' # allow redirections
+require 'net/http' # to get sites
+require 'nokogiri' # to parse sites
+require 'json' # to parse GET site responses
 
 # =================
 # project variables
@@ -113,6 +118,11 @@ boms = [ "#{name}.bom.csv" ]
 task :bom => boms
 CLOBBER.include(boms)
 
+desc "generate cost estimte"
+costs = [ "#{name}.cost.csv" ]
+task :cost => costs
+CLOBBER.include(costs)
+
 # ===============
 # file generation
 # ===============
@@ -193,6 +203,91 @@ rule ".bom.csv" => ".sch" do |t|
   end
 end
 
+desc "generate cost estimate from schematic"
+# this version uses Digi-Key, AliExpress, and LCSC
+# Digi-Key is easily scrapable, while Mouser isn't
+# Digi-Key is only one distributor, but the end prices across distributor is often similar
+rule ".cost.csv" => ".sch" do |t|
+  puts "scraping distributor sites to get prices. this may take some time"
+  sellers = ['digikey-id','aliexpress-id','lcsc-id'] # get seller SKU
+  boards = [1, 10, 100] # calculate the price for as many boards
+  total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards
+  unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board
+  # get component information
+  attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get
+  parts = bom2(t.prerequisites[0], attributes) # get field values
+  # put result in CVS
+  CSV.open(t.name, "wb") do |csv|
+    csv << ["refdes", "quantity", "manufacturer", "part number"] + (sellers.collect{|seller| [seller, "stock"] + boards.collect{|qty| ["unit price for #{qty} board(s)", "total price for #{qty} board(s)"]}}).flatten
+    parts.each do |part|
+      part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation
+      line = [part['refdes'], part['qty'], part['manufacturer'], part['manufacturer-id']] # start CSV line
+      sellers.each_index do |seller_i| # go through all seller
+        seller = sellers[seller_i] # current seller
+        if part[seller] and !part[seller].empty? then
+          line << part[seller]
+          price = case seller
+          when 'aliexpress-id'
+            scrape_aliexpress(part[seller])
+          when 'digikey-id'
+            nil
+          when 'lcsc-id'
+            scrape_lcsc(part[seller])
+          else
+            nil
+          end
+          if price then
+            line << price[:stock]
+            boards.each_index do |boards_i|
+              quantity = boards[boards_i]
+              # find lowest price (considering the quantity and quantity prices)
+              unit = nil
+              total = nil
+              price[:prices].each do |p|
+                if !unit or !total then
+                  unit = p[1].to_f
+                  total = [quantity, p[0].to_i].max * unit
+                end
+                if [quantity, p[0].to_i].max * p[1].to_f < total then
+                  unit = p[1].to_f
+                  total = [quantity, p[0].to_i].max * unit
+                end
+              end
+              if "USD" == price[:currency] then
+                unit = usd2eur(unit)
+                total = usd2eur(total)
+              end
+              line << unit
+              unit_price[seller_i][boards_i] += line[-1]
+              line << total
+              total_price[seller_i][boards_i] += line[-1]
+            end
+          else
+            line += [nil] * (1 + boards.size * 2)
+          end
+        else
+          line += [nil] * (2 + boards.size * 2)
+        end # seller
+      end # sellers
+      csv << line
+    end # parts
+    # summary
+    line = [nil] * 4
+    sellers.each_index do |seller_i|
+      line += [nil, nil]
+      boards.each_index do |boards_i|
+        line << unit_price[seller_i][boards_i]
+        line << total_price[seller_i][boards_i]
+      end
+    end
+    csv << line
+    # details
+    csv << []
+    csv << ["all prices and stocks have been retrieved from Digikey, AliExpress, and LCSC on #{Time.now.to_s}"]
+    csv << ["all prices are in EUR. prices originally in USD have been converted at a rate of #{usd2eur(1.0)}"]
+  end # CSV file
+end # end cost file
+
 # ================
 # helper functions
 # ================
@@ -229,3 +324,108 @@ def bom2(schematic, attributes)
   return to_return
 end
 
+# convert USD $ value to EUR €
+def usd2eur(usd)
+  return usd / eur2usd(1.0)
+end
+
+# convert EUR € value to USD $
+def eur2usd(eur)
+  # get rate if we don't have already
+  unless $eur2usd then
+    url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
+    doc = Nokogiri::HTML(open(URI.escape(url), :allow_redirections => :all))
+    $eur2usd = doc.xpath('//cube[@currency="USD"]')[0].attr('rate').to_f
+  end
+  return eur * $eur2usd
+end
+
+# get prices from LCSC using SKU
+def scrape_lcsc(sku)
+  to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
+  # get page
+  # the search page does not always list existing parts, instead it will try 
+  url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
+  puts "scraping #{url}" if $scrape_debug
+  doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
+
+  # verify if we have got a product page
+  if doc.xpath('//div[@id="product_details"]').empty? then
+    puts "no product details:\n #{doc}" if $scrape_debug
+    return nil
+  end
+  doc.xpath('//div[@id="product_details"]//div[contains(@class,"stock-number")]').each do |element|
+    next unless element["data-stock"]
+    to_return[:stock] = element["data-stock"]
+  end
+  to_return[:currency] = "USD" # we could verify in the price, but I'm lazy
+  doc.xpath('//input[contains(@class,"salam-price")]').each do |element|
+    next unless element["data-price"]
+    to_return[:prices] = []
+    element["data-price"].split("],[").each do |price|
+      price.gsub!("[", "")
+      price.gsub!("]", "")
+      price = price.split(",")
+      to_return[:prices] << [price[0].to_i, price[1].to_f]
+    end
+  end
+  return to_return
+end
+
+# get prices from AliExpress using SKU
+def scrape_aliexpress(sku)
+  to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
+  # get page
+  url = "https://www.aliexpress.com/item/#{sku}.html"
+  puts "scraping #{url}" if $scrape_debug
+  doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
+  # all the values can be found in javascript variables (stock is even only there)
+  js_docs = doc.xpath('//script')
+  if js_docs.empty? then
+    puts "script not found:\n#{doc}" if $scrape_debug
+    return nil
+  end
+  js_json = nil
+  js_docs.each do |js_doc|
+    js_text = js_doc.text
+    next unless js_text.include?("window.runParams = {")
+    js_var = js_text.split('data: ')[1].split('csrfToken: ')[0].gsub(/,[\w\n]*$/, '')
+    js_json =  JSON.parse(js_var)
+  end
+  unless js_json and js_json["priceModule"] then
+    puts "priceModule not found:\n#{js_json}" if $scrape_debug
+    return nil
+  end
+  # get currency
+  unless js_json["priceModule"]["formatedPrice"] then
+    puts "currency not found:\n#{js_json['priceModule']}" if $scrape_debug
+    return nil
+  end
+  if js_json["priceModule"]["formatedPrice"].start_with? "US" then
+    to_return[:currency] = "USD"
+  elsif js_json["priceModule"]["formatedPrice"].start_with? "EU" then
+    to_return[:currency] = "EUR"
+  end
+  # get quantity
+  unless js_json["quantityModule"] and js_json["quantityModule"]["totalAvailQuantity"] then
+    puts "quantityModule not found:\n#{js_json}" if $scrape_debug
+    return nil
+  end
+  to_return[:stock] = js_json["quantityModule"]["totalAvailQuantity"]
+  # get price
+  unless js_json["priceModule"]["numberPerLot"] and (js_json["priceModule"]["formatedActivityPrice"] or js_json["priceModule"]["formatedPrice"] ) then
+    puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
+    return nil
+  end
+  to_return[:prices] = []
+  lot = js_json["priceModule"]["numberPerLot"].to_i
+  price = js_json["priceModule"]["formatedActivityPrice"] || js_json["priceModule"]["formatedPrice"]
+  unless price then
+    puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
+    return nil
+  end
+  price = price.split('$')[1].to_f
+  to_return[:prices] << [ lot, price / lot ]
+
+  return to_return
+end