Rakefile: add cost target to generate cost estimate
This commit is contained in:
parent
e527eee164
commit
a927ab92f2
|
@ -8,6 +8,11 @@ Rakefile instead of Makefile for better text file parsing capabilities.
|
||||||
=end
|
=end
|
||||||
require 'rake/clean'
|
require 'rake/clean'
|
||||||
require 'csv' # to export BOM and costs
|
require 'csv' # to export BOM and costs
|
||||||
|
require 'open-uri' # to parse URLs
|
||||||
|
require 'open_uri_redirections' # allow redirections
|
||||||
|
require 'net/http' # to get sites
|
||||||
|
require 'nokogiri' # to parse sites
|
||||||
|
require 'json' # to parse GET site responses
|
||||||
|
|
||||||
# =================
|
# =================
|
||||||
# project variables
|
# project variables
|
||||||
|
@ -113,6 +118,11 @@ boms = [ "#{name}.bom.csv" ]
|
||||||
task :bom => boms
|
task :bom => boms
|
||||||
CLOBBER.include(boms)
|
CLOBBER.include(boms)
|
||||||
|
|
||||||
|
desc "generate cost estimte"
|
||||||
|
costs = [ "#{name}.cost.csv" ]
|
||||||
|
task :cost => costs
|
||||||
|
CLOBBER.include(costs)
|
||||||
|
|
||||||
# ===============
|
# ===============
|
||||||
# file generation
|
# file generation
|
||||||
# ===============
|
# ===============
|
||||||
|
@ -193,6 +203,91 @@ rule ".bom.csv" => ".sch" do |t|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
desc "generate cost estimate from schematic"
|
||||||
|
# this version uses Digi-Key, AliExpress, and LCSC
|
||||||
|
# Digi-Key is easily scrapable, while Mouser isn't
|
||||||
|
# Digi-Key is only one distributor, but the end prices across distributor is often similar
|
||||||
|
rule ".cost.csv" => ".sch" do |t|
|
||||||
|
puts "scraping distributor sites to get prices. this may take some time"
|
||||||
|
sellers = ['digikey-id','aliexpress-id','lcsc-id'] # get seller SKU
|
||||||
|
boards = [1, 10, 100] # calculate the price for as many boards
|
||||||
|
total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards
|
||||||
|
unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board
|
||||||
|
# get component information
|
||||||
|
attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get
|
||||||
|
parts = bom2(t.prerequisites[0], attributes) # get field values
|
||||||
|
# put result in CVS
|
||||||
|
CSV.open(t.name, "wb") do |csv|
|
||||||
|
csv << ["refdes", "quantity", "manufacturer", "part number"] + (sellers.collect{|seller| [seller, "stock"] + boards.collect{|qty| ["unit price for #{qty} board(s)", "total price for #{qty} board(s)"]}}).flatten
|
||||||
|
parts.each do |part|
|
||||||
|
part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation
|
||||||
|
line = [part['refdes'], part['qty'], part['manufacturer'], part['manufacturer-id']] # start CSV line
|
||||||
|
sellers.each_index do |seller_i| # go through all seller
|
||||||
|
seller = sellers[seller_i] # current seller
|
||||||
|
if part[seller] and !part[seller].empty? then
|
||||||
|
line << part[seller]
|
||||||
|
price = case seller
|
||||||
|
when 'aliexpress-id'
|
||||||
|
scrape_aliexpress(part[seller])
|
||||||
|
when 'digikey-id'
|
||||||
|
nil
|
||||||
|
when 'lcsc-id'
|
||||||
|
scrape_lcsc(part[seller])
|
||||||
|
else
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
if price then
|
||||||
|
line << price[:stock]
|
||||||
|
boards.each_index do |boards_i|
|
||||||
|
quantity = boards[boards_i]
|
||||||
|
# find lowest price (considering the quantity and quantity prices)
|
||||||
|
unit = nil
|
||||||
|
total = nil
|
||||||
|
price[:prices].each do |p|
|
||||||
|
if !unit or !total then
|
||||||
|
unit = p[1].to_f
|
||||||
|
total = [quantity, p[0].to_i].max * unit
|
||||||
|
end
|
||||||
|
if [quantity, p[0].to_i].max * p[1].to_f < total then
|
||||||
|
unit = p[1].to_f
|
||||||
|
total = [quantity, p[0].to_i].max * unit
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if "USD" == price[:currency] then
|
||||||
|
unit = usd2eur(unit)
|
||||||
|
total = usd2eur(total)
|
||||||
|
end
|
||||||
|
line << unit
|
||||||
|
unit_price[seller_i][boards_i] += line[-1]
|
||||||
|
line << total
|
||||||
|
total_price[seller_i][boards_i] += line[-1]
|
||||||
|
end
|
||||||
|
else
|
||||||
|
line += [nil] * (1 + boards.size * 2)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
line += [nil] * (2 + boards.size * 2)
|
||||||
|
end # seller
|
||||||
|
end # sellers
|
||||||
|
csv << line
|
||||||
|
end # parts
|
||||||
|
# summary
|
||||||
|
line = [nil] * 4
|
||||||
|
sellers.each_index do |seller_i|
|
||||||
|
line += [nil, nil]
|
||||||
|
boards.each_index do |boards_i|
|
||||||
|
line << unit_price[seller_i][boards_i]
|
||||||
|
line << total_price[seller_i][boards_i]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
csv << line
|
||||||
|
# details
|
||||||
|
csv << []
|
||||||
|
csv << ["all prices and stocks have been retrieved from Digikey, AliExpress, and LCSC on #{Time.now.to_s}"]
|
||||||
|
csv << ["all prices are in EUR. prices originally in USD have been converted at a rate of #{usd2eur(1.0)}"]
|
||||||
|
end # CSV file
|
||||||
|
end # end cost file
|
||||||
|
|
||||||
# ================
|
# ================
|
||||||
# helper functions
|
# helper functions
|
||||||
# ================
|
# ================
|
||||||
|
@ -229,3 +324,108 @@ def bom2(schematic, attributes)
|
||||||
return to_return
|
return to_return
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# convert USD $ value to EUR €
|
||||||
|
def usd2eur(usd)
|
||||||
|
return usd / eur2usd(1.0)
|
||||||
|
end
|
||||||
|
|
||||||
|
# convert EUR € value to USD $
|
||||||
|
def eur2usd(eur)
|
||||||
|
# get rate if we don't have already
|
||||||
|
unless $eur2usd then
|
||||||
|
url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
|
||||||
|
doc = Nokogiri::HTML(open(URI.escape(url), :allow_redirections => :all))
|
||||||
|
$eur2usd = doc.xpath('//cube[@currency="USD"]')[0].attr('rate').to_f
|
||||||
|
end
|
||||||
|
return eur * $eur2usd
|
||||||
|
end
|
||||||
|
|
||||||
|
# get prices from LCSC using SKU
|
||||||
|
def scrape_lcsc(sku)
|
||||||
|
to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
|
||||||
|
# get page
|
||||||
|
# the search page does not always list existing parts, instead it will try
|
||||||
|
url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
|
||||||
|
puts "scraping #{url}" if $scrape_debug
|
||||||
|
doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
|
||||||
|
|
||||||
|
# verify if we have got a product page
|
||||||
|
if doc.xpath('//div[@id="product_details"]').empty? then
|
||||||
|
puts "no product details:\n #{doc}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
doc.xpath('//div[@id="product_details"]//div[contains(@class,"stock-number")]').each do |element|
|
||||||
|
next unless element["data-stock"]
|
||||||
|
to_return[:stock] = element["data-stock"]
|
||||||
|
end
|
||||||
|
to_return[:currency] = "USD" # we could verify in the price, but I'm lazy
|
||||||
|
doc.xpath('//input[contains(@class,"salam-price")]').each do |element|
|
||||||
|
next unless element["data-price"]
|
||||||
|
to_return[:prices] = []
|
||||||
|
element["data-price"].split("],[").each do |price|
|
||||||
|
price.gsub!("[", "")
|
||||||
|
price.gsub!("]", "")
|
||||||
|
price = price.split(",")
|
||||||
|
to_return[:prices] << [price[0].to_i, price[1].to_f]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return to_return
|
||||||
|
end
|
||||||
|
|
||||||
|
# get prices from AliExpress using SKU
|
||||||
|
def scrape_aliexpress(sku)
|
||||||
|
to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
|
||||||
|
# get page
|
||||||
|
url = "https://www.aliexpress.com/item/#{sku}.html"
|
||||||
|
puts "scraping #{url}" if $scrape_debug
|
||||||
|
doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
|
||||||
|
# all the values can be found in javascript variables (stock is even only there)
|
||||||
|
js_docs = doc.xpath('//script')
|
||||||
|
if js_docs.empty? then
|
||||||
|
puts "script not found:\n#{doc}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
js_json = nil
|
||||||
|
js_docs.each do |js_doc|
|
||||||
|
js_text = js_doc.text
|
||||||
|
next unless js_text.include?("window.runParams = {")
|
||||||
|
js_var = js_text.split('data: ')[1].split('csrfToken: ')[0].gsub(/,[\w\n]*$/, '')
|
||||||
|
js_json = JSON.parse(js_var)
|
||||||
|
end
|
||||||
|
unless js_json and js_json["priceModule"] then
|
||||||
|
puts "priceModule not found:\n#{js_json}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
# get currency
|
||||||
|
unless js_json["priceModule"]["formatedPrice"] then
|
||||||
|
puts "currency not found:\n#{js_json['priceModule']}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
if js_json["priceModule"]["formatedPrice"].start_with? "US" then
|
||||||
|
to_return[:currency] = "USD"
|
||||||
|
elsif js_json["priceModule"]["formatedPrice"].start_with? "EU" then
|
||||||
|
to_return[:currency] = "EUR"
|
||||||
|
end
|
||||||
|
# get quantity
|
||||||
|
unless js_json["quantityModule"] and js_json["quantityModule"]["totalAvailQuantity"] then
|
||||||
|
puts "quantityModule not found:\n#{js_json}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
to_return[:stock] = js_json["quantityModule"]["totalAvailQuantity"]
|
||||||
|
# get price
|
||||||
|
unless js_json["priceModule"]["numberPerLot"] and (js_json["priceModule"]["formatedActivityPrice"] or js_json["priceModule"]["formatedPrice"] ) then
|
||||||
|
puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
to_return[:prices] = []
|
||||||
|
lot = js_json["priceModule"]["numberPerLot"].to_i
|
||||||
|
price = js_json["priceModule"]["formatedActivityPrice"] || js_json["priceModule"]["formatedPrice"]
|
||||||
|
unless price then
|
||||||
|
puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
price = price.split('$')[1].to_f
|
||||||
|
to_return[:prices] << [ lot, price / lot ]
|
||||||
|
|
||||||
|
return to_return
|
||||||
|
end
|
||||||
|
|
Loading…
Reference in New Issue