Rakefile: add cost target to generate cost estimate
This commit is contained in:
parent
e527eee164
commit
a927ab92f2
|
|
@ -8,6 +8,11 @@ Rakefile instead of Makefile for better text file parsing capabilities.
|
|||
=end
|
||||
require 'rake/clean'
|
||||
require 'csv' # to export BOM and costs
|
||||
require 'open-uri' # to parse URLs
|
||||
require 'open_uri_redirections' # allow redirections
|
||||
require 'net/http' # to get sites
|
||||
require 'nokogiri' # to parse sites
|
||||
require 'json' # to parse GET site responses
|
||||
|
||||
# =================
|
||||
# project variables
|
||||
|
|
@ -113,6 +118,11 @@ boms = [ "#{name}.bom.csv" ]
|
|||
task :bom => boms
|
||||
CLOBBER.include(boms)
|
||||
|
||||
desc "generate cost estimte"
|
||||
costs = [ "#{name}.cost.csv" ]
|
||||
task :cost => costs
|
||||
CLOBBER.include(costs)
|
||||
|
||||
# ===============
|
||||
# file generation
|
||||
# ===============
|
||||
|
|
@ -193,6 +203,91 @@ rule ".bom.csv" => ".sch" do |t|
|
|||
end
|
||||
end
|
||||
|
||||
desc "generate cost estimate from schematic"
|
||||
# this version uses Digi-Key, AliExpress, and LCSC
|
||||
# Digi-Key is easily scrapable, while Mouser isn't
|
||||
# Digi-Key is only one distributor, but the end prices across distributor is often similar
|
||||
rule ".cost.csv" => ".sch" do |t|
|
||||
puts "scraping distributor sites to get prices. this may take some time"
|
||||
sellers = ['digikey-id','aliexpress-id','lcsc-id'] # get seller SKU
|
||||
boards = [1, 10, 100] # calculate the price for as many boards
|
||||
total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards
|
||||
unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board
|
||||
# get component information
|
||||
attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get
|
||||
parts = bom2(t.prerequisites[0], attributes) # get field values
|
||||
# put result in CVS
|
||||
CSV.open(t.name, "wb") do |csv|
|
||||
csv << ["refdes", "quantity", "manufacturer", "part number"] + (sellers.collect{|seller| [seller, "stock"] + boards.collect{|qty| ["unit price for #{qty} board(s)", "total price for #{qty} board(s)"]}}).flatten
|
||||
parts.each do |part|
|
||||
part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation
|
||||
line = [part['refdes'], part['qty'], part['manufacturer'], part['manufacturer-id']] # start CSV line
|
||||
sellers.each_index do |seller_i| # go through all seller
|
||||
seller = sellers[seller_i] # current seller
|
||||
if part[seller] and !part[seller].empty? then
|
||||
line << part[seller]
|
||||
price = case seller
|
||||
when 'aliexpress-id'
|
||||
scrape_aliexpress(part[seller])
|
||||
when 'digikey-id'
|
||||
nil
|
||||
when 'lcsc-id'
|
||||
scrape_lcsc(part[seller])
|
||||
else
|
||||
nil
|
||||
end
|
||||
if price then
|
||||
line << price[:stock]
|
||||
boards.each_index do |boards_i|
|
||||
quantity = boards[boards_i]
|
||||
# find lowest price (considering the quantity and quantity prices)
|
||||
unit = nil
|
||||
total = nil
|
||||
price[:prices].each do |p|
|
||||
if !unit or !total then
|
||||
unit = p[1].to_f
|
||||
total = [quantity, p[0].to_i].max * unit
|
||||
end
|
||||
if [quantity, p[0].to_i].max * p[1].to_f < total then
|
||||
unit = p[1].to_f
|
||||
total = [quantity, p[0].to_i].max * unit
|
||||
end
|
||||
end
|
||||
if "USD" == price[:currency] then
|
||||
unit = usd2eur(unit)
|
||||
total = usd2eur(total)
|
||||
end
|
||||
line << unit
|
||||
unit_price[seller_i][boards_i] += line[-1]
|
||||
line << total
|
||||
total_price[seller_i][boards_i] += line[-1]
|
||||
end
|
||||
else
|
||||
line += [nil] * (1 + boards.size * 2)
|
||||
end
|
||||
else
|
||||
line += [nil] * (2 + boards.size * 2)
|
||||
end # seller
|
||||
end # sellers
|
||||
csv << line
|
||||
end # parts
|
||||
# summary
|
||||
line = [nil] * 4
|
||||
sellers.each_index do |seller_i|
|
||||
line += [nil, nil]
|
||||
boards.each_index do |boards_i|
|
||||
line << unit_price[seller_i][boards_i]
|
||||
line << total_price[seller_i][boards_i]
|
||||
end
|
||||
end
|
||||
csv << line
|
||||
# details
|
||||
csv << []
|
||||
csv << ["all prices and stocks have been retrieved from Digikey, AliExpress, and LCSC on #{Time.now.to_s}"]
|
||||
csv << ["all prices are in EUR. prices originally in USD have been converted at a rate of #{usd2eur(1.0)}"]
|
||||
end # CSV file
|
||||
end # end cost file
|
||||
|
||||
# ================
|
||||
# helper functions
|
||||
# ================
|
||||
|
|
@ -229,3 +324,108 @@ def bom2(schematic, attributes)
|
|||
return to_return
|
||||
end
|
||||
|
||||
# convert USD $ value to EUR €
|
||||
def usd2eur(usd)
|
||||
return usd / eur2usd(1.0)
|
||||
end
|
||||
|
||||
# convert EUR € value to USD $
|
||||
def eur2usd(eur)
|
||||
# get rate if we don't have already
|
||||
unless $eur2usd then
|
||||
url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
|
||||
doc = Nokogiri::HTML(open(URI.escape(url), :allow_redirections => :all))
|
||||
$eur2usd = doc.xpath('//cube[@currency="USD"]')[0].attr('rate').to_f
|
||||
end
|
||||
return eur * $eur2usd
|
||||
end
|
||||
|
||||
# get prices from LCSC using SKU
|
||||
def scrape_lcsc(sku)
|
||||
to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
|
||||
# get page
|
||||
# the search page does not always list existing parts, instead it will try
|
||||
url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
|
||||
puts "scraping #{url}" if $scrape_debug
|
||||
doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
|
||||
|
||||
# verify if we have got a product page
|
||||
if doc.xpath('//div[@id="product_details"]').empty? then
|
||||
puts "no product details:\n #{doc}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
doc.xpath('//div[@id="product_details"]//div[contains(@class,"stock-number")]').each do |element|
|
||||
next unless element["data-stock"]
|
||||
to_return[:stock] = element["data-stock"]
|
||||
end
|
||||
to_return[:currency] = "USD" # we could verify in the price, but I'm lazy
|
||||
doc.xpath('//input[contains(@class,"salam-price")]').each do |element|
|
||||
next unless element["data-price"]
|
||||
to_return[:prices] = []
|
||||
element["data-price"].split("],[").each do |price|
|
||||
price.gsub!("[", "")
|
||||
price.gsub!("]", "")
|
||||
price = price.split(",")
|
||||
to_return[:prices] << [price[0].to_i, price[1].to_f]
|
||||
end
|
||||
end
|
||||
return to_return
|
||||
end
|
||||
|
||||
# get prices from AliExpress using SKU
|
||||
def scrape_aliexpress(sku)
|
||||
to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
|
||||
# get page
|
||||
url = "https://www.aliexpress.com/item/#{sku}.html"
|
||||
puts "scraping #{url}" if $scrape_debug
|
||||
doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
|
||||
# all the values can be found in javascript variables (stock is even only there)
|
||||
js_docs = doc.xpath('//script')
|
||||
if js_docs.empty? then
|
||||
puts "script not found:\n#{doc}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
js_json = nil
|
||||
js_docs.each do |js_doc|
|
||||
js_text = js_doc.text
|
||||
next unless js_text.include?("window.runParams = {")
|
||||
js_var = js_text.split('data: ')[1].split('csrfToken: ')[0].gsub(/,[\w\n]*$/, '')
|
||||
js_json = JSON.parse(js_var)
|
||||
end
|
||||
unless js_json and js_json["priceModule"] then
|
||||
puts "priceModule not found:\n#{js_json}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
# get currency
|
||||
unless js_json["priceModule"]["formatedPrice"] then
|
||||
puts "currency not found:\n#{js_json['priceModule']}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
if js_json["priceModule"]["formatedPrice"].start_with? "US" then
|
||||
to_return[:currency] = "USD"
|
||||
elsif js_json["priceModule"]["formatedPrice"].start_with? "EU" then
|
||||
to_return[:currency] = "EUR"
|
||||
end
|
||||
# get quantity
|
||||
unless js_json["quantityModule"] and js_json["quantityModule"]["totalAvailQuantity"] then
|
||||
puts "quantityModule not found:\n#{js_json}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
to_return[:stock] = js_json["quantityModule"]["totalAvailQuantity"]
|
||||
# get price
|
||||
unless js_json["priceModule"]["numberPerLot"] and (js_json["priceModule"]["formatedActivityPrice"] or js_json["priceModule"]["formatedPrice"] ) then
|
||||
puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
to_return[:prices] = []
|
||||
lot = js_json["priceModule"]["numberPerLot"].to_i
|
||||
price = js_json["priceModule"]["formatedActivityPrice"] || js_json["priceModule"]["formatedPrice"]
|
||||
unless price then
|
||||
puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
|
||||
return nil
|
||||
end
|
||||
price = price.split('$')[1].to_f
|
||||
to_return[:prices] << [ lot, price / lot ]
|
||||
|
||||
return to_return
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in New Issue