From 4c3e594085ade086d7f78bfe01e1b95aa52275b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?King=20K=C3=A9vin?= Date: Sun, 19 Nov 2017 22:06:13 +0100 Subject: [PATCH] update cost generator --- .gitignore | 2 + Rakefile | 483 +++++++++++++++++++++++++++++++++-------------------- 2 files changed, 307 insertions(+), 178 deletions(-) diff --git a/.gitignore b/.gitignore index 925d39c..053ccd6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ attribs *_notes.txt *_bom.csv *_cost.csv +octopart.apikey +*_costs.csv *.net *.cmd *.new.pcb diff --git a/Rakefile b/Rakefile index 323fa0f..4111429 100644 --- a/Rakefile +++ b/Rakefile @@ -82,8 +82,13 @@ task :bom => boms CLEAN.include(boms) desc "get cost estimation for BOMs" -costs = targets.collect{|target| "#{target[:name]}_cost.csv"} -task :cost => costs +cost = targets.collect{|target| "#{target[:name]}_cost.csv"} +task :cost => cost +CLEAN.include(cost) + +desc "get cost comparison for BOMs" +costs = targets.collect{|target| "#{target[:name]}_costs.csv"} +task :costs => costs CLEAN.include(costs) desc "convert schematic to pcb layout" @@ -149,7 +154,7 @@ def bom2(schematic, attributes) [attributes.to_s] end # generate bom2 - list = `gnetlist -g bom2 -Oattribs=#{attributes*','} -q -o - #{schematic}` + list = `gnetlist -g bom2 -O attribs=#{attributes*','} -q -o - #{schematic}` list.encode!("UTF-8","ISO-8859-1") list.gsub!(/(\d[Mkmµ]?)\?/,'\1Ω') # UTF-8 characters like Ω are replaced with ? by gnetlist list.gsub!(/(https?:\/\/[^:]*):/,'"\1":') # ':' (like in links) are not protected @@ -168,134 +173,104 @@ end # return actual USD $ to EUR € rate def usd2eur_rate unless $u2e then - rate_http = Net::HTTP.get_response(URI.parse("https://rate-exchange.appspot.com/currency?from=USD&to=EUR")) + rate_http = Net::HTTP.get_response(URI.parse("https://api.fixer.io/latest?base=USD")) rate_json = JSON.parse(rate_http.body) - $u2e = rate_json["rate"].to_f + $u2e = rate_json["rates"]["EUR"].to_f end return $u2e end -# return part price -def octopart(seller,sku) - seller = {'digikey-id'=>"Digi-Key", 'farnell-id'=>"Farnell", 'mouser-id'=>"Mouser"}[seller] - return nil unless seller - to_return = {stock: nil, currency: nil, prices: []} - # octopart API key to get cost estimations - unless $octopart_api_key then - raise "provide octopart API key in 'octopart' file to get cost estimations" unless File.exist? "octopart" - key = IO.read("octopart").lines[0] - raise "provide octopart API key in 'octopart' file to get cost estimations" if key.empty? - $octopart_api_key = key - end - # query octopart - query = [{:seller => seller, :sku => sku}] - url = 'http://octopart.com/api/v3/parts/match?' - url += 'queries=' + URI.encode(JSON.generate(query)) - url += '&apikey=' + $octopart_api_key - resp = Net::HTTP.get_response(URI.parse(url)) - octo_info = JSON.parse(resp.body) - # get the right response - octo_info['results'].each do |result| - result['items'].each do |item| - item['offers'].each do |offer| - next unless offer['seller']['name']==seller - next unless offer['sku']==sku - to_return[:stock] = offer['in_stock_quantity'] - offer['prices'].each do |currency,prices| - next unless currency=="USD" or currency=="EUR" - next if to_return[:currency]=="EUR" - to_return[:currency] = currency - prices.each do |price| - price[1] = price[1].to_f - end - to_return[:prices] = prices - end - end +# get part prices (and stock) from octopart +# octopart provides a nice API (requiring a key) and prices for numerous distributors, but the result aren't always accurate +# provide either the ocotopart_id, or manufacturer_name and manufacturer_part, or distributor_name and distributor_sku +APIKEY_FILE = "octopart.apikey" +def octopart_prices(octopart_id, manufacturer_name, manufacturer_part, distributor_name, distributor_sku) + to_return = nil + # get API key + raise "octopart API key required in #{APIKEY_FILE}" unless File.file? APIKEY_FILE + apikey = IO.read(APIKEY_FILE).lines[0] + raise "octopart API key required in #{APIKEY_FILE}" if apikey.empty? + # get part information + url = "https://octopart.com/api/v3/parts/match?&apikey=#{apikey}&queries=" + to_return = if octopart_id then + url = "https://octopart.com/api/v3/parts/#{octopart_id}?apikey=#{apikey}" + resp = Net::HTTP.get_response(URI.parse(url)) + JSON.parse(resp.body) + elsif manufacturer_name and manufacturer_part then + url = "https://octopart.com/api/v3/parts/match?&apikey=#{apikey}&queries="+URI.encode(JSON.generate([{:brand => manufacturer_name, :mpn => manufacturer_part}])) + resp = Net::HTTP.get_response(URI.parse(url)) + json = JSON.parse(resp.body) + if json["results"].empty? or json["results"][0]["items"].empty? then + nil + else + json["results"][0]["items"][0] + end + elsif distributor_name and distributor_sku then + url = "https://octopart.com/api/v3/parts/match?&apikey=#{apikey}&queries="+URI.encode(JSON.generate([{:seller => distributor_name, :sku => distributor_sku}])) + resp = Net::HTTP.get_response(URI.parse(url)) + json = JSON.parse(resp.body) + if json["results"].empty? or json["results"][0]["items"].empty? then + nil + else + json["results"][0]["items"][0] end - end - return to_return -end - -DIGIKEY_URL = "http://www.digikey.de/product-detail/en/all/" -def scrape_digikey(sku) - to_return = {stock: nil, currency: "EUR", prices: []} - # get page - url = DIGIKEY_URL+sku - doc = Nokogiri::HTML(open(URI.escape(url))) - # get stock - stock_doc = doc.xpath('//td[@id="quantityavailable"]')[0] - to_return[:stock] = stock_doc.text.gsub(/[ ,]+/,"").scan(/\d+/)[0].to_i - # get prices - doc.xpath('//table[@id="pricing"]/tr').each do |row| - next unless (col=row.xpath('td')).size==3 - qty = col[0].text.gsub(/[^\d]+/,"") - qty = (qty =~ /^\d+$/ ? qty.to_i : nil) - price = col[1].text.gsub(/[^\d\.]+/,"") - price = (price =~ /^\d+(\.\d+)?$/ ? price.to_f : nil) - to_return[:prices] << [qty,price] if qty and price - end - return to_return -end - -FARNELL_URL = "http://de.farnell.com/" -def scrape_farnell(sku) - to_return = {stock: nil, currency: "EUR", prices: []} - # get page - url = FARNELL_URL+sku - doc = Nokogiri::HTML(open(URI.escape(url))) - # get stock - stock_doc = doc.xpath('//td[@class="prodDetailAvailability"]')[0] - if stock_doc then - to_return[:stock] = stock_doc.text.lines.to_a[-1].to_i - else # when several stocks are available - stock_doc = doc.xpath('//div[@class="stockDetail"]')[0] - to_return[:stock] = stock_doc.text.gsub(".","").scan(/\d+/)[-1].to_i # the last match should be for EU - end - # get prices - doc.xpath('//div[@class="price"]/*/tr').each do |row| - next unless row.xpath('td').size==2 - qty = row.xpath('td')[0].text.split("-")[0].gsub(/[^\d]+/,"") - qty = (qty =~ /^\d+$/ ? qty.to_i : nil) - price = row.xpath('td')[1].text.gsub(",",".").gsub(/[^\d\.]+/,"") - price = (price =~ /^\d+(\.\d+)?$/ ? price.to_f : nil) - to_return[:prices] << [qty,price] if qty and price - end - return to_return -end - -MOUSER_URL = "http://www.mouser.com/Search/ProductDetail.aspx?R=0virtualkey0virtualkey" -def scrape_mouser(sku) - to_return = {stock: nil, currency: "EUR", prices: []} - # get page - url = MOUSER_URL+sku - doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all)) - # get stock - stock_doc = doc.xpath('//table[contains(@id,"availability")]/tr/td')[0] - to_return[:stock] = stock_doc.text.gsub(".","").to_i - # get prices - doc.xpath('//table[@class="PriceBreaks"]/tr').each do |row| - qty_doc = row.xpath('td[@class="PriceBreakQuantity"]') - qty = qty_doc[0].text.gsub(/[\s:\.]+/,"").to_i unless qty_doc.empty? - price_doc = row.xpath('td[@class="PriceBreakPrice"]') - price = price_doc[0].text.gsub(/\s+/,"").gsub("€","").gsub(",",".").to_f unless price_doc.empty? - price = nil if price==0 - to_return[:prices] << [qty,price] if qty and price - end - return to_return -end - -def scrape_prices(seller,sku) - return case seller - when 'digikey-id' - scrape_digikey(sku) - when 'farnell-id' - scrape_farnell(sku) - when 'mouser-id' - scrape_mouser(sku) else nil end -end + sleep 0.33 if to_return # 3 queries/second rate limiting + return to_return +end + +# get prices and stock for part from Digi-Key using the Digi-Key part number +def scrape_digikey(sku) + to_return = {stock: nil, currency: "EUR", prices: []} + # get page + url = "https://www.digikey.de/product-detail/en/all/"+sku + doc = Nokogiri::HTML(open(URI.escape(url))) + return nil if doc.xpath('//td[@id="quantityAvailable"]').empty? or doc.xpath('//table[@id="product-dollars"]').empty? + # get stock + stock_doc = doc.xpath('//td[@id="quantityAvailable"]/span[@id="dkQty"]') + to_return[:stock] = (stock_doc.empty? ? 0 : stock_doc.text.gsub('.','').to_i) + # get prices + doc.xpath('//table[@id="product-dollars"]/tr').each do |row| + next unless (col=row.xpath('td')).size==3 + qty = col[0].text.gsub(/[^\d]+/,"") + qty = (qty =~ /^\d+$/ ? qty.to_i : nil) + price = col[1].text.gsub(/[^\d\.,]+/,"").gsub(".","").gsub(",",".") + price = (price =~ /^\d+(\.\d+)?$/ ? price.to_f : nil) + to_return[:prices] << [qty,price] if qty and price + end + + return to_return +end + +# get prices from AliExpress using SKU +def scrape_aliexpress(sku) + to_return = {stock: nil, currency: nil, quantity: nil, price: nil} # information to return (lot price, unit quantity, unit stock) + # get page + url = "https://www.aliexpress.com/item//#{sku}.html" + doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all)) + # all the values can be found in javascript variables (stock in even only there) + js_doc = doc.xpath('//div[@id="j-product-detail-bd"]//script[@type="text/javascript"]') + return nil if js_doc.empty? + js_text = js_doc[0].text + # get currency + return nil unless js_text.include?('window.runParams.currencyCode="') + to_return[:currency] = js_text.split('window.runParams.currencyCode="')[1].split('";')[0] + # get other values + return nil unless js_text.include?("var skuProducts=") + js_json = JSON.parse(js_text.split("var skuProducts=")[1].split(";")[0])[0] + to_return[:price] = js_json["skuVal"]["skuMultiCurrencyCalPrice"].to_f + if (js_json["skuVal"]["skuMultiCurrencyPerPiecePrice"]) then + to_return[:quantity] = (to_return[:price]/js_json["skuVal"]["skuMultiCurrencyPerPiecePrice"].to_f).round + else + to_return[:quantity] = 1 + end + to_return[:stock] = (js_json["skuVal"]["availQuantity"]*to_return[:quantity]).to_i + to_return[:prices] = [[to_return[:quantity], to_return[:price]]] + + return to_return +end # =============== # file generation @@ -307,7 +282,7 @@ targets.each do |target| # embed symbols sh "cp #{t.prerequisites.join(' ')} #{t.name}" sh "gschlas -e #{t.name}" - # on \ is to prevent ruby interpreting it, th other is for sed + # on \ is to prevent ruby interpreting it, the other is for sed # the version sh "sed --in-place 's/\\(version=\\)\\$Version\\$/\\1#{version}/' #{t.name}" # the date @@ -321,7 +296,7 @@ desc "copy layout to include complete version (version, date, and run teardrops targets.each do |target| file target[:vpcb] => target[:pcb] do |t| sh "cp #{t.prerequisites.join(' ')} #{t.name}" - # on \ is to prevent ruby interpreting it, th other is for sed + # one \ is to prevent ruby interpreting it, the other is for sed # the version and revision version_revision = "v#{version}.#{target[:pcb_rev].to_s.rjust(3,'0')}" sh "sed -i 's/\\$version\\$/#{version_revision}/' #{t.name}" @@ -367,7 +342,7 @@ end desc "generate BOM file from schematic" targets.each do |target| file "#{target[:name]}_bom.csv" => target[:sch] do |t| - attributes = ["category","device","value","description","manufacturer","manufacturer-id","datasheet","digikey-id","farnell-id","mouser-id"] + attributes = ["category","device","value","description","manufacturer","manufacturer-id","datasheet","digikey-id","farnell-id","mouser-id","aliexpress-id","alternatives"] bom_data = bom2(t.prerequisites[0],attributes) CSV.open(t.name, "wb") do |csv| all_attributes = ["refdes","qty"]+attributes @@ -381,76 +356,228 @@ end desc "generate cost estimation from schematic" targets.each do |target| + # this version uses Digi-Key and AliExpress + # Digi-Key is easily scrapable, while Mouser isn't + # Digi-Key is only one distributor, but the end prices across distributor is often similar file "#{target[:name]}_cost.csv" => target[:sch] do |t| - sellers = ['digikey-id','farnell-id','mouser-id'] # provide the (supported) seller SKU as value to this attribute name in the symbols/schematic - boards = [1,10] # calculate the price for as many boards - sum = Array.new(sellers.size){Array.new(boards.size,0.0)} # total cost - stocks = Array.new(sellers.size){Array.new(boards.size){[]}} # is there enough stock + sellers = ['digikey-id','aliexpress-id'] # get seller SKU + boards = [1,10,100] # calculate the price for as many boards + total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards + unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board # get component information - attributes = ["manufacturer","manufacturer-id"]+sellers - parts = bom2(t.prerequisites[0],attributes) - parts.collect!{|part| part['manufacturer'] and part['manufacturer-id'] ? part : nil} - parts.compact! + attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get + parts = bom2(t.prerequisites[0],attributes) # get field values # put result in CVS CSV.open(t.name, "wb") do |csv| - csv << ["refdes","quantity","manufacturer","part number"]+(sellers.collect{|seller| [seller,"stock","currency"]+boards.collect{|qty| ["unit price for #{qty} board(s)","total price for #{qty} board(s)"]}}).flatten + csv << ["refdes","quantity","manufacturer","part number"]+(sellers.collect{|seller| [seller,"stock"]+boards.collect{|qty| ["unit price for #{qty} board(s)","total price for #{qty} board(s)"]}}).flatten parts.each do |part| - part['qty'] = part['qty'].to_i - line = [part['refdes'],part['qty'],part['manufacturer'],part['manufacturer-id']] - sellers.each_index do |i| - seller = sellers[i] - if part[seller] then - begin - prices = scrape_prices(seller,part[seller]) - rescue - $stderr.puts "#{part['manufacturer']} #{part['manufacturer-id']} not available using #{seller} #{part[seller]}" - line += [part[seller]]+[nil]*(2+boards.size*2) - next - end + part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation + line = [part['refdes'],part['qty'],part['manufacturer'],part['manufacturer-id']] # start CSV line + sellers.each_index do |seller_i| # go through all seller + seller = sellers[seller_i] # current seller + if part[seller] and !part[seller].empty? then line << part[seller] - line << prices[:stock] - line << prices[:currency] - unit = [] # the unit price - boards.each_index do |j| - if prices[:prices].empty? then - line += [nil]*2 - else - prices[:prices].each do |price| - unit[j] = price[1] if (!unit[j] or price[1] target[:sch] do |t| + sellers = ['digikey-id','farnell-id','mouser-id','aliexpress-id'] # get seller SKU + SELLER_ID = {'digikey-id' => 459, 'farnell-id' => 819, 'mouser-id' => 2401} # octopart seller IDs + boards = [1,10,100] # calculate the price for as many boards + sum = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total cost + stocks = Array.new(sellers.size){Array.new(boards.size, true)} # is there enough stock + # get component information + attributes = ["value","manufacturer","manufacturer-id","octopart-id"]+sellers # BOM fields to get + parts = bom2(t.prerequisites[0],attributes) # get field values + # put result in CVS + CSV.open(t.name, "wb") do |csv| + csv << ["refdes","quantity","manufacturer","part number"]+(sellers.collect{|seller| [seller,"stock"]+boards.collect{|qty| ["unit price for #{qty} board(s)","total price for #{qty} board(s)"]}}).flatten + parts.each do |part| + part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation + line = [part['refdes'],part['qty'],part['manufacturer'],part['manufacturer-id']] # start CSV line + # get prices + prices = nil + if part['octopart-id'] and !part['octopart-id'].empty? then + prices = octopart_prices(part['octopart-id'], nil, nil, nil, nil) + elsif part['manufacturer'] and !part['manufacturer'].empty? and part['manufacturer-id'] and !part['manufacturer-id'].empty? + prices = octopart_prices(nil, part['manufacturer'], part['manufacturer-id'], nil, nil) + end + # get prices per seller + sellers.each_index do |seller_i| + seller = sellers[seller_i] # current seller + if 'aliexpress-id'==seller then # AliExpress is an exception since it's not in octopart + if part[seller] and !part[seller].empty? then # AliExpress link provided + price = scrape_aliexpress(part[seller]) + if price then + line << part[seller] + line << price[:stock] + boards.each_index do |boards_i| + quantity = boards[boards_i] + line << price[:price] + if quantity