From 7e9afbc7e295a90d393e9a9de2b26c5ff91e5aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?King=20K=C3=A9vin?= Date: Sun, 6 Apr 2014 10:10:11 +0200 Subject: [PATCH] added cost capability. it's scraping the websites instead of using octopart because their catalogue is lacking --- hardware/.gitignore | 1 + hardware/Rakefile | 205 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 184 insertions(+), 22 deletions(-) diff --git a/hardware/.gitignore b/hardware/.gitignore index b8bac72..156249b 100644 --- a/hardware/.gitignore +++ b/hardware/.gitignore @@ -8,6 +8,7 @@ TODO attribs *_notes.txt *_bom.csv +*_cost.csv *.net *.cmd *.new.pcb diff --git a/hardware/Rakefile b/hardware/Rakefile index 1942ecf..671fd61 100644 --- a/hardware/Rakefile +++ b/hardware/Rakefile @@ -4,17 +4,22 @@ Rakefile to manage gEDA hardware projects =end require 'rake/clean' -require 'csv' +require 'csv' # to export BOM and costs +require 'open-uri' # to parse URLs +require 'nokogiri' # to scrape sites +require 'net/http' # to ask octopart +require 'json' # to parse octopart reponses # ================= # project variables # ================= # main names used for filenames +raise "define project name(s) in 'name' file" unless File.exist? "name" names = IO.read("name").split("\n").select {|target| !target.empty?} raise "define project name(s) in 'name' file" if names.empty? - # project version, read from "version" file +raise "define project name(s) in 'name' file" unless File.exist? "name" version = IO.read("version").split("\n")[0] raise "define project version in 'version' file" unless version # current date for stamping output @@ -75,25 +80,10 @@ boms = targets.collect{|target| "#{target[:name]}_bom.csv"} task :bom => boms CLEAN.include(boms) -=begin -desc "verify schematic attributes" -task :verify => vsch do |t| - ["value","footprint"].each do |attribute| - bom2(t.prerequisites[0],attribute).each do |data| - next unless data[attribute]=="unknown" - puts "#{attribute}s not defined for #{data[:refdes]*','}" - end - end - uniq = true - numbered = true - bom2(t.prerequisites[0],"refdes").each do |data| - uniq &= data[:refdes].size==1 - numbered &= !data["refdes"].include?("?") - end - puts "not all refdes uniq" unless uniq - puts "not all refdes numbered" unless numbered -end -=end +desc "get cost estimation for the BOMs" +costs = targets.collect{|target| "#{target[:name]}_cost.csv"} +task :cost => costs +CLEAN.include(costs) desc "convert schematic to pcb layout" task :sch2pcb do @@ -173,6 +163,131 @@ def bom2(schematic, attributes) return to_return end +# return actual USD $ to EUR € rate +def usd2eur_rate + unless $u2e then + rate_http = Net::HTTP.get_response(URI.parse("https://rate-exchange.appspot.com/currency?from=USD&to=EUR")) + rate_json = JSON.parse(rate_http.body) + $u2e = rate_json["rate"].to_f + end + return $u2e +end + +# return part price +def octopart(seller,sku) + seller = {'digikey-id'=>"Digi-Key", 'farnell-id'=>"Farnell", 'mouser-id'=>"Mouser"}[seller] + return nil unless seller + to_return = {stock: nil, currency: nil, prices: []} + # octopart API key to get cost estimations + unless $octopart_api_key then + raise "provide octopart API key in 'octopart' file to get cost estimations" unless File.exist? "octopart" + key = IO.read("octopart").lines[0] + raise "provide octopart API key in 'octopart' file to get cost estimations" if key.empty? + $octopart_api_key = key + end + # query octopart + query = [{:seller => seller, :sku => sku}] + url = 'http://octopart.com/api/v3/parts/match?' + url += 'queries=' + URI.encode(JSON.generate(query)) + url += '&apikey=' + $octopart_api_key + resp = Net::HTTP.get_response(URI.parse(url)) + octo_info = JSON.parse(resp.body) + # get the right response + octo_info['results'].each do |result| + result['items'].each do |item| + item['offers'].each do |offer| + next unless offer['seller']['name']==seller + next unless offer['sku']==sku + to_return[:stock] = offer['in_stock_quantity'] + offer['prices'].each do |currency,prices| + next unless currency=="USD" or currency=="EUR" + next if to_return[:currency]=="EUR" + to_return[:currency] = currency + prices.each do |price| + price[1] = price[1].to_f + end + to_return[:prices] = prices + end + end + end + end + return to_return +end + +def scrape_digikey(sku) + to_return = {stock: nil, currency: "EUR", prices: []} + # get page + url = "http://www.digikey.de/product-detail/en/all/#{sku}/" + doc = Nokogiri::HTML(open(URI.escape(url))) + # get stock + stock_doc = doc.xpath('//td[@id="quantityavailable"]')[0] + to_return[:stock] = stock_doc.text.gsub(/[ ,]+/,"").scan(/\d+/)[0].to_i + # get prices + doc.xpath('//table[@id="pricing"]/tr').each do |row| + next unless (col=row.xpath('td')).size==3 + qty = col[0].text.gsub(/[ ,]/,"").to_i + price = col[1].text.gsub(/[ ,]/,"").to_f + to_return[:prices] << [qty,price] + end + return to_return +end + +def scrape_farnell(sku) + to_return = {stock: nil, currency: "EUR", prices: []} + # get page + url = "http://de.farnell.com/#{sku}" + doc = Nokogiri::HTML(open(URI.escape(url))) + # get stock + stock_doc = doc.xpath('//td[@class="prodDetailAvailability"]')[0] + if stock_doc then + to_return[:stock] = stock_doc.text.lines[-1].to_i + else # when several stocks are available + stock_doc = doc.xpath('//div[@class="stockDetail"]')[0] + to_return[:stock] = stock_doc.text.gsub(".","").scan(/\d+/)[-1].to_i # the last match should be for EU + end + # get prices + doc.xpath('//div[@class="price"]/*/tr').each do |row| + next unless row.xpath('td').size==2 + qty = row.xpath('td')[0].text.gsub(/\s+/,"").split("-")[0].to_i + price = row.xpath('td')[1].text.gsub(/\s+/,"").gsub("€","").gsub(",",".").to_f + to_return[:prices] << [qty,price] + end + return to_return +end + +def scrape_mouser(sku) + to_return = {stock: nil, currency: "EUR", prices: []} + # get page + url = "http://de.mouser.com/Search/ProductDetail.aspx?R=0virtualkey0virtualkey#{sku}" + doc = Nokogiri::HTML(open(URI.escape(url))) + # get stock + stock_doc = doc.xpath('//table[contains(@id,"availability")]/tr/td')[0] + to_return[:stock] = stock_doc.text.gsub(".","").to_i + # get prices + doc.xpath('//table[@class="PriceBreaks"]/tr').each do |row| + qty_doc = row.xpath('td[@class="PriceBreakQuantity"]') + qty = qty_doc[0].text.gsub(/[\s:\.]+/,"").to_i unless qty_doc.empty? + price_doc = row.xpath('td[@class="PriceBreakPrice"]') + price = price_doc[0].text.gsub(/\s+/,"").gsub("€","").gsub(",",".").to_f unless price_doc.empty? + price = nil if price==0 + to_return[:prices] << [qty,price] if qty and price + end + return to_return +end + +def scrape_prices(seller,sku) + return case seller + when 'digikey-id' + scrape_digikey(sku) + when 'farnell-id' + scrape_farnell(sku) + when 'mouser-id' + scrape_mouser(sku) + else + nil + end +end + # =============== # file generation # =============== @@ -239,7 +354,7 @@ end desc "generate BOM file from schematic" targets.each do |target| - file "#{target[:name]}_bom.csv" => target[:vsch] do |t| + file "#{target[:name]}_bom.csv" => target[:sch] do |t| attributes = ["category","device","value","description","manufacturer","manufacturer-id","digikey-id","farnell-id","mouser-id"] bom_data = bom2(t.prerequisites[0],attributes) CSV.open(t.name, "wb") do |csv| @@ -252,6 +367,52 @@ targets.each do |target| end end +desc "generate cost estimation from schematic" +targets.each do |target| + file "#{target[:name]}_cost.csv" => target[:sch] do |t| + sellers = ['digikey-id','farnell-id','mouser-id'] + # get component information + attributes = ["manufacturer","manufacturer-id"]+sellers + parts = bom2(t.prerequisites[0],attributes) + # put result in CVS + CSV.open(t.name, "wb") do |csv| + csv << ["refdes","quantity","manufacturer","part number"]+(sellers.collect{|seller| [seller,"stock","currency","unit price (1 board)","total price (1 board)","unit price (10 boards)","total price (10 board)"]}).flatten + parts.each do |part| + part['qty'] = part['qty'].to_i + next unless part['manufacturer'] and part['manufacturer-id'] + line = [part['refdes'],part['qty'],part['manufacturer'],part['manufacturer-id']] + sellers.each do |seller| + if part[seller] then + begin + prices = scrape_prices(seller,part[seller]) + rescue + $stderr.puts "#{part['manufacturer']} #{part['manufacturer-id']} not available using #{seller} #{part[seller]}" + line += [part[seller]]+[nil]*6 + next + end + line << part[seller] + line << prices[:stock] + line << prices[:currency] + unit1 = nil # the unit price to use for 1 board + unit10 = nil # the unit price to use for 10 boards + prices[:prices].each do |price| + unit1 = price[1] if (!unit1 or price[1] target[:vpcb] do |t|