# encoding: utf-8 # ruby: 2.1.0 =begin Rakefile to manage gEDA hardware projects =end require 'rake/clean' require 'csv' # to export BOM and costs require 'open-uri' # to parse URLs require 'open_uri_redirections' # allow redirections require 'net/http' # to get sites require 'nokogiri' # to parse sites require 'json' # to parse get site responses require 'date' # to parse dates # ================= # project variables # ================= # main names used for filenames raise "define project name(s) in 'name' file" unless File.exist? "name" names = IO.read("name").split("\n").select {|target| !target.empty?} raise "define project name(s) in 'name' file" if names.empty? # project version, read from "version" file raise "define project version in 'version' file" unless File.exist? "version" version = IO.read("version").split("\n")[0] raise "define project version in 'version' file" unless version # current date for stamping output date = Time.now.strftime("%Y-%m-%d") # create targets for each name targets = [] names.each do |name| # schema sch = "#{name}.sch" # pcb layout pcb = "#{name}.pcb" # get common schematic/layout revision number, based on the total number of commits # this means the layout revision number can increment although only the schematic has been modified (and vice versa), but this is more convenient to have synched revision numbers rev = `git log --pretty=oneline "#{sch}" "#{pcb}" | wc -l`.chomp.to_i sch_rev = rev pcb_rev = rev date = Date.parse(`git log -1 --format=%cd "#{sch}" "#{pcb}"`) # get schematic and layout revision numbers, based on the respective number of commits # this allows to have independent number in case an edit in one does not affect the other, but you need to keep track of which revision numbers correspond #sch_rev = `git log --pretty=oneline "#{sch}" | wc -l`.chomp.to_i #pcb_rev = `git log --pretty=oneline "#{pcb}" | wc -l`.chomp.to_i # schema name with version and revision vsch = "#{name}_v#{version}.#{sch_rev.to_s.rjust(3,'0')}.sch" # pcb layout name with version and revision vpcb = "#{name}_v#{version}.#{pcb_rev.to_s.rjust(3,'0')}.pcb" # add to targets targets << { name: name, sch: sch, pcb: pcb, sch_rev: sch_rev, pcb_rev: pcb_rev, vsch: vsch, vpcb: vpcb } end # ========== # main tasks # ========== desc "main building task" task :default => [:version,:print,:notes,:bom,:gerber] desc "create release file" release = "hardware-release_v#{version}.tar.gz" task :release => [:default,:photo,:cost,release] CLOBBER.include(release) desc "set version in schematic and layout" versions = targets.collect{|target| [target[:vsch],target[:vpcb]]}.flatten task :version => versions CLEAN.include(versions) targets.each do |target| CLOBBER.include("#{target[:name]}_*.sch") CLOBBER.include("#{target[:name]}_*.pcb") end desc "print schematic and layout (as pdf)" prints = targets.collect{|target| ["#{target[:name]}_schematic.pdf","#{target[:name]}_layout.pdf"]}.flatten task :print => prints CLEAN.include(prints) desc "export notes from schematic" notes = targets.collect{|target| "#{target[:name]}_notes.txt"} task :notes => notes CLEAN.include(notes) desc "export BOMs from schematic" boms = targets.collect{|target| "#{target[:name]}_bom.csv"} task :bom => boms CLEAN.include(boms) desc "get cost estimation for BOMs" cost = targets.collect{|target| "#{target[:name]}_cost.csv"} task :cost => cost CLEAN.include(cost) desc "get cost comparison for BOMs" costs = targets.collect{|target| "#{target[:name]}_costs.csv"} task :costs => costs CLEAN.include(costs) desc "convert schematic to pcb layout" task :sch2pcb do targets.each do |target| sh "gsch2pcb #{target[:sch]} --elements-dir #{File.dirname(__FILE__)}/lib/footprints --skip-m4 --output-name #{target[:name]}" end end targets.each do |target| CLOBBER.include("#{target[:name]}.net") CLOBBER.include("#{target[:name]}.cmd") CLOBBER.include("#{target[:name]}.new.pcb") end photos = targets.collect{|target| ["#{target[:name]}_layout-top.png","#{target[:name]}_layout-bottom.png"]}.flatten desc "render layout" task :photo => photos CLOBBER.include(photos) desc "export gerber" task :gerber => :version do targets.each do |target| export = true # export only if the gerbers are all older than the layout Dir.foreach(".") do |file| next unless file.start_with? target[:name] next unless file.end_with? ".gbr" or file.end_with? ".cnc" export &= (File.ctime(target[:vpcb])>File.ctime(file)) end sh "pcb -x gerber --gerberfile #{target[:name]} --all-layers #{target[:vpcb]}" if export end end CLOBBER.include("*.gbr") CLOBBER.include("*.cnc") desc "reformat gerber and drill output (some programs like LPKF CircuitPro have difficulties with gEDA pcb output)" task :reformat do Dir.foreach(".") do |file| next unless File.file? file if file.end_with? ".gbr" then sh "gerbv --export rs274x --output '#{file}' '#{file}'" elsif file.end_with? ".cnc" then sh "gerbv --export drill --output '#{file}' '#{file}'" end end end # ================ # helper functions # ================ # generate gnetlist bom2 and parse them # arguments: schematic=schematic to use, attributes=attributes to use for generating bom2 # returns an array of hash. key is the attribute name, value is the attribute value def bom2(schematic, attributes) to_return = [] # force attributes to be an array attributes = case attributes when String [attributes] when Array attributes else [attributes.to_s] end # generate bom2 list = `gnetlist -g bom2 -O attribs=#{attributes*','} -q -o - #{schematic}` list.encode!("UTF-8","ISO-8859-1") list.gsub!(/(\d[Mkmµ]?)\?/,'\1Ω') # UTF-8 characters like Ω are replaced with ? by gnetlist list.gsub!(/(https?:\/\/[^:]*):/,'"\1":') # ':' (like in links) are not protected # parse bom2 csv = CSV.parse(list,{:col_sep => ":"}) csv[1..-1].each do |row| line = {} row.each_index do |col| line[csv[0][col]] = row[col] unless row[col]=="unknown" end to_return << line end return to_return end # return actual USD $ to EUR € rate def usd2eur_rate unless $u2e then rate_http = Net::HTTP.get_response(URI.parse("https://api.fixer.io/latest?base=USD")) rate_json = JSON.parse(rate_http.body) $u2e = rate_json["rates"]["EUR"].to_f end return $u2e end # get part prices (and stock) from octopart # octopart provides a nice API (requiring a key) and prices for numerous distributors, but the result aren't always accurate # provide either the ocotopart_id, or manufacturer_name and manufacturer_part, or distributor_name and distributor_sku APIKEY_FILE = "octopart.apikey" def octopart_prices(octopart_id, manufacturer_name, manufacturer_part, distributor_name, distributor_sku) to_return = nil # get API key raise "octopart API key required in #{APIKEY_FILE}" unless File.file? APIKEY_FILE apikey = IO.read(APIKEY_FILE).lines[0] raise "octopart API key required in #{APIKEY_FILE}" if apikey.empty? # get part information url = "https://octopart.com/api/v3/parts/match?&apikey=#{apikey}&queries=" to_return = if octopart_id then url = "https://octopart.com/api/v3/parts/#{octopart_id}?apikey=#{apikey}" resp = Net::HTTP.get_response(URI.parse(url)) JSON.parse(resp.body) elsif manufacturer_name and manufacturer_part then url = "https://octopart.com/api/v3/parts/match?&apikey=#{apikey}&queries="+URI.encode(JSON.generate([{:brand => manufacturer_name, :mpn => manufacturer_part}])) resp = Net::HTTP.get_response(URI.parse(url)) json = JSON.parse(resp.body) if json["results"].empty? or json["results"][0]["items"].empty? then nil else json["results"][0]["items"][0] end elsif distributor_name and distributor_sku then url = "https://octopart.com/api/v3/parts/match?&apikey=#{apikey}&queries="+URI.encode(JSON.generate([{:seller => distributor_name, :sku => distributor_sku}])) resp = Net::HTTP.get_response(URI.parse(url)) json = JSON.parse(resp.body) if json["results"].empty? or json["results"][0]["items"].empty? then nil else json["results"][0]["items"][0] end else nil end sleep 0.33 if to_return # 3 queries/second rate limiting return to_return end # get prices and stock for part from Digi-Key using the Digi-Key part number def scrape_digikey(sku) to_return = {stock: nil, currency: "EUR", prices: []} # get page url = "https://www.digikey.de/product-detail/en/all/"+sku doc = Nokogiri::HTML(open(URI.escape(url))) return nil if doc.xpath('//td[@id="quantityAvailable"]').empty? or doc.xpath('//table[@id="product-dollars"]').empty? # get stock stock_doc = doc.xpath('//td[@id="quantityAvailable"]/span[@id="dkQty"]') to_return[:stock] = (stock_doc.empty? ? 0 : stock_doc.text.gsub('.','').to_i) # get prices doc.xpath('//table[@id="product-dollars"]/tr').each do |row| next unless (col=row.xpath('td')).size==3 qty = col[0].text.gsub(/[^\d]+/,"") qty = (qty =~ /^\d+$/ ? qty.to_i : nil) price = col[1].text.gsub(/[^\d\.,]+/,"").gsub(".","").gsub(",",".") price = (price =~ /^\d+(\.\d+)?$/ ? price.to_f : nil) to_return[:prices] << [qty,price] if qty and price end return to_return end # get prices from AliExpress using SKU def scrape_aliexpress(sku) to_return = {stock: nil, currency: nil, quantity: nil, price: nil} # information to return (lot price, unit quantity, unit stock) # get page url = "https://www.aliexpress.com/item//#{sku}.html" doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all)) # all the values can be found in javascript variables (stock in even only there) js_doc = doc.xpath('//div[@id="j-product-detail-bd"]//script[@type="text/javascript"]') return nil if js_doc.empty? js_text = js_doc[0].text # get currency return nil unless js_text.include?('window.runParams.currencyCode="') to_return[:currency] = js_text.split('window.runParams.currencyCode="')[1].split('";')[0] # get other values return nil unless js_text.include?("var skuProducts=") js_json = JSON.parse(js_text.split("var skuProducts=")[1].split(";")[0])[0] to_return[:price] = js_json["skuVal"]["skuMultiCurrencyCalPrice"].to_f if (js_json["skuVal"]["skuMultiCurrencyPerPiecePrice"]) then to_return[:quantity] = (to_return[:price]/js_json["skuVal"]["skuMultiCurrencyPerPiecePrice"].to_f).round else to_return[:quantity] = 1 end to_return[:stock] = (js_json["skuVal"]["availQuantity"]*to_return[:quantity]).to_i to_return[:prices] = [[to_return[:quantity], to_return[:price]]] return to_return end # =============== # file generation # =============== desc "copy schematic to include complete version (version, revision, and date)" targets.each do |target| file target[:vsch] => target[:sch] do |t| # embed symbols sh "cp #{t.prerequisites.join(' ')} #{t.name}" sh "gschlas -e #{t.name}" # on \ is to prevent ruby interpreting it, the other is for sed # the version sh "sed --in-place 's/\\(version=\\)\\$Version\\$/\\1#{version}/' #{t.name}" # the date sh "sed --in-place 's/\\(date=\\)\\$Date\\$/\\1#{date}/' #{t.name}" # the revision sh "sed --in-place 's/\\(revision=\\)\\$Revision\\$/\\1#{target[:sch_rev]}/' #{t.name}" end end desc "copy layout to include complete version (version, date, and run teardrops when available)" targets.each do |target| file target[:vpcb] => target[:pcb] do |t| sh "cp #{t.prerequisites.join(' ')} #{t.name}" # one \ is to prevent ruby interpreting it, the other is for sed # the version and revision version_revision = "v#{version}.#{target[:pcb_rev].to_s.rjust(3,'0')}" sh "sed -i 's/\\$version\\$/#{version_revision}/' #{t.name}" # the date sh "sed -i 's/\\$date\\$/#{date}/' #{t.name}" # run teardrop for vias and pins if File.exist? "#{Dir.home}/.pcb/plugins/teardrops.so" then sh "pcb --action-string \"djopt(splitlines) Teardrops() s() q()\" #{t.name}" end end end desc "generate printable version (PDF) of schematic" targets.each do |target| file "#{target[:name]}_schematic.pdf" => target[:vsch] do |t| sh "gaf export --format pdf --color --output '#{t.name}' #{t.prerequisites.join(' ')} 2> /dev/null" end end desc "generate printable documentation (PDF) from layout" targets.each do |target| file "#{target[:name]}_layout.pdf" => target[:vpcb] do |t| sh "pcb -x ps --media A4 --psfile #{t.name}.ps #{t.prerequisites.join(' ')} 2> /dev/null" sh "ps2pdf -sPAPERSIZE=a4 #{t.name}.ps #{t.name} 2> /dev/null" sh "rm #{t.name}.ps 2> /dev/null" end end desc "generate note file from schematic, listing the 'note' attributes from elements" targets.each do |target| file "#{target[:name]}_notes.txt" => target[:vsch] do |t| notes_data = bom2(t.prerequisites[0],"note") File.open(t.name,"w") do |notes_file| notes_data.each do |note| next unless note['note'] note['note'] = note['note'].gsub('. ',".\n").gsub(/\n+$/,'') notes_file.puts "#{note['refdes']}:\n#{note['note']}\n\n" end end end end desc "generate BOM file from schematic" targets.each do |target| file "#{target[:name]}_bom.csv" => target[:sch] do |t| attributes = ["category","device","value","description","manufacturer","manufacturer-id","datasheet","digikey-id","farnell-id","mouser-id","aliexpress-id","alternatives"] bom_data = bom2(t.prerequisites[0],attributes) CSV.open(t.name, "wb") do |csv| all_attributes = ["refdes","qty"]+attributes csv << all_attributes bom_data.each do |line| csv << all_attributes.collect{|attribute| line[attribute]} end end end end desc "generate cost estimation from schematic" targets.each do |target| # this version uses Digi-Key and AliExpress # Digi-Key is easily scrapable, while Mouser isn't # Digi-Key is only one distributor, but the end prices across distributor is often similar file "#{target[:name]}_cost.csv" => target[:sch] do |t| sellers = ['digikey-id','aliexpress-id'] # get seller SKU boards = [1,10,100] # calculate the price for as many boards total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board # get component information attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get parts = bom2(t.prerequisites[0],attributes) # get field values # put result in CVS CSV.open(t.name, "wb") do |csv| csv << ["refdes","quantity","manufacturer","part number"]+(sellers.collect{|seller| [seller,"stock"]+boards.collect{|qty| ["unit price for #{qty} board(s)","total price for #{qty} board(s)"]}}).flatten parts.each do |part| part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation line = [part['refdes'],part['qty'],part['manufacturer'],part['manufacturer-id']] # start CSV line sellers.each_index do |seller_i| # go through all seller seller = sellers[seller_i] # current seller if part[seller] and !part[seller].empty? then line << part[seller] price = case seller when 'aliexpress-id' scrape_aliexpress(part[seller]) when 'digikey-id' scrape_digikey(part[seller]) else nil end if price then line << price[:stock] boards.each_index do |boards_i| quantity = boards[boards_i] # find lowest price (considering the quantity and quantity prices) unit = nil total = nil price[:prices].each do |p| if !unit or !total then unit = p[1].to_f total = [quantity,p[0].to_i].max*unit end if [quantity,p[0].to_i].max*p[1].to_f target[:sch] do |t| sellers = ['digikey-id','farnell-id','mouser-id','aliexpress-id'] # get seller SKU SELLER_ID = {'digikey-id' => 459, 'farnell-id' => 819, 'mouser-id' => 2401} # octopart seller IDs boards = [1,10,100] # calculate the price for as many boards sum = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total cost stocks = Array.new(sellers.size){Array.new(boards.size, true)} # is there enough stock # get component information attributes = ["value","manufacturer","manufacturer-id","octopart-id"]+sellers # BOM fields to get parts = bom2(t.prerequisites[0],attributes) # get field values # put result in CVS CSV.open(t.name, "wb") do |csv| csv << ["refdes","quantity","manufacturer","part number"]+(sellers.collect{|seller| [seller,"stock"]+boards.collect{|qty| ["unit price for #{qty} board(s)","total price for #{qty} board(s)"]}}).flatten parts.each do |part| part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation line = [part['refdes'],part['qty'],part['manufacturer'],part['manufacturer-id']] # start CSV line # get prices prices = nil if part['octopart-id'] and !part['octopart-id'].empty? then prices = octopart_prices(part['octopart-id'], nil, nil, nil, nil) elsif part['manufacturer'] and !part['manufacturer'].empty? and part['manufacturer-id'] and !part['manufacturer-id'].empty? prices = octopart_prices(nil, part['manufacturer'], part['manufacturer-id'], nil, nil) end # get prices per seller sellers.each_index do |seller_i| seller = sellers[seller_i] # current seller if 'aliexpress-id'==seller then # AliExpress is an exception since it's not in octopart if part[seller] and !part[seller].empty? then # AliExpress link provided price = scrape_aliexpress(part[seller]) if price then line << part[seller] line << price[:stock] boards.each_index do |boards_i| quantity = boards[boards_i] line << price[:price] if quantity target[:vpcb] do |t| sh "pcb -x png --dpi 600 --format PNG --photo-mode --outfile #{t.name} #{t.prerequisites.join(' ')}" end end desc "generate photo realistic picture from layout (bottom side)" targets.each do |target| file "#{target[:name]}_layout-bottom.png" => target[:vpcb] do |t| sh "pcb -x png --dpi 600 --format PNG --photo-mode --photo-flip-x --outfile #{t.name} #{t.prerequisites.join(' ')}" end end desc "create archive with release files" SOURCES = targets.collect{|target| [target[:sch],target[:pcb]]}.flatten ATTACHMENTS = ["cern_ohl_v_1_2_howto.pdf","CHANGES.txt","LICENSE.txt","PRODUCT.txt"] file release => SOURCES+prints+notes+boms+costs+photos+ATTACHMENTS do |t| gerbers = Dir.entries(".").select{|file| file.end_with? ".gbr" or file.end_with? ".cnc"} sh "tar --create --auto-compress --file '#{t.name}' #{(t.prerequisites+gerbers).join(' ')}" end