435 lines
16 KiB
Ruby
435 lines
16 KiB
Ruby
# encoding: utf-8
|
|
# ruby: 2.1.0
|
|
=begin
|
|
Rakefile to manage hardware projects
|
|
|
|
uses Lepton EDA for schematic and pcb-rnd for board layouts.
|
|
Rakefile instead of Makefile for better text file parsing capabilities.
|
|
=end
|
|
require 'rake/clean'
|
|
require 'csv' # to export BOM and costs
|
|
require 'open-uri' # to parse URLs
|
|
require 'open_uri_redirections' # allow redirections
|
|
require 'net/http' # to get sites
|
|
require 'nokogiri' # to parse sites
|
|
require 'json' # to parse GET site responses
|
|
|
|
# =================
|
|
# project variables
|
|
# =================
|
|
|
|
# common name used for file names
|
|
name = "usb_cable_tester"
|
|
# project version, read from "version" file
|
|
raise "define project version in 'version' file" unless File.exist? "version"
|
|
version = IO.read("version").split("\n")[0]
|
|
# current date for stamping output
|
|
date = Time.now.strftime("%Y-%m-%d")
|
|
# revision based on number of changes on schematic or board layout and current git commit
|
|
changes = `git log --pretty=oneline "#{name}.sch" "#{name}.lht" | wc -l`.chomp.to_i
|
|
commit = `git rev-parse --short HEAD`.chomp
|
|
revision = "#{changes} (#{commit})"
|
|
|
|
# local QEDA parts
|
|
parts_local = []
|
|
parts_local << "mcu/ic_mcu_st_stm32f103xc@LQFP144"
|
|
parts_local << "connector/connector_usb-a-3.0_fci_10117835"
|
|
parts_local << "connector/connector_usb-b-3.0_amphenol_gsb4211"
|
|
parts_local << "connector/connector_usb-mini-b_edac_690-005-299-043"
|
|
parts_local << "connector/connector_usb-micro-b-3.0_gct_usb3110"
|
|
parts_local << "connector/connector_usb-micro-b-2.0_ali_32910334970"
|
|
parts_local << "connector/connector_usb-c_hro_type-c-31-m-04"
|
|
parts_local << "connector/connector_lightning_iphone5"
|
|
parts_local << "vreg/vreg_ldo_torex_xc6206"
|
|
parts_local << "display/display_lcd_lcm1602"
|
|
parts_local << "display/display_lcd_i2cadapter"
|
|
parts_local << "display/display_oled_0.96in"
|
|
parts_local << "vreg/vreg_pmic_tpower_tp4056"
|
|
parts_local << "transistor/transistor_pmos_nxp_bss84"
|
|
parts_local << "mechanical/mechanical_button_6mm"
|
|
parts_local << "oscillator/hc-49@SM"
|
|
parts_local << "resistor/trimpot_3mm"
|
|
parts_local << "diode/diode_tvs_st_usblc6-2@SC6"
|
|
parts_local << "connector/connector_molex_530470210"
|
|
parts_local << "mcu/ic_mcu_st_stm8s003x3@P"
|
|
# the corresponding files
|
|
library = parts_local.collect {|part| "library/#{part.split('@')[0].downcase}.yaml"}
|
|
# github QEDA parts
|
|
parts_github = []
|
|
parts_github << "capacitor/c0603"
|
|
parts_github << "resistor/r0603"
|
|
parts_github << "te/1206sfh"
|
|
# path to qeda"
|
|
qeda = "qeda"
|
|
|
|
# ==========
|
|
# main tasks
|
|
# ==========
|
|
|
|
desc "main building task"
|
|
task :default => [:print, :fabrication, :bom]
|
|
|
|
desc "print schematic and layout (as pdf)"
|
|
prints = [ "#{name}.sch.pdf", "#{name}.brd.ps", "#{name}.brd-top.png", "#{name}.brd-bottom.png" ]
|
|
task :print => prints
|
|
CLEAN.include([ "#{name}.versioned.sch", "#{name}.versioned.lht" ])
|
|
CLOBBER.include(prints)
|
|
|
|
desc "generate fabrication gerbers (as archive)"
|
|
gerbers = [ "#{name}.brd.asb", "#{name}.brd.ast", "#{name}.brd.gbl", "#{name}.brd.gbo", "#{name}.brd.gbp", "#{name}.brd.gbs", "#{name}.brd.gko", "#{name}.brd.gtl", "#{name}.brd.gto", "#{name}.brd.gtp", "#{name}.brd.gts", "#{name}.brd.xln" ]
|
|
fab = [ "#{name}.brd.zip" ]
|
|
task :fabrication => fab
|
|
CLEAN.include(gerbers)
|
|
CLOBBER.include(fab)
|
|
|
|
desc "generate footprints from parts"
|
|
task :library => library do
|
|
# reset
|
|
sh "#{qeda} reset"
|
|
# configure
|
|
sh "#{qeda} config pattern.preferManufacturer false"
|
|
sh "#{qeda} config pattern.densityLevel N"
|
|
sh "#{qeda} config pattern.smoothPadCorners true"
|
|
# add local files
|
|
parts_local.each do |part|
|
|
sh "#{qeda} add #{part}"
|
|
end
|
|
# from github library
|
|
parts_github.each do |part|
|
|
sh "#{qeda} add #{part}"
|
|
end
|
|
# generate outputs
|
|
sh "#{qeda} config output geda"
|
|
sh "#{qeda} generate ."
|
|
sh "#{qeda} config output coraleda"
|
|
sh "#{qeda} generate ."
|
|
end
|
|
|
|
desc "export netlist from schematic"
|
|
net = [ "#{name}.tdx" ]
|
|
task :netlist => net
|
|
CLOBBER.include(net)
|
|
|
|
desc "export notes from schematic"
|
|
notes = [ "#{name}.notes.txt" ]
|
|
task :notes => notes
|
|
CLOBBER.include(notes)
|
|
|
|
desc "export BOMs from schematic"
|
|
boms = [ "#{name}.bom.csv" ]
|
|
task :bom => boms
|
|
CLOBBER.include(boms)
|
|
|
|
desc "generate cost estimte"
|
|
costs = [ "#{name}.cost.csv" ]
|
|
task :cost => costs
|
|
CLOBBER.include(costs)
|
|
|
|
# ===============
|
|
# file generation
|
|
# ===============
|
|
|
|
desc "generate schematic with version information all symbols embedded"
|
|
rule ".versioned.sch" => ".sch" do |t|
|
|
sh "cp #{t.source} #{t.name}"
|
|
sh "lepton-embed --embed #{t.name} 2> /dev/null"
|
|
sh "sed --in-place 's/\\$version\\$/#{version}/' #{t.name}"
|
|
sh "sed --in-place 's/\\$date\\$/#{date}/' #{t.name}"
|
|
sh "sed --in-place 's/\\$revision\\$/#{revision}/' #{t.name}"
|
|
end
|
|
|
|
desc "generate board layout with version information"
|
|
rule ".versioned.lht" => ".lht" do |t|
|
|
sh "cp #{t.source} #{t.name}"
|
|
sh "sed --in-place 's/\\$version\\$/#{version}/' #{t.name}"
|
|
sh "sed --in-place 's/\\$date\\$/#{date}/' #{t.name}"
|
|
sh "sed --in-place 's/\\$revision\\$/#{revision}/' #{t.name}"
|
|
end
|
|
|
|
desc "generate printable version (PDF) of schematic"
|
|
rule ".sch.pdf" => ".versioned.sch" do |t|
|
|
sh "lepton-cli export --color --paper iso_a4 --layout landscape --color --output #{t.name} #{t.source} 2> /dev/null"
|
|
end
|
|
|
|
desc "generate printable version (PostScript) of board layout"
|
|
rule ".brd.ps" => ".versioned.lht" do |t|
|
|
sh "pcb-rnd -x ps --psfile #{t.name} #{t.source} 2> /dev/null"
|
|
end
|
|
|
|
desc "generate photo realistic picture from layout (top side)"
|
|
rule ".brd-top.png" => ".versioned.lht" do |t|
|
|
sh "pcb-rnd -x png --dpi 1200 --photo-mode --outfile #{t.name} #{t.source} 2> /dev/null"
|
|
end
|
|
|
|
desc "generate photo realistic picture from layout (bottom side)"
|
|
rule ".brd-bottom.png" => ".versioned.lht" do |t|
|
|
sh "pcb-rnd -x png --dpi 1200 --photo-mode --photo-flip-y --outfile #{t.name} #{t.source} 2> /dev/null"
|
|
end
|
|
|
|
desc "archive gerbers"
|
|
rule ".brd.zip" => ".versioned.lht" do |t|
|
|
base = File.basename(t.source, ".versioned.lht")
|
|
puts base
|
|
sh "pcb-rnd -x cam gerber:JLC_PCB --outfile #{base}.brd #{t.source} 2> /dev/null"
|
|
gerbers = [ "#{base}.brd.asb", "#{base}.brd.ast", "#{base}.brd.gbl", "#{base}.brd.gbo", "#{base}.brd.gbp", "#{base}.brd.gbs", "#{base}.brd.gko", "#{base}.brd.gtl", "#{base}.brd.gto", "#{base}.brd.gtp", "#{base}.brd.gts", "#{base}.brd.xln" ]
|
|
fab = [ "#{name}.brd.zip" ]
|
|
sh "zip --quiet #{t.name} #{gerbers.join(' ')}"
|
|
end
|
|
|
|
desc "export netlist from schematic"
|
|
rule ".tdx" => ".sch" do |t|
|
|
sh "lepton-netlist -g tEDAx -o #{t.name} #{t.source} 2> /dev/null"
|
|
end
|
|
|
|
desc "generate note file from schematic, listing the 'note' attributes from elements"
|
|
rule ".notes.txt" => ".sch" do |t|
|
|
notes_data = bom2(t.prerequisites[0], ["note", "value"])
|
|
File.open(t.name,"w") do |notes_file|
|
|
notes_data.each do |note|
|
|
next unless note['note']
|
|
note['note'] = note['note'].gsub('. ',".\n").gsub(/\n+$/,'')
|
|
notes_file.puts "#{note['value']} (#{note['refdes']}):\n#{note['note']}\n\n"
|
|
end
|
|
end
|
|
end
|
|
|
|
desc "generate BOM file from schematic"
|
|
rule ".bom.csv" => ".sch" do |t|
|
|
attributes = ["category", "device", "value", "description", "manufacturer", "manufacturer-id", "datasheet", "lcsc-id", "aliexpress-id", "alternatives"]
|
|
bom_data = bom2(t.prerequisites[0],attributes)
|
|
CSV.open(t.name, "wb") do |csv|
|
|
all_attributes = ["refdes","qty"]+attributes
|
|
csv << all_attributes
|
|
bom_data.each do |line|
|
|
csv << all_attributes.collect{|attribute| line[attribute]}
|
|
end
|
|
end
|
|
end
|
|
|
|
desc "generate cost estimate from schematic"
|
|
# this version uses Digi-Key, AliExpress, and LCSC
|
|
# Digi-Key is easily scrapable, while Mouser isn't
|
|
# Digi-Key is only one distributor, but the end prices across distributor is often similar
|
|
rule ".cost.csv" => ".sch" do |t|
|
|
puts "scraping distributor sites to get prices. this may take some time"
|
|
sellers = ['digikey-id','aliexpress-id','lcsc-id'] # get seller SKU
|
|
boards = [1, 10, 100] # calculate the price for as many boards
|
|
total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards
|
|
unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board
|
|
# get component information
|
|
attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get
|
|
parts = bom2(t.prerequisites[0], attributes) # get field values
|
|
# put result in CVS
|
|
CSV.open(t.name, "wb") do |csv|
|
|
csv << ["refdes", "quantity", "manufacturer", "part number"] + (sellers.collect{|seller| [seller, "stock"] + boards.collect{|qty| ["unit price for #{qty} board(s)", "total price for #{qty} board(s)"]}}).flatten
|
|
parts.each do |part|
|
|
part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation
|
|
line = [part['refdes'], part['qty'], part['manufacturer'], part['manufacturer-id']] # start CSV line
|
|
sellers.each_index do |seller_i| # go through all seller
|
|
seller = sellers[seller_i] # current seller
|
|
if part[seller] and !part[seller].empty? then
|
|
line << part[seller]
|
|
price = case seller
|
|
when 'aliexpress-id'
|
|
scrape_aliexpress(part[seller])
|
|
when 'digikey-id'
|
|
nil
|
|
when 'lcsc-id'
|
|
scrape_lcsc(part[seller])
|
|
else
|
|
nil
|
|
end
|
|
if price then
|
|
line << price[:stock]
|
|
boards.each_index do |boards_i|
|
|
quantity = boards[boards_i]
|
|
# find lowest price (considering the quantity and quantity prices)
|
|
unit = nil
|
|
total = nil
|
|
price[:prices].each do |p|
|
|
if !unit or !total then
|
|
unit = p[1].to_f
|
|
total = [quantity, p[0].to_i].max * unit
|
|
end
|
|
if [quantity, p[0].to_i].max * p[1].to_f < total then
|
|
unit = p[1].to_f
|
|
total = [quantity, p[0].to_i].max * unit
|
|
end
|
|
end
|
|
if "USD" == price[:currency] then
|
|
unit = usd2eur(unit)
|
|
total = usd2eur(total)
|
|
end
|
|
line << unit
|
|
unit_price[seller_i][boards_i] += line[-1]
|
|
line << total
|
|
total_price[seller_i][boards_i] += line[-1]
|
|
end
|
|
else
|
|
line += [nil] * (1 + boards.size * 2)
|
|
end
|
|
else
|
|
line += [nil] * (2 + boards.size * 2)
|
|
end # seller
|
|
end # sellers
|
|
csv << line
|
|
end # parts
|
|
# summary
|
|
line = [nil] * 4
|
|
sellers.each_index do |seller_i|
|
|
line += [nil, nil]
|
|
boards.each_index do |boards_i|
|
|
line << unit_price[seller_i][boards_i]
|
|
line << total_price[seller_i][boards_i]
|
|
end
|
|
end
|
|
csv << line
|
|
# details
|
|
csv << []
|
|
csv << ["all prices and stocks have been retrieved from Digikey, AliExpress, and LCSC on #{Time.now.to_s}"]
|
|
csv << ["all prices are in EUR. prices originally in USD have been converted at a rate of #{usd2eur(1.0)}"]
|
|
end # CSV file
|
|
end # end cost file
|
|
|
|
# ================
|
|
# helper functions
|
|
# ================
|
|
|
|
# generate gnetlist bom2 and parse them
|
|
# arguments: schematic=schematic to use, attributes=attributes to use for generating bom2
|
|
# returns an array of hash. key is the attribute name, value is the attribute value
|
|
def bom2(schematic, attributes)
|
|
to_return = []
|
|
# force attributes to be an array
|
|
attributes = case attributes
|
|
when String
|
|
[attributes]
|
|
when Array
|
|
attributes
|
|
else
|
|
[attributes.to_s]
|
|
end
|
|
# generate bom2
|
|
list = `lepton-netlist --backend bom2 --backend-option attribs=#{attributes*','} --quiet --output - #{schematic} 2> /dev/null`
|
|
list = list.each_line {|l| '"' + l + '"' + '\n' }
|
|
list.gsub!(/^(.+)/, '"\1')
|
|
list.gsub!(/(.+)$/, '\1"')
|
|
list.gsub!(/(?!http):(?!\/\/)/, '\1":"\2') # protect the values between ':' (such as URLs)
|
|
# parse bom2
|
|
csv = CSV.parse(list, col_sep: ":", quote_char: '"')
|
|
csv[1..-1].each do |row|
|
|
line = {}
|
|
row.each_index do |col|
|
|
line[csv[0][col]] = row[col] unless row[col] == "unknown"
|
|
end
|
|
to_return << line
|
|
end
|
|
return to_return
|
|
end
|
|
|
|
# convert USD $ value to EUR €
|
|
def usd2eur(usd)
|
|
return usd / eur2usd(1.0)
|
|
end
|
|
|
|
# convert EUR € value to USD $
|
|
def eur2usd(eur)
|
|
# get rate if we don't have already
|
|
unless $eur2usd then
|
|
url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
|
|
doc = Nokogiri::HTML(open(URI.escape(url), :allow_redirections => :all))
|
|
$eur2usd = doc.xpath('//cube[@currency="USD"]')[0].attr('rate').to_f
|
|
end
|
|
return eur * $eur2usd
|
|
end
|
|
|
|
# get prices from LCSC using SKU
|
|
def scrape_lcsc(sku)
|
|
to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
|
|
# get page
|
|
# the search page does not always list existing parts, instead it will try
|
|
url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
|
|
puts "scraping #{url}" if $scrape_debug
|
|
doc = Nokogiri::HTML(URI.open(url),:allow_redirections => :all)
|
|
|
|
# verify if we have got a product page
|
|
if doc.xpath('//div[@id="product_details"]').empty? then
|
|
puts "no product details:\n #{doc}" if $scrape_debug
|
|
return nil
|
|
end
|
|
doc.xpath('//div[@id="product_details"]//div[contains(@class,"stock-number")]').each do |element|
|
|
next unless element["data-stock"]
|
|
to_return[:stock] = element["data-stock"]
|
|
end
|
|
to_return[:currency] = "USD" # we could verify in the price, but I'm lazy
|
|
doc.xpath('//input[contains(@class,"salam-price")]').each do |element|
|
|
next unless element["data-price"]
|
|
to_return[:prices] = []
|
|
element["data-price"].split("],[").each do |price|
|
|
price.gsub!("[", "")
|
|
price.gsub!("]", "")
|
|
price = price.split(",")
|
|
to_return[:prices] << [price[0].to_i, price[1].to_f]
|
|
end
|
|
end
|
|
return to_return
|
|
end
|
|
|
|
# get prices from AliExpress using SKU
|
|
def scrape_aliexpress(sku)
|
|
to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
|
|
# get page
|
|
url = "https://www.aliexpress.com/item/#{sku}.html"
|
|
puts "scraping #{url}" if $scrape_debug
|
|
doc = Nokogiri::HTML(URI.open(url),:allow_redirections => :all)
|
|
# all the values can be found in javascript variables (stock is even only there)
|
|
js_docs = doc.xpath('//script')
|
|
if js_docs.empty? then
|
|
puts "script not found:\n#{doc}" if $scrape_debug
|
|
return nil
|
|
end
|
|
js_json = nil
|
|
js_docs.each do |js_doc|
|
|
js_text = js_doc.text
|
|
next unless js_text.include?("window.runParams = {")
|
|
js_var = js_text.split('data: ')[1].split('csrfToken: ')[0].gsub(/,[\w\n]*$/, '')
|
|
js_json = JSON.parse(js_var)
|
|
end
|
|
unless js_json and js_json["priceModule"] then
|
|
puts "priceModule not found:\n#{js_json}" if $scrape_debug
|
|
return nil
|
|
end
|
|
# get currency
|
|
unless js_json["priceModule"]["formatedPrice"] then
|
|
puts "currency not found:\n#{js_json['priceModule']}" if $scrape_debug
|
|
return nil
|
|
end
|
|
if js_json["priceModule"]["formatedPrice"].start_with? "US" then
|
|
to_return[:currency] = "USD"
|
|
elsif js_json["priceModule"]["formatedPrice"].start_with? "EU" then
|
|
to_return[:currency] = "EUR"
|
|
end
|
|
# get quantity
|
|
unless js_json["quantityModule"] and js_json["quantityModule"]["totalAvailQuantity"] then
|
|
puts "quantityModule not found:\n#{js_json}" if $scrape_debug
|
|
return nil
|
|
end
|
|
to_return[:stock] = js_json["quantityModule"]["totalAvailQuantity"]
|
|
# get price
|
|
unless js_json["priceModule"]["numberPerLot"] and (js_json["priceModule"]["formatedActivityPrice"] or js_json["priceModule"]["formatedPrice"] ) then
|
|
puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
|
|
return nil
|
|
end
|
|
to_return[:prices] = []
|
|
lot = js_json["priceModule"]["numberPerLot"].to_i
|
|
price = js_json["priceModule"]["formatedActivityPrice"] || js_json["priceModule"]["formatedPrice"]
|
|
unless price then
|
|
puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
|
|
return nil
|
|
end
|
|
price = price.split('$')[1].to_f
|
|
to_return[:prices] << [ lot, price / lot ]
|
|
|
|
return to_return
|
|
end
|