# encoding: utf-8
# ruby: 2.1.0
=begin
Rakefile to manage hardware projects

uses Lepton EDA for schematic and pcb-rnd for board layouts.
Rakefile instead of Makefile for better text file parsing capabilities.
=end
require 'rake/clean'
require 'csv' # to export BOM and costs
require 'open-uri' # to parse URLs
require 'open_uri_redirections' # allow redirections
require 'net/http' # to get sites
require 'nokogiri' # to parse sites
require 'json' # to parse GET site responses

# =================
# project variables
# =================

# common name used for file names
name = "usb_cable_tester"
# project version, read from "version" file
raise "define project version in 'version' file" unless File.exist? "version"
version = IO.read("version").split("\n")[0]
# current date for stamping output
date = Time.now.strftime("%Y-%m-%d")
# revision based on number of changes on schematic or board layout and current git commit
changes = `git log --pretty=oneline "#{name}.sch" "#{name}.lht" | wc -l`.chomp.to_i
commit = `git rev-parse --short HEAD`.chomp
revision = "#{changes} (#{commit})"

# local QEDA parts
parts_local = []
parts_local << "mcu/ic_mcu_st_stm32f103xc@LQFP144"
parts_local << "connector/connector_usb-a-3.0_fci_10117835"
parts_local << "connector/connector_usb-b-3.0_amphenol_gsb4211"
parts_local << "connector/connector_usb-mini-b_edac_690-005-299-043"
parts_local << "connector/connector_usb-micro-b-3.0_gct_usb3110"
parts_local << "connector/connector_usb-micro-b-2.0_ali_32910334970"
parts_local << "connector/connector_usb-c_hro_type-c-31-m-04"
parts_local << "connector/connector_lightning_iphone5"
parts_local << "vreg/vreg_ldo_torex_xc6206"
parts_local << "connector/connector_battery_18650"
parts_local << "display/display_lcd_lcm1602"
parts_local << "display/display_lcd_i2cadapter"
parts_local << "display/display_oled_0.96in"
parts_local << "vreg/vreg_pmic_tpower_tp4056"
parts_local << "transistor/transistor_pmos_nxp_bss84"
parts_local << "mechanical/mechanical_button_6mm"
parts_local << "connector/connector_XH-2.54-2P"
# the corresponding files
library = parts_local.collect {|part| "library/#{part.split('@')[0].downcase}.yaml"}
#  github QEDA parts
parts_github = []
parts_github << "ecs/csm-7"
parts_github << "capacitor/c0603"
parts_github << "resistor/r0603"
parts_github << "diode/led0603"
# path to qeda"
qeda = "../qeda/bin/qeda"

# ==========
# main tasks
# ==========

desc "main building task"
task :default => [:print, :fabrication, :bom]

desc "print schematic and layout (as pdf)"
prints = [ "#{name}.sch.pdf", "#{name}.brd.ps", "#{name}.brd-top.png", "#{name}.brd-bottom.png" ]
task :print => prints
CLEAN.include([ "#{name}.versioned.sch", "#{name}.versioned.lht" ])
CLOBBER.include(prints)

desc "generate fabrication gerbers (as archive)"
gerbers = [ "#{name}.brd.asb", "#{name}.brd.ast", "#{name}.brd.gbl", "#{name}.brd.gbo", "#{name}.brd.gbp", "#{name}.brd.gbs", "#{name}.brd.gko", "#{name}.brd.gtl", "#{name}.brd.gto", "#{name}.brd.gtp", "#{name}.brd.gts", "#{name}.brd.xln" ]
fab = [ "#{name}.brd.zip" ]
task :fabrication => fab
CLEAN.include(gerbers)
CLOBBER.include(fab)

desc "generate footprints from parts"
task :library => library do
  # reset
  sh "#{qeda} reset"
  # configure
  sh "#{qeda} config pattern.preferManufacturer false"
  sh "#{qeda} config pattern.densityLevel M"
  sh "#{qeda} config pattern.smoothPadCorners true"
  # add local files
  parts_local.each do |part|
    sh "#{qeda} add #{part}"
  end
  # from github library
  parts_github.each do |part|
    sh "#{qeda} add #{part}"
  end
  # generate outputs
  sh "#{qeda} config output geda"
  sh "#{qeda} generate ."
  sh "#{qeda} config output coraleda"
  sh "#{qeda} generate ."
end

desc "export netlist from schematic"
net = [ "#{name}.tdx" ]
task :netlist => net
CLOBBER.include(net)

desc "export notes from schematic"
notes = [ "#{name}.notes.txt" ]
task :notes => notes
CLOBBER.include(notes)

desc "export BOMs from schematic"
boms = [ "#{name}.bom.csv" ]
task :bom => boms
CLOBBER.include(boms)

desc "generate cost estimte"
costs = [ "#{name}.cost.csv" ]
task :cost => costs
CLOBBER.include(costs)

# ===============
# file generation
# ===============

desc "generate schematic with version information all symbols embedded"
rule ".versioned.sch" => ".sch" do |t|
  sh "cp #{t.source} #{t.name}"
  sh "lepton-embed --embed #{t.name} 2> /dev/null"
  sh "sed --in-place 's/\\$version\\$/#{version}/' #{t.name}"
  sh "sed --in-place 's/\\$date\\$/#{date}/' #{t.name}"
  sh "sed --in-place 's/\\$revision\\$/#{revision}/' #{t.name}"
end

desc "generate board layout with version information"
rule ".versioned.lht" => ".lht" do |t|
  sh "cp #{t.source} #{t.name}"
  sh "sed --in-place 's/\\$version\\$/#{version}/' #{t.name}"
  sh "sed --in-place 's/\\$date\\$/#{date}/' #{t.name}"
  sh "sed --in-place 's/\\$revision\\$/#{revision}/' #{t.name}"
end

desc "generate printable version (PDF) of schematic"
rule ".sch.pdf" => ".versioned.sch" do |t|
  sh "lepton-cli export --color --paper iso_a4 --layout landscape --color --output #{t.name} #{t.source} 2> /dev/null"
end

desc "generate printable version (PostScript) of board layout"
rule ".brd.ps" => ".versioned.lht" do |t|
  sh "pcb-rnd -x ps --psfile #{t.name} #{t.source} 2> /dev/null"
end

desc "generate photo realistic picture from layout (top side)"
rule ".brd-top.png" => ".versioned.lht" do |t|
  sh "pcb-rnd -x png --dpi 1200 --photo-mode --outfile #{t.name} #{t.source} 2> /dev/null"
end

desc "generate photo realistic picture from layout (bottom side)"
rule ".brd-bottom.png" => ".versioned.lht" do |t|
  sh "pcb-rnd -x png --dpi 1200 --photo-mode --photo-flip-y --outfile #{t.name} #{t.source} 2> /dev/null"
end

desc "archive gerbers"
rule ".brd.zip" => ".versioned.lht" do |t|
  base = File.basename(t.source, ".versioned.lht")
  puts base
  sh "pcb-rnd -x cam gerber:JLC_PCB --outfile #{base}.brd #{t.source} 2> /dev/null"
  gerbers = [ "#{base}.brd.asb", "#{base}.brd.ast", "#{base}.brd.gbl", "#{base}.brd.gbo", "#{base}.brd.gbp", "#{base}.brd.gbs", "#{base}.brd.gko", "#{base}.brd.gtl", "#{base}.brd.gto", "#{base}.brd.gtp", "#{base}.brd.gts", "#{base}.brd.xln" ]
fab = [ "#{name}.brd.zip" ]
  sh "zip --quiet #{t.name} #{gerbers.join(' ')}"
end

desc "export netlist from schematic"
rule ".tdx" => ".sch" do |t|
  sh "lepton-netlist -g tEDAx -o #{t.name} #{t.source} 2> /dev/null"
end

desc "generate note file from schematic, listing the 'note' attributes from elements"
rule ".notes.txt" => ".sch" do |t|
  notes_data = bom2(t.prerequisites[0], ["note", "value"])
  File.open(t.name,"w") do |notes_file|
    notes_data.each do |note|
      next unless note['note']
      note['note'] = note['note'].gsub('. ',".\n").gsub(/\n+$/,'')
      notes_file.puts "#{note['value']} (#{note['refdes']}):\n#{note['note']}\n\n"
    end
  end
end

desc "generate BOM file from schematic"
rule ".bom.csv" => ".sch" do |t|
  attributes = ["category", "device", "value", "description", "manufacturer", "manufacturer-id", "datasheet", "lcsc-id", "aliexpress-id", "alternatives"]
  bom_data = bom2(t.prerequisites[0],attributes)
  CSV.open(t.name, "wb") do |csv|
    all_attributes = ["refdes","qty"]+attributes
    csv << all_attributes
    bom_data.each do |line|
      csv << all_attributes.collect{|attribute| line[attribute]}
    end
  end
end

desc "generate cost estimate from schematic"
# this version uses Digi-Key, AliExpress, and LCSC
# Digi-Key is easily scrapable, while Mouser isn't
# Digi-Key is only one distributor, but the end prices across distributor is often similar
rule ".cost.csv" => ".sch" do |t|
  puts "scraping distributor sites to get prices. this may take some time"
  sellers = ['digikey-id','aliexpress-id','lcsc-id'] # get seller SKU
  boards = [1, 10, 100] # calculate the price for as many boards
  total_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # total price for x boards
  unit_price = Array.new(sellers.size){Array.new(boards.size, 0.0)} # unit price for 1 board
  # get component information
  attributes = ["value","manufacturer","manufacturer-id"]+sellers # BOM fields to get
  parts = bom2(t.prerequisites[0], attributes) # get field values
  # put result in CVS
  CSV.open(t.name, "wb") do |csv|
    csv << ["refdes", "quantity", "manufacturer", "part number"] + (sellers.collect{|seller| [seller, "stock"] + boards.collect{|qty| ["unit price for #{qty} board(s)", "total price for #{qty} board(s)"]}}).flatten
    parts.each do |part|
      part['qty'] = part['qty'].to_i # converted quantity from BOM string to integer for later calculation
      line = [part['refdes'], part['qty'], part['manufacturer'], part['manufacturer-id']] # start CSV line
      sellers.each_index do |seller_i| # go through all seller
        seller = sellers[seller_i] # current seller
        if part[seller] and !part[seller].empty? then
          line << part[seller]
          price = case seller
          when 'aliexpress-id'
            scrape_aliexpress(part[seller])
          when 'digikey-id'
            nil
          when 'lcsc-id'
            scrape_lcsc(part[seller])
          else
            nil
          end
          if price then
            line << price[:stock]
            boards.each_index do |boards_i|
              quantity = boards[boards_i]
              # find lowest price (considering the quantity and quantity prices)
              unit = nil
              total = nil
              price[:prices].each do |p|
                if !unit or !total then
                  unit = p[1].to_f
                  total = [quantity, p[0].to_i].max * unit
                end
                if [quantity, p[0].to_i].max * p[1].to_f < total then
                  unit = p[1].to_f
                  total = [quantity, p[0].to_i].max * unit
                end
              end
              if "USD" == price[:currency] then
                unit = usd2eur(unit)
                total = usd2eur(total)
              end
              line << unit
              unit_price[seller_i][boards_i] += line[-1]
              line << total
              total_price[seller_i][boards_i] += line[-1]
            end
          else
            line += [nil] * (1 + boards.size * 2)
          end
        else
          line += [nil] * (2 + boards.size * 2)
        end # seller
      end # sellers
      csv << line
    end # parts
    # summary
    line = [nil] * 4
    sellers.each_index do |seller_i|
      line += [nil, nil]
      boards.each_index do |boards_i|
        line << unit_price[seller_i][boards_i]
        line << total_price[seller_i][boards_i]
      end
    end
    csv << line
    # details
    csv << []
    csv << ["all prices and stocks have been retrieved from Digikey, AliExpress, and LCSC on #{Time.now.to_s}"]
    csv << ["all prices are in EUR. prices originally in USD have been converted at a rate of #{usd2eur(1.0)}"]
  end # CSV file
end # end cost file

# ================
# helper functions
# ================

# generate gnetlist bom2 and parse them
# arguments: schematic=schematic to use, attributes=attributes to use for generating bom2
# returns an array of hash. key is the attribute name, value is the attribute value
def bom2(schematic, attributes)
  to_return = []
  # force attributes to be an array
  attributes = case attributes
  when String
   [attributes]
  when Array
    attributes
  else
    [attributes.to_s]
  end
  # generate bom2
  list = `lepton-netlist --backend bom2 --backend-option attribs=#{attributes*','} --quiet --output - #{schematic} 2> /dev/null`
  list = list.each_line {|l| '"' + l + '"' + '\n' }
  list.gsub!(/^(.+)/, '"\1')
  list.gsub!(/(.+)$/, '\1"')
  list.gsub!(/(?!http):(?!\/\/)/, '\1":"\2') # protect the values between ':' (such as URLs)
  # parse bom2
  csv = CSV.parse(list, col_sep: ":", quote_char: '"')
  csv[1..-1].each do |row|
    line = {}
    row.each_index do |col|
      line[csv[0][col]] = row[col] unless row[col] == "unknown"
    end
    to_return << line
  end
  return to_return
end

# convert USD $ value to EUR €
def usd2eur(usd)
  return usd / eur2usd(1.0)
end

# convert EUR € value to USD $
def eur2usd(eur)
  # get rate if we don't have already
  unless $eur2usd then
    url = "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
    doc = Nokogiri::HTML(open(URI.escape(url), :allow_redirections => :all))
    $eur2usd = doc.xpath('//cube[@currency="USD"]')[0].attr('rate').to_f
  end
  return eur * $eur2usd
end

# get prices from LCSC using SKU
def scrape_lcsc(sku)
  to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
  # get page
  # the search page does not always list existing parts, instead it will try 
  url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
  puts "scraping #{url}" if $scrape_debug
  doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))

  # verify if we have got a product page
  if doc.xpath('//div[@id="product_details"]').empty? then
    puts "no product details:\n #{doc}" if $scrape_debug
    return nil
  end
  doc.xpath('//div[@id="product_details"]//div[contains(@class,"stock-number")]').each do |element|
    next unless element["data-stock"]
    to_return[:stock] = element["data-stock"]
  end
  to_return[:currency] = "USD" # we could verify in the price, but I'm lazy
  doc.xpath('//input[contains(@class,"salam-price")]').each do |element|
    next unless element["data-price"]
    to_return[:prices] = []
    element["data-price"].split("],[").each do |price|
      price.gsub!("[", "")
      price.gsub!("]", "")
      price = price.split(",")
      to_return[:prices] << [price[0].to_i, price[1].to_f]
    end
  end
  return to_return
end

# get prices from AliExpress using SKU
def scrape_aliexpress(sku)
  to_return = {stock: nil, currency: nil, prices: nil} # information to return (lot price, unit quantity, unit stock)
  # get page
  url = "https://www.aliexpress.com/item/#{sku}.html"
  puts "scraping #{url}" if $scrape_debug
  doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
  # all the values can be found in javascript variables (stock is even only there)
  js_docs = doc.xpath('//script')
  if js_docs.empty? then
    puts "script not found:\n#{doc}" if $scrape_debug
    return nil
  end
  js_json = nil
  js_docs.each do |js_doc|
    js_text = js_doc.text
    next unless js_text.include?("window.runParams = {")
    js_var = js_text.split('data: ')[1].split('csrfToken: ')[0].gsub(/,[\w\n]*$/, '')
    js_json =  JSON.parse(js_var)
  end
  unless js_json and js_json["priceModule"] then
    puts "priceModule not found:\n#{js_json}" if $scrape_debug
    return nil
  end
  # get currency
  unless js_json["priceModule"]["formatedPrice"] then
    puts "currency not found:\n#{js_json['priceModule']}" if $scrape_debug
    return nil
  end
  if js_json["priceModule"]["formatedPrice"].start_with? "US" then
    to_return[:currency] = "USD"
  elsif js_json["priceModule"]["formatedPrice"].start_with? "EU" then
    to_return[:currency] = "EUR"
  end
  # get quantity
  unless js_json["quantityModule"] and js_json["quantityModule"]["totalAvailQuantity"] then
    puts "quantityModule not found:\n#{js_json}" if $scrape_debug
    return nil
  end
  to_return[:stock] = js_json["quantityModule"]["totalAvailQuantity"]
  # get price
  unless js_json["priceModule"]["numberPerLot"] and (js_json["priceModule"]["formatedActivityPrice"] or js_json["priceModule"]["formatedPrice"] ) then
    puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
    return nil
  end
  to_return[:prices] = []
  lot = js_json["priceModule"]["numberPerLot"].to_i
  price = js_json["priceModule"]["formatedActivityPrice"] || js_json["priceModule"]["formatedPrice"]
  unless price then
    puts "priceModule malformatted:\n#{js_json['priceModule']}" if $scrape_debug
    return nil
  end
  price = price.split('$')[1].to_f
  to_return[:prices] << [ lot, price / lot ]

  return to_return
end