partdb/server.rb

#!/usr/bin/env ruby
# encoding: utf-8
# ruby: 3.0.2
=begin
backend to query part database

Copyright (C) 2023 King Kévin <kingkevin@cuvoodoo.info>
SPDX-License-Identifier: GPL-3.0-or-later

to install sinatra
sudo pacman -S ruby-sinatra ruby-webrick
pikaur -S ruby-mysql2
=end
require 'set'
require 'mysql2'
require 'json'
require 'sinatra'
require 'uri'
require 'net/http'

# allow dumping crashes in browser
DEBUG = false
# maximum number of parts returned
PARTS_LIMIT = 100
# credentials for database
CREDENTIALS = "credentials.json"
# folder name for served pages
PUBLIC = "public"
# folder name for part attachments (in PUBLIC)
ATTACHMENTS = "attachments"
# port for this service
PORT = 4245

raise "database information #{CREDENTIALS} do not exist" unless File.file? CREDENTIALS

# open server
configure do
  if DEBUG then
    set :show_exceptions, true
    set :logging, true
  else
    set :show_exceptions, false
    set :environment, :production
    set :logging, false
  end
  set :protection, :except => :json_csrf

  set :bind, 'localhost'
  set :port, PORT
  set :public_folder, "public"
  set :static, true
end

before do
  response.headers["Access-Control-Allow-Origin"] = "*"
  response.headers["Access-Control-Allow-Headers"] = "Content-Type"
  if request.request_method == 'OPTIONS'
    response.headers["Access-Control-Allow-Methods"] = "GET,POST"
    halt 200
  end
  
  # all replies are only JSON
  content_type 'application/json'
  # open database
  credentials = {}
  JSON.parse(IO.read(CREDENTIALS)).each {|key,value| credentials[key.to_sym] = value}
  Mysql2::Client.default_query_options.merge!(:as => :hash)
  @db = Mysql2::Client.new(credentials)
end

after do
  response.headers["Access-Control-Allow-Origin"] = "*"
  response.headers["Access-Control-Allow-Headers"] = "Content-Type"
end

get '/' do
  redirect to('/index.html')
end

def get_part_by_id(id)
  return nil unless id
  statement = @db.prepare("SELECT part.id, part.name, part.description, part.details, part.datasheet, manufacturer.name AS manufacturer, part.mpn AS mpn, package.name AS package, part.page AS page, part.family AS parent, p2.name AS family FROM part LEFT JOIN package ON package.id = part.package LEFT JOIN manufacturer ON manufacturer.id = part.manufacturer LEFT JOIN part AS p2 ON p2.id = part.family WHERE part.id = ?")
  part = statement.execute(id).to_a[0]
  return nil unless part
  parent = get_part_by_id(part["parent"])
  # merge parent info
  if parent then
    part.each do |k,v|
      part[k] ||= parent[k]
    end
  end
  # add all distributors
  distributors = @db.query("SELECT * FROM distributor").to_a
  statement = @db.prepare("SELECT * FROM distribution WHERE part = ?")
  distributions = statement.execute(id).to_a
  distributors.each do |distributor|
    distributions.each do |distribution|
      if distribution["distributor"] == distributor["id"] then
       distributor["sku"] = distribution["sku"]
       distributor["url"] = distributor["product_page"].gsub("%s", distribution["sku"])
      end
    end
    distributor.delete("id")
    distributor.delete("homepage")
    distributor.delete("product_page")
  end
  part["distributors"] = distributors
  # add inventory
  statement = @db.prepare("SELECT location.name AS location, inventory.quantity AS stock FROM inventory LEFT JOIN location ON location.id = inventory.location WHERE inventory.part = ? ORDER BY inventory.quantity DESC LIMIT 1")
  inventory = statement.execute(id).to_a[0]
  if inventory then
    part["location"] = inventory["location"]
    part["stock"] = inventory["stock"]
  end
  # add properties
  part["properties"] = {}
  statement = @db.prepare("SELECT property.name AS name, property_value.value AS value FROM property_value JOIN property ON property.id = property_value.property WHERE property_value.part = ?")
  statement.execute(id).each do |row|
    part["properties"][row["name"]] ||= []
    part["properties"][row["name"]] << row["value"]
  end
  if parent then
    parent["properties"].each do |k,v|
      part["properties"][k] ||= v
    end
  end
  # add attachments
  part["attachments"] = []
  dir = PUBLIC + "/" + ATTACHMENTS + "/" + part["name"]
  if File.directory?(dir) then
    Dir.entries(dir).each do |file|
      path = dir + "/" + file
      next unless File.file? path
      part["attachments"] << ATTACHMENTS + "/" + part["name"] + "/" + file
    end
  end
  part["attachments"].sort!
  if parent then
    part["attachments"] += parent["attachments"]
  end
  # clean up
  delete = ["parent"]
  delete.each do |k|
    part.delete k
  end
  return part
end

def get_part_by_name(name)
  statement = @db.prepare("SELECT id FROM part WHERE part.name = ?")
  id = statement.execute(name).to_a[0]
  if id then
    return get_part_by_id(id["id"])
  else
    return nil
  end
end

get '/part/:name' do
  part = get_part_by_name(params['name'])
  halt 404 unless part
  part.to_json
end

get '/search/:terms' do
  terms = params['terms'].split(" ")
  terms.keep_if {|term| term.length >= 3}
  halt 400 if terms.empty?
  # search in names, description, and category
  statements = []
  statements << @db.prepare("SELECT id FROM part WHERE name LIKE ?")
  statements << @db.prepare("SELECT id FROM part WHERE mpn LIKE ?")
  statements << @db.prepare("SELECT id FROM part WHERE description LIKE ?")
  statements << @db.prepare("SELECT property_value.part AS id FROM property_value JOIN property ON property.id = property_value.property WHERE property.name = 'category' AND property_value.value LIKE ?")
  term_ids = []
  terms.each do |term|
    ids = Set.new
    # OR term location
    statements.each do |statement|
      statement.execute("%#{term}%").each do |row|
        ids << row["id"]
      end
    end
    term_ids << ids
  end
  # get all children
  statement = @db.prepare("SELECT id FROM part WHERE family IN (?)")
  term_ids.each do |term_id|
    statement.execute(term_id.to_a * ",").each do |row|
      term_id << row["id"]
    end
  end
  # AND terms
  ids = term_ids.shift
  term_ids.each do |term_id|
    ids &= term_id
  end
  parts = ids.collect {|id| get_part_by_id(id)}
  parts.compact!
  parts = parts[0, PARTS_LIMIT]
  parts.sort! {|x,y| x["name"] <=> y["name"]}
  parts.to_json
end

def delete_part(id)
  # first delete all children
  statement = @db.prepare("SELECT id FROM part WHERE family = ?")
  statement.execute(id).each do |row|
    puts "child: #{row['id']}"
    delete_part(row['id'])
    puts "deleted"
  end
  # delete all fields
  statements = []
  statements << @db.prepare("DELETE FROM property_value WHERE part = ?")
  statements << @db.prepare("DELETE FROM assembly WHERE assembled = ?")
  statements << @db.prepare("DELETE FROM assembly WHERE component = ?")
  statements << @db.prepare("DELETE FROM distribution WHERE part = ?")
  statements << @db.prepare("DELETE FROM property_value WHERE part = ?")
  statements << @db.prepare("DELETE FROM inventory WHERE part = ?")
  statements << @db.prepare("DELETE FROM part WHERE id = ?")
  statements.each do |statement|
    statement.execute(id)
  end
end

get '/delete/:id' do
  statement = @db.prepare("SELECT id FROM part WHERE id = ?")
  result = statement.execute(params['id'])
  halt 400 if result.to_a.empty?
  delete_part(params['id'])
  return 200
end

def add_part(part)
  if part["id"] then
    # ensure part to update exists
    statement = @db.prepare("SELECT id FROM part WHERE id = ?")
    raise StandardError.new("id not valid") if statement.execute(part["id"]).to_a.empty?
  else
    # add new part
    raise StandardError.new("name required") unless part["name"] and part["name"].length > 0
    statement = @db.prepare("SELECT id FROM part WHERE name = ?")
    raise StandardError.new("name already existing") unless statement.execute(part["name"]).to_a.empty?
    insert = @db.prepare("INSERT INTO part (name) VALUES (?)");
    insert.execute(part["name"])
    part["id"] = statement.execute(part["name"]).to_a[0]["id"]
  end
  # update family
  family = nil
  field = "family"
  part[field] = nil if part[field] and 0 == part[field].length
  if part[field] then
    statement = @db.prepare("SELECT id FROM part WHERE name = ?")
    family = statement.execute(part[field]).to_a
    raise StandardError.new("family not existing") if family.empty?
    update = @db.prepare("UPDATE part SET #{field} = ? WHERE id = ?")
    update.execute(family[0]["id"], part["id"])
    family = get_part_by_id(family[0]["id"])
  end
  # update fields
  fields_txt = ["name", "description", "details", "mpn", "package", "datasheet", "page"];
  fields_txt.each do |field|
    next unless part[field]
    part[field] = nil if part[field].kind_of?(String) and 0 == part[field].length
    next if family and family[field] == part[field]
    update = @db.prepare("UPDATE part SET #{field} = ? WHERE id = ?")
    update.execute(part[field], part["id"])
  end
  # update manufacturer and package
  field_ref = ["manufacturer"]
  field_ref.each do |field|
    part[field] = nil if part[field] and 0 == part[field].length
    next if family and family[field] == part[field]
    if part[field] then
      statement = @db.prepare("SELECT id FROM #{field} WHERE LOWER(name) = ?")
      ref = statement.execute(part[field].downcase).to_a[0]
      unless ref then
        insert = @db.prepare("INSERT INTO #{field} (name) VALUES (?)");
        insert.execute(part[field])
      end
      ref = statement.execute(part[field].downcase).to_a[0]
      update = @db.prepare("UPDATE part SET #{field} = ? WHERE id = ?")
      update.execute(ref["id"], part["id"])
    else
      update = @db.prepare("UPDATE part SET #{field} = NULL WHERE id = ?")
      update.execute(part["id"])
    end
  end
  # update inventory
  field = "location"
  part[field] = nil if part[field] and 0 == part[field].length
  part["location"] = nil if part["stock"] and 0 == part["stock"].length
  if part[field] then
    statement = @db.prepare("SELECT id FROM #{field} WHERE LOWER(name) = ?")
    ref = statement.execute(part[field].downcase).to_a[0]
    unless ref then
      insert = @db.prepare("INSERT INTO #{field} (name) VALUES (?)");
      insert.execute(part[field])
    end
    ref = statement.execute(part[field].downcase).to_a[0]
    statement = @db.prepare("SELECT id FROM inventory WHERE part = ? AND location = ?")
    ref_inv = statement.execute(part["id"], ref["id"]).to_a[0]
    unless ref_inv then
      insert = @db.prepare("INSERT INTO inventory (part, location, quantity) VALUES (?,?,?)");
      insert.execute(part["id"], ref["id"], part["stock"].to_i)
    end
    ref_inv = statement.execute(part["id"], ref["id"]).to_a[0]
    update = @db.prepare("UPDATE inventory SET quantity = ? WHERE id = ?")
    update.execute(part["stock"].to_i, ref_inv["id"])
  else
    delete = @db.prepare("DELETE FROM inventory WHERE part = ?")
    delete.execute(part["id"])
  end
  # update distributors
  field = "distributors"
  part[field] = nil if part[field] and 0 == part[field].length
  delete = @db.prepare("DELETE FROM distribution WHERE part = ?")
  delete.execute(part["id"])
  if part[field] then
    part[field].each do |distributor,sku|
      next unless sku and !sku.empty?
      statement = @db.prepare("SELECT id FROM distributor WHERE LOWER(name) = ?")
      ref = statement.execute(distributor.downcase).to_a[0]
      raise StandardError.new("distributor unknown") unless ref
      insert = @db.prepare("INSERT INTO distribution (distributor,part,sku) VALUES (?,?,?)");
      insert.execute(ref["id"], part["id"], sku)
    end
  end
  # update properties
  field = "properties"
  part[field] = nil if part[field] and 0 == part[field].length
  delete = @db.prepare("DELETE FROM property_value WHERE part = ?")
  delete.execute(part["id"])
  if part[field] then
    part[field].each do |name,values|
      next unless values and !values.empty?
      statement = @db.prepare("SELECT id FROM property WHERE LOWER(name) = ?")
      ref = statement.execute(name.downcase).to_a[0]
      unless ref then
        insert = @db.prepare("INSERT INTO property (name) VALUES (?)");
        insert.execute(name)
      end
      ref = statement.execute(name.downcase).to_a[0]
      insert = @db.prepare("INSERT INTO property_value (property,part,value) VALUES (?,?,?)");
      values.each do |value|
        next if family and family["properties"] and family["properties"][name].include?(value)
        insert.execute(ref["id"], part["id"], value)
      end
    end
  end
end

post '/part' do
  request.body.rewind
  begin
    part = JSON.parse(request.body.read)
  rescue
    halt 401, "not json"
  end
  puts part if DEBUG
  begin
    add_part(part)
  rescue StandardError => e
    halt 401, e.message
  end
  return 200
end

get '/import/lcsc/:lcsc' do
  halt 401 unless params['lcsc'] and params['lcsc'] =~ /^C\d+$/i
  uri = URI("https://wmsc.lcsc.com/wmsc/product/detail?productCode=#{params['lcsc']}")
  res = Net::HTTP.get_response(uri)
  halt 401, "could not get part" unless res.is_a?(Net::HTTPSuccess)
  json = JSON.parse(res.body)
  #puts json
  halt 401, "part not found" unless 200 == json["code"] and json["result"]
  result = json["result"]
  part = {}
  part["name"] = result["productModel"]
  part["mpn"] = result["productModel"]
  part["description"] = result["productDescEn"]
  part["details"] = result["productIntroEn"]
  part["manufacturer"] = result["brandNameEn"]
  part["package"] = result["encapStandard"]
  part["distributors"] = {"LCSC" => result["productCode"]}
  part["attachments"] = result["productImages"]
  part["datasheet"] = result["pdfUrl"]
  existing = get_part_by_name(part["name"])
  halt 401, "part name already exists" if existing
  puts part
  begin
    add_part(part)
  rescue StandardError => e
    halt 401, e.message
  end
  i = 0
  (part["attachments"] + [part["datasheet"]]).each do |attachement|
    file = attachement.split("/")[-1]
    dir = PUBLIC + "/" + ATTACHMENTS + "/" + part["name"]
    path = "#{dir}/#{i}_#{file}"
    i += 1
    unless File.file?(path) then
      uri = URI(attachement)
      res = Net::HTTP.get_response(uri)
      if (res.is_a?(Net::HTTPSuccess)) then
        Dir.mkdir(dir) unless File.directory?(dir)
        File.open(path, "wb") do |f|
          f.write res.body
        end
      end
    end
  end
  return 200
end