Rakefile: update scarper lib
This commit is contained in:
parent
70773ea77c
commit
06c9e9f8f4
|
@ -350,7 +350,7 @@ def scrape_lcsc(sku)
|
||||||
# the search page does not always list existing parts, instead it will try
|
# the search page does not always list existing parts, instead it will try
|
||||||
url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
|
url = "https://lcsc.com/pre_search/link?type=lcsc&&value=#{sku}"
|
||||||
puts "scraping #{url}" if $scrape_debug
|
puts "scraping #{url}" if $scrape_debug
|
||||||
doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
|
doc = Nokogiri::HTML(URI.open(url),:allow_redirections => :all)
|
||||||
|
|
||||||
# verify if we have got a product page
|
# verify if we have got a product page
|
||||||
if doc.xpath('//div[@id="product_details"]').empty? then
|
if doc.xpath('//div[@id="product_details"]').empty? then
|
||||||
|
@ -381,7 +381,7 @@ def scrape_aliexpress(sku)
|
||||||
# get page
|
# get page
|
||||||
url = "https://www.aliexpress.com/item/#{sku}.html"
|
url = "https://www.aliexpress.com/item/#{sku}.html"
|
||||||
puts "scraping #{url}" if $scrape_debug
|
puts "scraping #{url}" if $scrape_debug
|
||||||
doc = Nokogiri::HTML(open(URI.escape(url),:allow_redirections => :all))
|
doc = Nokogiri::HTML(URI.open(url),:allow_redirections => :all)
|
||||||
# all the values can be found in javascript variables (stock is even only there)
|
# all the values can be found in javascript variables (stock is even only there)
|
||||||
js_docs = doc.xpath('//script')
|
js_docs = doc.xpath('//script')
|
||||||
if js_docs.empty? then
|
if js_docs.empty? then
|
||||||
|
|
Loading…
Reference in New Issue