diff --git a/config/initializers/rails_admin.rb b/config/initializers/rails_admin.rb index 0a188da..f2d3163 100644 --- a/config/initializers/rails_admin.rb +++ b/config/initializers/rails_admin.rb @@ -1,4 +1,18 @@ require Rails.root.join('lib', 'rails_admin_approve_change.rb') +require Rails.root.join('lib', 'rails_admin_import_data.rb') + +module RailsAdmin + module Config + module Actions + class ImportCSV < RailsAdmin::Config::Actions::Base + RailsAdmin::Config::Actions.register(self) + end + class ImportImages < RailsAdmin::Config::Actions::Base + RailsAdmin::Config::Actions.register(self) + end + end + end +end RailsAdmin.config do |config| @@ -32,5 +46,9 @@ ## With an audit adapter, you can add: history_index history_show + + import_csv + import_images + end end diff --git a/config/locales/en.yml b/config/locales/en.yml index 0653957..da0ada2 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -21,3 +21,9 @@ en: hello: "Hello world" + admin: + actions: + import_csv: + menu: "Import Data" + import_images: + menu: "Import Images" diff --git a/lib/import_data.rb b/lib/import_data.rb new file mode 100644 index 0000000..bf8a25a --- /dev/null +++ b/lib/import_data.rb @@ -0,0 +1,211 @@ +def auth_google_drive_client + require 'google/api_client' + + key = OpenSSL::PKey::RSA.new(ENV['GOOGLE_DRIVE_KEY'], 'notasecret') + + client = Google::APIClient.new({:application_name => "cambase-io", :application_version => "1.0"}) + client.authorization = Signet::OAuth2::Client.new( + :person => ENV['GOOGLE_DRIVE_PERSON'], + :token_credential_uri => 'https://accounts.google.com/o/oauth2/token', + :audience => 'https://accounts.google.com/o/oauth2/token', + :scope => 'https://www.googleapis.com/auth/drive.readonly', + :issuer => ENV['GOOGLE_DRIVE_ISSUER'], + :signing_key => key + ) + client.authorization.fetch_access_token! + + client +end + +def download_csv_from_google_drive + + client = auth_google_drive_client + + spreadsheet_url = "https://docs.google.com/a/mhlabs.net/spreadsheets/d/1TVUA5rVoCvA-ZHo1dw_uqr-e3n1qOYy-j7Ia1xetWTo/export?exportFormat=csv&gid=0" + + file_content = client.execute(:uri => spreadsheet_url).body + File.open("#{Rails.root}/tmp/update.csv", 'wb') do |file| + file.write(file_content) + end +end + +def import_csv_from_google_drive + SmarterCSV.process("#{Rails.root}/tmp/update.csv").each do |camera| + camera = Hash[camera.map{ |k, v| [k, v.to_s] }] + camera[:manufacturer_id] = Manufacturer.where(:name => camera[:"name_[manufacturer]"]).first_or_create.id + camera.delete :id + camera.delete :created_at + camera.delete :updated_at + camera.delete :"id_[manufacturer]" + camera.delete :"info_[manufacturer]" + camera.delete :"created_at_[manufacturer]" + camera.delete :"updated_at_[manufacturer]" + camera.delete :"manufacturer_slug_[manufacturer]" + camera.delete :"url_[manufacturer]" + camera.delete :"id_[images]" + camera.delete :"position_[images]" + camera.delete :"created_at_[images]" + camera.delete :"updated_at_[images]" + camera.delete :"file_[images]" + camera.delete :"name_[manufacturer]" + + camera.update(camera){|key,value| clean_exported_csv_values(value)} + + c = Camera.where(:model => camera[:model]).first_or_initialize + c.update_attributes(camera) + puts "#{c.manufacturer.name} \n#{c.model} \n #{c.errors.messages.inspect} \n\n" unless c.errors.messages.blank? + end +end + +def import_documents_from_google_drive + client = auth_google_drive_client + + drive = client.discovered_api('drive', 'v2') + + result = client.execute( + api_method: drive.files.list, + parameters: { + maxResults: 1000, + q: "mimeType = 'application/pdf' and ('04010857713529984123' in owners or '02928049532232239685' in owners or '13940272261418201147' in owners)" + } + ) + + result.data.items.each do |item| + folder = print_parents(item.id).first.id + folder_name = print_file(folder).title + puts folder_name + camera = Camera.find_by_model(folder_name) + if camera and item.downloadUrl + file_content = client.execute(:uri => item.downloadUrl).body + File.open("#{Rails.root}/tmp/#{item.title}", 'wb') do |file| + file.write(file_content) + document = Document.create(:file => file) + camera.documents.append(document) + end + File.delete("#{Rails.root}/tmp/#{item.title}") + end + end +end + +def import_images_from_google_drive + client = auth_google_drive_client + + drive = client.discovered_api('drive', 'v2') + + result = client.execute( + api_method: drive.files.list, + parameters: { + maxResults: 1000, + q: "mimeType contains 'image' and ('04010857713529984123' in owners or '02928049532232239685' in owners or '13940272261418201147' in owners)" + } + ) + + result.data.items.each do |item| + folder = print_parents(item.id).first.id + if folder + folder_name = print_file(folder).title + puts folder_name + camera = Camera.find_by_model(folder_name) + if camera and item.downloadUrl + file_content = client.execute(:uri => item.downloadUrl).body + File.open("#{Rails.root}/tmp/#{item.originalFilename}", 'wb') do |file| + file.write(file_content) + image = Image.create(:file => file) + camera.images.append(image) + puts "#{camera.manufacturer.name} \n#{camera.model} \n #{camera.errors.messages.inspect} \n\n" unless camera.errors.messages.blank? + end + File.delete("#{Rails.root}/tmp/#{item.originalFilename}") + end + end + puts item.originalFilename + puts + sleep 5 + end + +end + +def print_parents(file_id) + client = auth_google_drive_client + drive = client.discovered_api('drive', 'v2') + result = client.execute( + :api_method => drive.parents.list, + :parameters => { 'fileId' => file_id } + ) + if result.status == 200 + parents = result.data + parents.items.each do |parent| + parent + end + else + puts "An error occurred: #{result.data['error']['message']}" + end +end + +def print_file(file_id) + client = auth_google_drive_client + drive = client.discovered_api('drive', 'v2') + result = client.execute( + :api_method => drive.files.get, + :parameters => { 'fileId' => file_id } + ) + if result.status == 200 + file = result.data + else + puts "An error occurred: #{result.data['error']['message']}" + end +end + +def search_for(query) + client = auth_google_drive_client + drive = client.discovered_api('drive', 'v2') + result = client.execute( + api_method: drive.files.list, + parameters: { + maxResults: 1000, + q: "#{query}" + } + ) +end + +def clean_csv_values(value) + case value + when /^YES/i + true + when /^NO|N0/i + false + when /Bi-directional/i + true + when /FishEye|FixedDome|Vandal-Resistant Dome|PTZ Domes/i + 'dome' + when /Dome|Box|Bullet/i + value.downcase + when '4CIF', '4cif' + '704×480' + when '1080p', '1920x1081' + '1920x1080' + when '2058x1536', '2050x1536', '2048X1536' + '2048x1536' + when '40x480' + '640x480' + when '?', 'Wireless' + nil + else + value + end +end + +def clean_exported_csv_values(value) + case value + when '?', '-', 'nil' + nil + else + value + end +end + +def call_rake(task, options = {}) + puts 'rake initiated!' + options[:rails_env] ||= Rails.env + args = options.map { |n, v| "#{n.to_s.upcase}='#{v}'" } + system "bundle exec rake #{task} #{args.join(' ')} --trace 2>&1 >> #{Rails.root}/log/rake.log &" +end diff --git a/lib/rails_admin_import_data.rb b/lib/rails_admin_import_data.rb new file mode 100644 index 0000000..e714a0d --- /dev/null +++ b/lib/rails_admin_import_data.rb @@ -0,0 +1,52 @@ +require 'rails_admin/config/actions' +require 'rails_admin/config/actions/base' +require Rails.root.join('lib', 'import_data.rb') + +module RailsAdminPublish +end + +module RailsAdmin + module Config + module Actions + class ImportCSV < RailsAdmin::Config::Actions::Base + register_instance_option :link_icon do + 'icon-list-alt' + end + + register_instance_option :collection? do + true + end + + register_instance_option :controller do + Proc.new do + download_csv_from_google_drive + + import_csv_from_google_drive + + redirect_to back_or_index + end + end + end + class ImportImages < RailsAdmin::Config::Actions::Base + register_instance_option :link_icon do + 'icon-picture' + end + + register_instance_option :collection? do + true + end + + register_instance_option :controller do + Proc.new do + + flash[:notice] = "Image import started" + + call_rake('import_images_from_google_drive') + + redirect_to back_or_index + end + end + end + end + end +end diff --git a/lib/tasks/import_data.rake b/lib/tasks/import_data.rake index 9bc6e9c..cefe048 100644 --- a/lib/tasks/import_data.rake +++ b/lib/tasks/import_data.rake @@ -1,3 +1,5 @@ +require Rails.root.join('lib', 'import_data.rb') + desc "Import csv" task :import_csv => :environment do @@ -40,33 +42,6 @@ task :import_csv => :environment do puts "\nCSV Import complete! \n\n" end -def clean_csv_values(value) - case value - when /^YES/i - true - when /^NO|N0/i - false - when /Bi-directional/i - true - when /FishEye|FixedDome|Vandal-Resistant Dome|PTZ Domes/i - 'dome' - when /Dome|Box|Bullet/i - value.downcase - when '4CIF', '4cif' - '704×480' - when '1080p', '1920x1081' - '1920x1080' - when '2058x1536', '2050x1536', '2048X1536' - '2048x1536' - when '40x480' - '640x480' - when '?', 'Wireless' - nil - else - value - end -end - desc "Import images" task :import_images => :environment do @@ -93,75 +68,26 @@ task :import_images => :environment do puts "\nImage Import complete! \n\n" end -desc "Import documents" +desc "Import documents from google drive" -task :import_documents => :environment do - - require 'google/api_client' +task :import_documents_from_google_drive => :environment do + import_documents_from_google_drive +end - key = OpenSSL::PKey::RSA.new(ENV['GOOGLE_DRIVE_KEY'], 'notasecret') +desc "Import images from google drive" - @client = Google::APIClient.new({:application_name => "cambase-io", :application_version => "1.0"}) - @client.authorization = Signet::OAuth2::Client.new( - :person => ENV['GOOGLE_DRIVE_PERSON'], - :token_credential_uri => 'https://accounts.google.com/o/oauth2/token', - :audience => 'https://accounts.google.com/o/oauth2/token', - :scope => 'https://www.googleapis.com/auth/drive.readonly', - :issuer => ENV['GOOGLE_DRIVE_ISSUER'], - :signing_key => key - ) - @client.authorization.fetch_access_token! - @drive = @client.discovered_api('drive', 'v2') +task :import_images_from_google_drive => :environment do + import_images_from_google_drive +end - result = @client.execute( - api_method: @drive.files.list, - parameters: { - maxResults: 1000, - q: "mimeType = 'application/pdf' and ('04010857713529984123' in owners or '02928049532232239685' in owners) " - } - ) +desc "Downloads csv from google drive" - result.data.items.each do |item| - folder = print_parents(item.id).first.id - folder_name = print_file(folder).title - puts folder_name - camera = Camera.find_by_model(folder_name) - if camera and item.downloadUrl - file_content = @client.execute(:uri => item.downloadUrl).body - File.open("#{Rails.root}/tmp/#{item.title}", 'wb') do |file| - file.write(file_content) - document = Document.create(:file => file) - camera.documents.append(document) - end - File.delete("#{Rails.root}/tmp/#{item.title}") - end - end +task :download_csv_from_google_drive => :environment do + download_csv_from_google_drive end -def print_parents(file_id) - result = @client.execute( - :api_method => @drive.parents.list, - :parameters => { 'fileId' => file_id } - ) - if result.status == 200 - parents = result.data - parents.items.each do |parent| - parent - end - else - puts "An error occurred: #{result.data['error']['message']}" - end -end +desc "Import csv from google drive" -def print_file(file_id) - result = @client.execute( - :api_method => @drive.files.get, - :parameters => { 'fileId' => file_id } - ) - if result.status == 200 - file = result.data - else - puts "An error occurred: #{result.data['error']['message']}" - end +task :import_csv_from_google_drive => :environment do + import_csv_from_google_drive end -