Deal with invalid UTF-8 encodings

CoralineAda · Sep 18, 2017 · c7dcea9 · c7dcea9
1 parent 87ea7cb
commit c7dcea9
Show file tree

Hide file tree

Showing 36 changed files with 402 additions and 1 deletion.
diff --git a/alice/parser/google.rb b/alice/parser/google.rb
@@ -31,6 +31,7 @@ def all_answers
 
     def results
       answers = full_search + reductivist_search
+      answers.map!{|a| a.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')}
       answers.reject!{|a| a.include?("...")}
       sorted_answers = answers.sort{|a,b| declarative_index(a) <=> declarative_index(b)}
       best_answer = sorted_answers.any? && sorted_answers.first.split.join(' ') || ""

diff --git a/alice/parser/wikipedia.rb b/alice/parser/wikipedia.rb
@@ -21,7 +21,7 @@ def answer
 
     def results
       if result = ::Wikipedia.find(topic)
-        content = Grammar::LanguageHelper.sentences_from(result.sanitized_content)
+        content = Grammar::LanguageHelper.sentences_from(result.sanitized_content.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: ''))
       else
         content = []
       end

diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/1/1/segments_h b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/1/1/segments_h
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/_gk.cfe b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/_gk.cfe
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/_gk.cfs b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/_gk.cfs
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/_gk.si b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/_gk.si
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/segments_h b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/11/1/segments_h
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/16/1/segments_2 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/16/1/segments_2
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/16/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/16/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/16/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/16/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/21/1/segments_3 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/21/1/segments_3
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/21/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/21/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/21/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/21/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/23/1/segments_2 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/23/1/segments_2
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/23/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/23/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/23/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/23/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/28/1/segments_3 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/28/1/segments_3
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/28/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/28/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/28/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/28/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/30/1/segments_2 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/30/1/segments_2
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/30/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/30/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/30/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/30/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/35/1/segments_3 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/35/1/segments_3
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/35/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/35/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/35/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/35/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/37/1/segments_2 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/37/1/segments_2
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/37/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/37/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/37/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/37/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/42/1/segments_3 b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/42/1/segments_3
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/42/1/write.lock b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/42/1/write.lock
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/42/failure-message b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/42/failure-message
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/_lu.cfe b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/_lu.cfe
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/_lu.cfs b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/_lu.cfs
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/_lu.si b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/_lu.si
diff --git a/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/segments_h b/db/neo4j/development/data/databases/graph.db/schema/index/lucene/6/1/segments_h
diff --git a/db/neo4j/development/logs/debug.log b/db/neo4j/development/logs/debug.log