From 70d6977ccc07b667da0ed165b7706afbb5190816 Mon Sep 17 00:00:00 2001
From: Samantha McVey <samantham@posteo.net>
Date: Wed, 9 Nov 2016 20:57:13 -0800
Subject: [PATCH] Use charlock_holmes to do encoding detection. In my tests it
has properly identified incorrect encodings that used to be present on older
commits. This will help ensure this won't happen again, giving people instant
feedback and allowing all pull requests to be checked
---
Gemfile | 1 +
Rakefile | 6 +++++-
tests/encoding.rb | 16 ++++++++++------
3 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/Gemfile b/Gemfile
index eb73fd89..018aeb17 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,3 +1,4 @@
group :test do
gem 'rake'
+ gem 'charlock_holmes'
end
diff --git a/Rakefile b/Rakefile
index b41beab5..0193d816 100644
--- a/Rakefile
+++ b/Rakefile
@@ -2,6 +2,10 @@ task default: %w[test]
task :test do
Dir["./tests/*.rb"].each do |test_file|
- ruby test_file
+ begin
+ ruby test_file
+ rescue
+ puts "FAILED #{test_file}!"
+ end
end
end
diff --git a/tests/encoding.rb b/tests/encoding.rb
index c4d41d19..ae7e495f 100644
--- a/tests/encoding.rb
+++ b/tests/encoding.rb
@@ -1,14 +1,18 @@
#!/usr/bin/env ruby
+require 'charlock_holmes'
$file_count = 0;
markdown_files = Dir["./**/*.html.markdown"]
markdown_files.each do |file|
begin
- file_bin = File.open(file, "rb")
- contents = file_bin.read
- if ! contents.valid_encoding?
- puts "#{file} has an invalid encoding! Please save the file in UTF-8!"
- else
+ contents = File.read(file)
+ detection = CharlockHolmes::EncodingDetector.detect(contents)
+ case detection[:encoding]
+ when 'UTF-8'
$file_count = $file_count + 1
+ when 'ISO-8859-1'
+ $file_count = $file_count + 1
+ else
+ puts "#{file} was detected as #{detection[:encoding]} encoding! Please save the file in UTF-8!"
end
rescue Exception => msg
puts msg
@@ -20,6 +24,6 @@ if files_failed != 0
puts "Please resave the file as UTF-8."
exit 1
else
- puts "Success. All #{$file_count} files passed UTF-8 validity checks"
+ puts "Success. All #{$file_count} files Ruby's UTF-8 validity checks. This won't catch most problems."
exit 0
end