Merge pull request #125 from zackw/add-c0-and-c1-controls

Add control characters and whitespace

blns.json

@@ -91,6 +91,12 @@
   ",./;'[]\\-=", 
   "<>?:\"{}|_+", 
   "!@#$%^&*()`~", 
+  "\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f", 
+  "€‚ƒ„†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ", 
+  "\t\u000b\f …             ​

  　", 
+  "­؀؁؂؃؄؅؜۝܏᠎​‌‍‎‏‪‫‬‭‮⁠⁡⁢⁣⁤⁦⁧⁨⁩𑂽𛲠𛲡𛲢𛲣𝅳𝅴𝅵𝅶𝅷𝅸𝅹𝅺󠀁󠀠󠀡󠀢󠀣󠀤󠀥󠀦󠀧󠀨󠀩󠀪󠀫󠀬󠀭󠀮󠀯󠀰󠀱󠀲󠀳󠀴󠀵󠀶󠀷󠀸󠀹󠀺󠀻󠀼󠀽󠀾󠀿󠁀󠁁󠁂󠁃󠁄󠁅󠁆󠁇󠁈󠁉󠁊󠁋󠁌󠁍󠁎󠁏󠁐󠁑󠁒󠁓󠁔󠁕󠁖󠁗󠁘󠁙󠁚󠁛󠁜󠁝󠁞󠁟󠁠󠁡󠁢󠁣󠁤󠁥󠁦󠁧󠁨󠁩󠁪󠁫󠁬󠁭󠁮󠁯󠁰󠁱󠁲󠁳󠁴󠁵󠁶󠁷󠁸󠁹󠁺󠁻󠁼󠁽󠁾󠁿", 
+  "", 
+  "￾", 
   "Ω≈ç√∫˜µ≤≥÷", 
   "åß∂ƒ©˙∆˚¬…æ", 
   "œ∑´®†¥¨ˆøπ“‘", 
@@ -135,19 +141,19 @@
   "・(￣∀￣)・:*:", 
   "ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ", 
   ",。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’", 
-  "(╯°□°）╯︵ ┻━┻)  ", 
+  "(╯°□°）╯︵ ┻━┻)", 
   "(ﾉಥ益ಥ）ﾉ ┻━┻", 
   "┬─┬ノ( º _ ºノ)", 
   "( ͡° ͜ʖ ͡°)", 
   "😍", 
   "👩🏽", 
-  "👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 ", 
+  "👾 🙇 💁 🙅 🙆 🙋 🙎 🙍", 
   "🐵 🙈 🙉 🙊", 
   "❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙", 
   "✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿", 
   "🚾 🆒 🆓 🆕 🆖 🆗 🆙 🏧", 
   "0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟", 
-  "🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸                                                                                          ", 
+  "🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸", 
   "🇺🇸🇷🇺🇸🇦🇫🇦🇲", 
   "🇺🇸🇷🇺🇸🇦", 
   "１２３", 
@@ -157,7 +163,7 @@
   "הָיְתָהtestالصفحات التّحول", 
   "﷽", 
   "ﷺ", 
-  "مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ، ", 
+  "مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ، ", 
   "​", 
   " ", 
   "᠎", 
@@ -189,15 +195,15 @@
   "<script>alert(123)</script>", 
   "&lt;script&gt;alert(&#39;123&#39;);&lt;/script&gt;", 
   "<img src=x onerror=alert(123) />", 
-  "<svg><script>123<1>alert(123)</script> ", 
+  "<svg><script>123<1>alert(123)</script>", 
   "\"><script>alert(123)</script>", 
   "'><script>alert(123)</script>", 
   "><script>alert(123)</script>", 
   "</script><script>alert(123)</script>", 
   "< / script >< script >alert(123)< / script >", 
-  " onfocus=JaVaSCript:alert(123) autofocus ", 
-  "\" onfocus=JaVaSCript:alert(123) autofocus ", 
-  "' onfocus=JaVaSCript:alert(123) autofocus ", 
+  " onfocus=JaVaSCript:alert(123) autofocus", 
+  "\" onfocus=JaVaSCript:alert(123) autofocus", 
+  "' onfocus=JaVaSCript:alert(123) autofocus", 
   "＜script＞alert(123)＜/script＞", 
   "<sc<script>ript>alert(123)</sc</script>ript>", 
   "--><script>alert(123)</script>", 
@@ -218,7 +224,7 @@
   "<script\\x0Ctype=\"text/javascript\">javascript:alert(1);</script>", 
   "<script\\x2Ftype=\"text/javascript\">javascript:alert(1);</script>", 
   "<script\\x0Atype=\"text/javascript\">javascript:alert(1);</script>", 
-  "'`\"><\\x3Cscript>javascript:alert(1)</script>        ", 
+  "'`\"><\\x3Cscript>javascript:alert(1)</script>", 
   "'`\"><\\x00script>javascript:alert(1)</script>", 
   "ABC<div style=\"x\\x3Aexpression(javascript:alert(1)\">DEF", 
   "ABC<div style=\"x:expression\\x5C(javascript:alert(1)\">DEF", 

blns.txt

@@ -98,12 +98,46 @@ INF
 
 #	Special Characters
 #
-#	Strings which contain common special ASCII characters (may need to be escaped)
+# ASCII punctuation.  All of these characters may need to be escaped in some
+# contexts.  Divided into three groups based on (US-layout) keyboard position.
 
 ,./;'[]\-=
 <>?:"{}|_+
 !@#$%^&*()`~
 
+# Non-whitespace C0 controls: U+0001 through U+0008, U+000E through U+001F,
+# and U+007F (DEL)
+# Often forbidden to appear in various text-based file formats (e.g. XML),
+# or reused for internal delimiters on the theory that they should never
+# appear in input.
+# The next line may appear to be blank or mojibake in some viewers.
+

+
+# Non-whitespace C1 controls: U+0080 through U+0084 and U+0086 through U+009F.
+# Commonly misinterpreted as additional graphic characters.
+# The next line may appear to be blank, mojibake, or dingbats in some viewers.
+€‚ƒ„†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ
+
+# Whitespace: all of the characters with category Zs, Zl, or Zp (in Unicode
+# version 8.0.0), plus U+0009 (HT), U+000B (VT), U+000C (FF), U+0085 (NEL),
+# and U+200B (ZERO WIDTH SPACE), which are in the C categories but are often
+# treated as whitespace in some contexts.
+# This file unfortunately cannot express strings containing
+# U+0000, U+000A, or U+000D (NUL, LF, CR).
+# The next line may appear to be blank or mojibake in some viewers.
+# The next line may be flagged for "trailing whitespace" in some viewers.
+	 …             ​

  　
+
+# Unicode additional control characters: all of the characters with
+# general category Cf (in Unicode 8.0.0).
+# The next line may appear to be blank or mojibake in some viewers.
+­؀؁؂؃؄؅؜۝܏᠎​‌‍‎‏‪‫‬‭‮⁠⁡⁢⁣⁤⁦⁧⁨⁩𑂽𛲠𛲡𛲢𛲣𝅳𝅴𝅵𝅶𝅷𝅸𝅹𝅺󠀁󠀠󠀡󠀢󠀣󠀤󠀥󠀦󠀧󠀨󠀩󠀪󠀫󠀬󠀭󠀮󠀯󠀰󠀱󠀲󠀳󠀴󠀵󠀶󠀷󠀸󠀹󠀺󠀻󠀼󠀽󠀾󠀿󠁀󠁁󠁂󠁃󠁄󠁅󠁆󠁇󠁈󠁉󠁊󠁋󠁌󠁍󠁎󠁏󠁐󠁑󠁒󠁓󠁔󠁕󠁖󠁗󠁘󠁙󠁚󠁛󠁜󠁝󠁞󠁟󠁠󠁡󠁢󠁣󠁤󠁥󠁦󠁧󠁨󠁩󠁪󠁫󠁬󠁭󠁮󠁯󠁰󠁱󠁲󠁳󠁴󠁵󠁶󠁷󠁸󠁹󠁺󠁻󠁼󠁽󠁾󠁿
+
+# "Byte order marks", U+FEFF and U+FFFE, each on its own line.
+# The next two lines may appear to be blank or mojibake in some viewers.
+
+￾
+
 #	Unicode Symbols
 #
 #	Strings which contain common unicode symbols (e.g. smart quotes)
@@ -171,14 +205,14 @@ INF
 #
 #	Strings which consists of Japanese-style emoticons which are popular on the web
 
-ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ 
+ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ
 (｡◕ ∀ ◕｡)
 ｀ｨ(´∀｀∩
 __ﾛ(,_,*)
 ・(￣∀￣)・:*:
 ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ
 ,。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’
-(╯°□°）╯︵ ┻━┻)  
+(╯°□°）╯︵ ┻━┻)
 (ﾉಥ益ಥ）ﾉ ┻━┻
 ┬─┬ノ( º _ ºノ)
 ( ͡° ͜ʖ ͡°)
@@ -189,7 +223,7 @@ __ﾛ(,_,*)
 
 😍
 👩🏽
-👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 
+👾 🙇 💁 🙅 🙆 🙋 🙎 🙍
 🐵 🙈 🙉 🙊
 ❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙
 ✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿
@@ -201,7 +235,7 @@ __ﾛ(,_,*)
 #       Regional Indicator Symbols can be displayed differently across
 #       fonts, and have a number of special behaviors
 
-🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸                                                                                          
+🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸
 🇺🇸🇷🇺🇸🇦🇫🇦🇲
 🇺🇸🇷🇺🇸🇦
 
@@ -223,19 +257,6 @@ __ﾛ(,_,*)
 ﷺ
 مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ، 
 
-#	Unicode Spaces
-#
-#	Strings which contain unicode space characters with special properties (c.f. https://www.cs.tut.fi/~jkorpela/chars/spaces.html)
-
-​
- 
-᠎
-　
-
-␣
-␢
-␡
-
 #	Trick Unicode
 #
 #	Strings which contain unicode with unusual properties (e.g. Right-to-left override) (c.f. http://www.unicode.org/charts/PDF/U2000.pdf)
@@ -283,15 +304,15 @@ Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮
 <script>alert(123)</script>
 &lt;script&gt;alert(&#39;123&#39;);&lt;/script&gt;
 <img src=x onerror=alert(123) />
-<svg><script>123<1>alert(123)</script> 
+<svg><script>123<1>alert(123)</script>
 "><script>alert(123)</script>
 '><script>alert(123)</script>
 ><script>alert(123)</script>
 </script><script>alert(123)</script>
 < / script >< script >alert(123)< / script >
- onfocus=JaVaSCript:alert(123) autofocus 
-" onfocus=JaVaSCript:alert(123) autofocus 
-' onfocus=JaVaSCript:alert(123) autofocus 
+ onfocus=JaVaSCript:alert(123) autofocus
+" onfocus=JaVaSCript:alert(123) autofocus
+' onfocus=JaVaSCript:alert(123) autofocus
 ＜script＞alert(123)＜/script＞
 <sc<script>ript>alert(123)</sc</script>ript>
 --><script>alert(123)</script>
@@ -312,7 +333,7 @@ src=JaVaSCript:prompt(132)
 <script\x0Ctype="text/javascript">javascript:alert(1);</script>
 <script\x2Ftype="text/javascript">javascript:alert(1);</script>
 <script\x0Atype="text/javascript">javascript:alert(1);</script>
-'`"><\x3Cscript>javascript:alert(1)</script>        
+'`"><\x3Cscript>javascript:alert(1)</script>
 '`"><\x00script>javascript:alert(1)</script>
 ABC<div style="x\x3Aexpression(javascript:alert(1)">DEF
 ABC<div style="x:expression\x5C(javascript:alert(1)">DEF