spacepaste

Paste details

reply | raw

  1.  
  2. # -*- coding: UTF-8 -*-
  3. def recode_utf8(data):
  4. """
  5. Given a string which is either:
  6. * unicode
  7. * well-encoded utf8
  8. * well-encoded latin1
  9. * poorly-encoded utf8+latin1
  10. Return the equivalent utf8-encoded byte string.
  11. """
  12. if isinstance(data, unicode):
  13. # The input is already decoded. Just return the utf8.
  14. return data.encode('UTF-8')
  16. try:
  17. decoded = data.decode('UTF-8')
  18. except UnicodeDecodeError:
  19. # Indicates latin1 encoded bytes.
  20. decoded = data.decode('latin1')
  22. while True:
  23. # Check if the data is poorly-encoded as utf8+latin1
  24. try:
  25. encoded = decoded.encode('latin1')
  26. except UnicodeEncodeError:
  27. # Indicates non-latin1-encodable characters; it's not utf8+latin1.
  28. return decoded.encode('UTF-8')
  30. try:
  31. decoded = encoded.decode('UTF-8')
  32. except UnicodeDecodeError:
  33. # Can't decode the latin1 as utf8; it's not utf8+latin1.
  34. return decoded.encode('UTF-8')
  37. import unittest as T
  38. class TestRecodeUtf8(T.TestCase):
  39. latin1 = u'München' # encodable to latin1
  40. utf8 = u'Łódź' # not encodable to latin1
  42. def test_unicode(self):
  43. "An un-encoded unicode string should just become utf8-encoded"
  44. self.assertEqual(
  45. recode_utf8(self.utf8),
  46. self.utf8.encode('UTF-8'),
  47. )
  49. def test_utf8(self):
  50. "A utf8-encoded string should be unchanged"
  51. utf8 = self.utf8.encode('UTF-8')
  52. self.assertEqual(
  53. recode_utf8(utf8),
  54. utf8,
  55. )
  57. def test_latin1(self):
  58. "A latin1-encoded string should become utf8-encoded"
  59. self.assertEqual(
  60. recode_utf8(self.latin1.encode('latin1')),
  61. self.latin1.encode('UTF-8'),
  62. )
  64. def test_utf8_plus_latin1(self):
  65. "A poorly-encoded utf8+latin1 string should become utf8-encoded"
  66. utf8 = self.utf8.encode('UTF-8')
  67. poorly_encoded = utf8.decode('latin1').encode('UTF-8')
  68. self.assertEqual(
  69. recode_utf8(poorly_encoded),
  70. utf8,
  71. )
  73. def test_utf8_plus_latin1_several_times(self):
  74. "A string mangled by utf8+latin1 several times should become utf8-encoded"
  75. utf8 = self.utf8.encode('UTF-8')
  76. poorly_encoded = utf8
  77. for x in range(10):
  78. poorly_encoded = poorly_encoded.decode('latin1').encode('UTF-8')
  80. self.assertEqual(
  81. recode_utf8(poorly_encoded),
  82. utf8,
  83. )
  87. if __name__ == '__main__':
  88. T.main()
  89.  
Powered by spacepaste. Paste removal.