chars.py

# Corresponds to at least 377 messages in 3278732
CHARS = {
    '\n': 0,
    ' ': 1,
    'e': 2,
    't': 3,
    'o': 4,
    'i': 5,
    'a': 6,
    's': 7,
    'n': 8,
    'r': 9,
    'l': 10,
    'h': 11,
    'u': 12,
    'd': 13,
    'c': 14,
    'm': 15,
    '.': 16,
    'p': 17,
    'y': 18,
    'f': 19,
    'g': 20,
    'w': 21,
    'b': 22,
    'v': 23,
    'k': 24,
    ',': 25,
    'I': 26,
    ':': 27,
    '\x02': 28,
    '\x03': 29,
    ')': 30,
    '(': 31,
    'T': 32,
    'x': 33,
    'S': 34,
    "'": 35,
    'A': 36,
    '/': 37,
    '-': 38,
    '=': 39,
    'C': 40,
    '1': 41,
    'j': 42,
    '"': 43,
    'E': 44,
    ';': 45,
    'D': 46,
    'P': 47,
    'M': 48,
    '0': 49,
    '2': 50,
    'O': 51,
    'L': 52,
    'F': 53,
    'N': 54,
    'H': 55,
    'R': 56,
    'B': 57,
    '?': 58,
    'q': 59,
    'W': 60,
    '{': 61,
    '}': 62,
    '_': 63,
    'U': 64,
    '3': 65,
    '>': 66,
    'Y': 67,
    '<': 68,
    '4': 69,
    '5': 70,
    '[': 71,
    ']': 72,
    'G': 73,
    'z': 74,
    'V': 75,
    '8': 76,
    '6': 77,
    '+': 78,
    '7': 79,
    '#': 80,
    '9': 81,
    '$': 82,
    'J': 83,
    '!': 84,
    '*': 85,
    'Q': 86,
    'X': 87,
    '@': 88,
    'K': 89,
    '%': 90,
    '\\': 91,
    '&': 92,
    '|': 93,
    'Z': 94,
    '^': 95,
    '~': 96,
    '`': 97,
    '’': 98,
    '”': 99,
    '“': 100,
    '…': 101,
    '—': 102,
    '–': 103,
    '\t': 104,
    '‘': 105,
    '\u200b': 106,
    '´': 107,
    'é': 108,
    'á': 109,
    'ü': 110,
    'ö': 111,
    'ä': 112,
    'ó': 113,
    '§': 114,
    'ç': 115,
    'à': 116,
    'í': 117,
    '£': 118,
    '©': 119,
    '»': 120,
    '�': 121,
    '€': 122,
    '×': 123,
    '→': 124,
    'а': 125,
    '°': 126,
    'è': 127,
    'и': 128,
    'о': 129,
    'е': 130,
    'ã': 131,
    'р': 132,
    'å': 133,
    'т': 134,
    '﹕': 135,
    '•': 136,
    'с': 137,
    'н': 138,
    '─': 139,
    '«': 140,
    'к': 141,
    'ñ': 142,
    'â': 143,
    'в': 144,
    '└': 145,
    'ú': 146,
    'л': 147,
    'Ã': 148,
    'ê': 149,
    '├': 150,
    'ا': 151,
    'д': 152,
    '®': 153,
    'у': 154,
    'ı': 155,
    'ø': 156,
    '™': 157,
    '¿': 158,
    '\u200c': 159,
    'м': 160,
    '│': 161,
    'ï': 162,
    'ß': 163,
    '，': 164,
    '✓': 165,
    'п': 166,
    'я': 167,
    'æ': 168,
    '²': 169,
    'ر': 170,
    'µ': 171,
    'ل': 172,
    'م': 173,
    'Ö': 174,
    'Ä': 175,
    '：': 176,
    'ë': 177,
    '·': 178,
    'Ü': 179,
    'б': 180,
    'Â': 181,
    'š': 182,
    'з': 183,
    '−': 184,
    'É': 185,
    'й': 186,
    'ы': 187,
    'ب': 188,
    'ł': 189,
    'ь': 190,
    'ô': 191,
    'г': 192,
    '±': 193,
    'ت': 194,
    'ن': 195,
    'Å': 196,
    'λ': 197,
    'و': 198,
    # out-of-vocabulary is 199
}

WEIGHTS = {
    '\n': 1.000000,
    ' ': 1.000293,
    'e': 1.000738,
    't': 1.001031,
    'o': 1.002215,
    'i': 1.002581,
    'a': 1.002823,
    's': 1.003117,
    'n': 1.003513,
    'r': 1.003747,
    'l': 1.009632,
    'h': 1.012405,
    'u': 1.014209,
    'd': 1.014485,
    'c': 1.016998,
    'm': 1.032959,
    '.': 1.036550,
    'p': 1.038507,
    'y': 1.048746,
    'f': 1.050153,
    'g': 1.058613,
    'w': 1.074814,
    'b': 1.088029,
    'v': 1.135608,
    'k': 1.239276,
    ',': 1.252771,
    'I': 1.253381,
    ':': 1.377302,
    '\x02': 1.389897,
    '\x03': 1.389898,
    ')': 1.407768,
    '(': 1.417021,
    'T': 1.441953,
    'x': 1.511402,
    'S': 1.611368,
    "'": 1.663351,
    'A': 1.748086,
    '/': 1.748631,
    '-': 1.911289,
    '=': 1.919187,
    'C': 1.929076,
    '1': 2.127655,
    'j': 2.174542,
    '"': 2.274096,
    'E': 2.286056,
    ';': 2.321038,
    'D': 2.334237,
    'P': 2.335984,
    'M': 2.344737,
    '0': 2.372879,
    '2': 2.428316,
    'O': 2.440422,
    'L': 2.492283,
    'F': 2.538985,
    'N': 2.568904,
    'H': 2.598685,
    'R': 2.605083,
    'B': 2.664218,
    '?': 2.670296,
    'q': 2.710728,
    'W': 2.800406,
    '{': 2.889346,
    '}': 2.919107,
    '_': 3.019114,
    'U': 3.224621,
    '3': 3.328087,
    '>': 3.390669,
    'Y': 3.530247,
    '<': 3.955393,
    '4': 4.042060,
    '5': 4.045327,
    '[': 4.110864,
    ']': 4.118868,
    'G': 4.404828,
    'z': 4.461779,
    'V': 4.595594,
    '8': 5.107662,
    '6': 5.110767,
    '+': 5.682070,
    '7': 5.839341,
    '#': 5.867168,
    '9': 5.965975,
    '$': 6.145787,
    'J': 6.249781,
    '!': 6.413653,
    '*': 7.057462,
    'Q': 8.739711,
    'X': 9.362589,
    '@': 9.634124,
    'K': 9.676918,
    '%': 11.799051,
    '\\': 13.316458,
    '&': 13.676060,
    '|': 19.746951,
    'Z': 30.696711,
    '^': 40.288089,
    '~': 54.900954,
    '`': 58.157461,
    '’': 150.233000,
    '”': 312.136805,
    '“': 323.246081,
    '…': 426.301521,
    '—': 431.746774,
    '–': 460.942640,
    '\t': 467.515329,
    '‘': 585.374933,
    '\u200b': 678.395407,
    '´': 701.773330,
    'é': 766.226922,
    'á': 1699.681182,
    'ü': 1781.894022,
    'ö': 1819.470033,
    'ä': 1836.798319,
    'ó': 1875.677918,
    '§': 1905.104590,
    'ç': 2258.047521,
    'à': 2265.850035,
    'í': 2341.917857,
    '£': 2489.510251,
    '©': 2600.067407,
    '»': 2622.948000,
    '�': 2648.372375,
    '€': 2687.446721,
    '×': 2766.822785,
    '→': 2792.747019,
    'а': 2855.997387,
    '°': 3007.967890,
    'è': 3113.660969,
    'и': 3230.231527,
    'о': 3239.807312,
    'е': 3239.807312,
    'ã': 3380.087629,
    'р': 3440.383001,
    'å': 3476.866384,
    'т': 3480.557325,
    '﹕': 3540.696544,
    '•': 3618.857616,
    'с': 3642.983333,
    'н': 3839.209602,
    '─': 3945.469314,
    '«': 3978.986650,
    'к': 4017.996324,
    'ñ': 4017.996324,
    'â': 4067.847395,
    'в': 4171.354962,
    '└': 4274.687093,
    'ú': 4509.883081,
    'л': 4522.324138,
    'Ã': 4624.379408,
    'ê': 4731.147186,
    '├': 4821.595588,
    'ا': 5028.657975,
    'д': 5091.125776,
    '®': 5099.043546,
    'у': 5322.540584,
    'ı': 5322.540584,
    'ø': 5383.719212,
    '™': 5437.288557,
    '¿': 5455.382696,
    '\u200c': 5557.093220,
    'м': 5557.093220,
    '│': 5731.966783,
    'ï': 5939.646739,
    'ß': 5961.245455,
    '，': 6015.935780,
    '✓': 6082.903525,
    'п': 6082.903525,
    'я': 6105.558659,
    'æ': 6174.548023,
    '²': 6197.892250,
    'ر': 6268.996176,
    'µ': 6268.996176,
    'ل': 6391.198830,
    'م': 6466.834320,
    'Ö': 6492.445545,
    'Ä': 6623.606061,
    '：': 6760.175258,
    'ë': 6802.251037,
    '·': 6931.680761,
    'Ü': 6946.366525,
    'б': 7174.365427,
    'Â': 7253.727876,
    'š': 7367.831461,
    'з': 7485.582192,
    '−': 7554.573733,
    'É': 7572.020785,
    'й': 7696.443662,
    'ы': 7751.028369,
    'ب': 7769.395735,
    'ł': 7769.395735,
    'ь': 7806.392857,
    'ô': 7977.335766,
    'г': 7996.792683,
    '±': 8135.694789,
    'ت': 8196.712500,
    'ن': 8258.652393,
    'Å': 8385.383632,
    'λ': 8428.496144,
    'و': 8605.472441,
}

# Calculated as number of distinct documents with OOV chars / # documents with \n (most frequent)
OOV_WEIGHT = 1.388574898980894