diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..66377bf --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +SRC_PATH = ./src +DIRS = $(shell find $(SRC_PATH) -maxdepth 3 -type d) +SRCS_CXX += $(foreach dir, $(DIRS), $(wildcard $(dir)/*.cpp)) +VPATH = $(DIRS) +OBJECTS=$(addprefix obj/,$(notdir $(SRCS_CXX:.cpp=.o))) +# OBJECTS=$(SRCS_C:.c=.o) +DEPENDS=$(OBJECTS:.o=.d) +CXXFLAGS+=-O3 -std=c++11 +# CXXFLAGS+=-Wall -fdiagnostics-color=auto -std=c++11 -fext-numeric-literals -O2 -pthread -fopenmp -Wl,-rpath-link=/home/ck/work/am_msk/libs/lib +# CXXFLAGS+=-Wall -fdiagnostics-color=auto -std=c++11 -fext-numeric-literals -O2 -pthread -fopenmp -Wl,-rpath-link=/home/ck/work/am_msk/libs/lib + +ARM_CXX=/home/ck/work/am_msk/cross-pi-gcc-8.3.0-2/bin/arm-linux-gnueabihf-g++ +PC_CXX=g++ + +CXX=$(PC_CXX) + +LDLIBS=-lm -lfftw3f -lopenblas + +# LDPATH+=-L/home/ck/work/am_msk/libs/lib + +# INCLUDEPATH+=-I /home/ck/work/am_msk/libs/include + + +# test: +# $(OBJECTS) + + +fastasr : $(OBJECTS) + $(CXX) -o fastasr $(CXXFLAGS) $(OBJECTS) $(LDLIBS) $(LDPATH) +obj/%.o:%.cpp + $(CXX) -c -o $@ $< $(CXXFLAGS) $(INCLUDEPATH) $(PREDEFINE) + +obj/%.d:%.cpp + @set -e; rm -f $@; $(CC) -MM $< $(INCLUDEFLAGS) > $@.$$$$; \ + sed 's,\($*\)\.o[ :]*,obj/\1.o $@ : ,g' < $@.$$$$ > $@; \ + rm -f $@.$$$$ + +include $(DEPENDS) + + + +.PHONY : clean install + +install : fastasr + expect ./download.ext + +clean: + @rm -f obj/* + @rm -f fastasr diff --git a/README.md b/README.md index 2fc1179..ce01362 100644 --- a/README.md +++ b/README.md @@ -1,2 +1 @@ -# FastASR -基于PaddleSpeech所使用的conformer模型,使用C++的高效实现模型推理,在树莓派4B等ARM平台运行也可流畅运行。 +hello diff --git a/src/Audio.cpp b/src/Audio.cpp new file mode 100644 index 0000000..4019368 --- /dev/null +++ b/src/Audio.cpp @@ -0,0 +1,372 @@ +#include "Audio.h" +#include "predefine_coe.h" +#include +#include +#include +#include +#include +using namespace std; + +Audio::Audio(const char *filename) +{ + loadwav(filename); + audio2feature(); +} + +Audio::~Audio() +{ + free(speech); +} + +void Audio::loadwav(const char *filename) +{ + + FILE *fp; + fp = fopen(filename, "rb"); + fseek(fp, 0, SEEK_END); + uint32_t nFileLen = ftell(fp); + fseek(fp, 44, SEEK_SET); + + speech_len = (nFileLen - 44) / 2; + printf("Audio time is %f s.\n", (float)speech_len / 16000); + speech = (int16_t *)malloc(sizeof(int16_t) * speech_len); + int ret = fread(speech, sizeof(int16_t), speech_len, fp); + + fclose(fp); +} + +void Audio::audio2feature() +{ + const float *window = (const float *)&window_hex; + int window_size = 400; + int fft_size = 512; + int window_shift = 160; + int fbank_time = (int)((speech_len - window_size) / window_shift + 1); + + fbank_feature = new Tensor(fbank_time, 80); + + float *fft_input = (float *)fftwf_malloc(sizeof(float) * fft_size); + fftwf_complex *fft_out = + (fftwf_complex *)fftwf_malloc(sizeof(fftwf_complex) * fft_size); + memset(fft_input, 0, sizeof(float) * fft_size); + fftwf_plan p; + p = fftwf_plan_dft_r2c_1d(fft_size, fft_input, fft_out, FFTW_ESTIMATE); + + int i, j, idx; + float fbank[80]; + for (i = 0, idx = 0; i < fbank_time; i++, idx += window_shift) { + float tmp_mean = 0; + for (j = 0; j < window_size; j++) { + tmp_mean += speech[idx + j]; + } + + tmp_mean = tmp_mean / window_size; + + float pre_val = (float)speech[idx] - tmp_mean; + + for (j = 0; j < window_size; j++) { + float win = window[j]; + float cur_val = (float)speech[idx + j] - tmp_mean; + fft_input[j] = win * (cur_val - 0.97 * pre_val); + pre_val = cur_val; + } + fftwf_execute(p); + melspect((float *)fft_out, fbank_feature->buff + i * 80); + } + fftwf_free(fft_out); + fftwf_free(fft_input); +} + +void Audio::melspect(float *din, float *dout) +{ + float fftmag[256]; + float tmp; + const float *melcoe = (const float *)melcoe_hex; + int i; + for (i = 0; i < 256; i++) { + float real = din[2 * i]; + float imag = din[2 * i + 1]; + fftmag[i] = real * real + imag * imag; + } + dout[0] = melcoe[0] * fftmag[1] + melcoe[1] * fftmag[2]; + dout[1] = melcoe[2] * fftmag[2]; + dout[2] = melcoe[3] * fftmag[3]; + dout[3] = melcoe[4] * fftmag[3] + melcoe[5] * fftmag[4]; + dout[4] = melcoe[6] * fftmag[4] + melcoe[7] * fftmag[5]; + dout[5] = melcoe[8] * fftmag[5] + melcoe[9] * fftmag[6]; + dout[6] = melcoe[10] * fftmag[6] + melcoe[11] * fftmag[7]; + dout[7] = melcoe[12] * fftmag[7]; + dout[8] = melcoe[13] * fftmag[8]; + dout[9] = melcoe[14] * fftmag[8] + melcoe[15] * fftmag[9]; + dout[10] = melcoe[16] * fftmag[9] + melcoe[17] * fftmag[10]; + dout[11] = melcoe[18] * fftmag[10] + melcoe[19] * fftmag[11]; + dout[12] = melcoe[20] * fftmag[11] + melcoe[21] * fftmag[12] + + melcoe[22] * fftmag[13]; + dout[13] = melcoe[23] * fftmag[12] + melcoe[24] * fftmag[13] + + melcoe[25] * fftmag[14]; + dout[14] = melcoe[26] * fftmag[14] + melcoe[27] * fftmag[15]; + dout[15] = melcoe[28] * fftmag[15] + melcoe[29] * fftmag[16]; + dout[16] = melcoe[30] * fftmag[16] + melcoe[31] * fftmag[17]; + dout[17] = melcoe[32] * fftmag[17] + melcoe[33] * fftmag[18]; + dout[18] = melcoe[34] * fftmag[18] + melcoe[35] * fftmag[19] + + melcoe[36] * fftmag[20]; + dout[19] = melcoe[37] * fftmag[19] + melcoe[38] * fftmag[20] + + melcoe[39] * fftmag[21]; + dout[20] = melcoe[40] * fftmag[21] + melcoe[41] * fftmag[22]; + dout[21] = melcoe[42] * fftmag[22] + melcoe[43] * fftmag[23] + + melcoe[44] * fftmag[24]; + dout[22] = melcoe[45] * fftmag[23] + melcoe[46] * fftmag[24] + + melcoe[47] * fftmag[25]; + dout[23] = melcoe[48] * fftmag[25] + melcoe[49] * fftmag[26] + + melcoe[50] * fftmag[27]; + dout[24] = melcoe[51] * fftmag[26] + melcoe[52] * fftmag[27] + + melcoe[53] * fftmag[28]; + dout[25] = melcoe[54] * fftmag[28] + melcoe[55] * fftmag[29] + + melcoe[56] * fftmag[30]; + dout[26] = melcoe[57] * fftmag[29] + melcoe[58] * fftmag[30] + + melcoe[59] * fftmag[31] + melcoe[60] * fftmag[32]; + dout[27] = melcoe[61] * fftmag[31] + melcoe[62] * fftmag[32] + + melcoe[63] * fftmag[33]; + dout[28] = melcoe[64] * fftmag[33] + melcoe[65] * fftmag[34] + + melcoe[66] * fftmag[35]; + dout[29] = melcoe[67] * fftmag[34] + melcoe[68] * fftmag[35] + + melcoe[69] * fftmag[36] + melcoe[70] * fftmag[37]; + dout[30] = melcoe[71] * fftmag[36] + melcoe[72] * fftmag[37] + + melcoe[73] * fftmag[38] + melcoe[74] * fftmag[39]; + dout[31] = melcoe[75] * fftmag[38] + melcoe[76] * fftmag[39] + + melcoe[77] * fftmag[40] + melcoe[78] * fftmag[41]; + dout[32] = melcoe[79] * fftmag[40] + melcoe[80] * fftmag[41] + + melcoe[81] * fftmag[42] + melcoe[82] * fftmag[43]; + dout[33] = melcoe[83] * fftmag[42] + melcoe[84] * fftmag[43] + + melcoe[85] * fftmag[44] + melcoe[86] * fftmag[45]; + dout[34] = melcoe[87] * fftmag[44] + melcoe[88] * fftmag[45] + + melcoe[89] * fftmag[46] + melcoe[90] * fftmag[47]; + dout[35] = melcoe[91] * fftmag[46] + melcoe[92] * fftmag[47] + + melcoe[93] * fftmag[48] + melcoe[94] * fftmag[49]; + dout[36] = melcoe[95] * fftmag[48] + melcoe[96] * fftmag[49] + + melcoe[97] * fftmag[50] + melcoe[98] * fftmag[51]; + dout[37] = melcoe[99] * fftmag[50] + melcoe[100] * fftmag[51] + + melcoe[101] * fftmag[52] + melcoe[102] * fftmag[53] + + melcoe[103] * fftmag[54]; + dout[38] = melcoe[104] * fftmag[52] + melcoe[105] * fftmag[53] + + melcoe[106] * fftmag[54] + melcoe[107] * fftmag[55] + + melcoe[108] * fftmag[56]; + dout[39] = melcoe[109] * fftmag[55] + melcoe[110] * fftmag[56] + + melcoe[111] * fftmag[57] + melcoe[112] * fftmag[58]; + dout[40] = melcoe[113] * fftmag[57] + melcoe[114] * fftmag[58] + + melcoe[115] * fftmag[59] + melcoe[116] * fftmag[60] + + melcoe[117] * fftmag[61]; + dout[41] = melcoe[118] * fftmag[59] + melcoe[119] * fftmag[60] + + melcoe[120] * fftmag[61] + melcoe[121] * fftmag[62] + + melcoe[122] * fftmag[63] + melcoe[123] * fftmag[64]; + dout[42] = melcoe[124] * fftmag[62] + melcoe[125] * fftmag[63] + + melcoe[126] * fftmag[64] + melcoe[127] * fftmag[65] + + melcoe[128] * fftmag[66]; + dout[43] = melcoe[129] * fftmag[65] + melcoe[130] * fftmag[66] + + melcoe[131] * fftmag[67] + melcoe[132] * fftmag[68] + + melcoe[133] * fftmag[69]; + dout[44] = melcoe[134] * fftmag[67] + melcoe[135] * fftmag[68] + + melcoe[136] * fftmag[69] + melcoe[137] * fftmag[70] + + melcoe[138] * fftmag[71] + melcoe[139] * fftmag[72]; + dout[45] = melcoe[140] * fftmag[70] + melcoe[141] * fftmag[71] + + melcoe[142] * fftmag[72] + melcoe[143] * fftmag[73] + + melcoe[144] * fftmag[74] + melcoe[145] * fftmag[75]; + dout[46] = melcoe[146] * fftmag[73] + melcoe[147] * fftmag[74] + + melcoe[148] * fftmag[75] + melcoe[149] * fftmag[76] + + melcoe[150] * fftmag[77] + melcoe[151] * fftmag[78]; + dout[47] = melcoe[152] * fftmag[76] + melcoe[153] * fftmag[77] + + melcoe[154] * fftmag[78] + melcoe[155] * fftmag[79] + + melcoe[156] * fftmag[80] + melcoe[157] * fftmag[81]; + dout[48] = melcoe[158] * fftmag[79] + melcoe[159] * fftmag[80] + + melcoe[160] * fftmag[81] + melcoe[161] * fftmag[82] + + melcoe[162] * fftmag[83] + melcoe[163] * fftmag[84]; + dout[49] = melcoe[164] * fftmag[82] + melcoe[165] * fftmag[83] + + melcoe[166] * fftmag[84] + melcoe[167] * fftmag[85] + + melcoe[168] * fftmag[86] + melcoe[169] * fftmag[87] + + melcoe[170] * fftmag[88]; + dout[50] = melcoe[171] * fftmag[85] + melcoe[172] * fftmag[86] + + melcoe[173] * fftmag[87] + melcoe[174] * fftmag[88] + + melcoe[175] * fftmag[89] + melcoe[176] * fftmag[90] + + melcoe[177] * fftmag[91]; + dout[51] = melcoe[178] * fftmag[89] + melcoe[179] * fftmag[90] + + melcoe[180] * fftmag[91] + melcoe[181] * fftmag[92] + + melcoe[182] * fftmag[93] + melcoe[183] * fftmag[94] + + melcoe[184] * fftmag[95]; + dout[52] = melcoe[185] * fftmag[92] + melcoe[186] * fftmag[93] + + melcoe[187] * fftmag[94] + melcoe[188] * fftmag[95] + + melcoe[189] * fftmag[96] + melcoe[190] * fftmag[97] + + melcoe[191] * fftmag[98]; + dout[53] = melcoe[192] * fftmag[96] + melcoe[193] * fftmag[97] + + melcoe[194] * fftmag[98] + melcoe[195] * fftmag[99] + + melcoe[196] * fftmag[100] + melcoe[197] * fftmag[101] + + melcoe[198] * fftmag[102]; + dout[54] = melcoe[199] * fftmag[99] + melcoe[200] * fftmag[100] + + melcoe[201] * fftmag[101] + melcoe[202] * fftmag[102] + + melcoe[203] * fftmag[103] + melcoe[204] * fftmag[104] + + melcoe[205] * fftmag[105] + melcoe[206] * fftmag[106]; + dout[55] = melcoe[207] * fftmag[103] + melcoe[208] * fftmag[104] + + melcoe[209] * fftmag[105] + melcoe[210] * fftmag[106] + + melcoe[211] * fftmag[107] + melcoe[212] * fftmag[108] + + melcoe[213] * fftmag[109] + melcoe[214] * fftmag[110]; + dout[56] = melcoe[215] * fftmag[107] + melcoe[216] * fftmag[108] + + melcoe[217] * fftmag[109] + melcoe[218] * fftmag[110] + + melcoe[219] * fftmag[111] + melcoe[220] * fftmag[112] + + melcoe[221] * fftmag[113] + melcoe[222] * fftmag[114]; + dout[57] = melcoe[223] * fftmag[111] + melcoe[224] * fftmag[112] + + melcoe[225] * fftmag[113] + melcoe[226] * fftmag[114] + + melcoe[227] * fftmag[115] + melcoe[228] * fftmag[116] + + melcoe[229] * fftmag[117] + melcoe[230] * fftmag[118] + + melcoe[231] * fftmag[119]; + dout[58] = melcoe[232] * fftmag[115] + melcoe[233] * fftmag[116] + + melcoe[234] * fftmag[117] + melcoe[235] * fftmag[118] + + melcoe[236] * fftmag[119] + melcoe[237] * fftmag[120] + + melcoe[238] * fftmag[121] + melcoe[239] * fftmag[122] + + melcoe[240] * fftmag[123]; + dout[59] = melcoe[241] * fftmag[120] + melcoe[242] * fftmag[121] + + melcoe[243] * fftmag[122] + melcoe[244] * fftmag[123] + + melcoe[245] * fftmag[124] + melcoe[246] * fftmag[125] + + melcoe[247] * fftmag[126] + melcoe[248] * fftmag[127] + + melcoe[249] * fftmag[128]; + dout[60] = melcoe[250] * fftmag[124] + melcoe[251] * fftmag[125] + + melcoe[252] * fftmag[126] + melcoe[253] * fftmag[127] + + melcoe[254] * fftmag[128] + melcoe[255] * fftmag[129] + + melcoe[256] * fftmag[130] + melcoe[257] * fftmag[131] + + melcoe[258] * fftmag[132]; + dout[61] = melcoe[259] * fftmag[129] + melcoe[260] * fftmag[130] + + melcoe[261] * fftmag[131] + melcoe[262] * fftmag[132] + + melcoe[263] * fftmag[133] + melcoe[264] * fftmag[134] + + melcoe[265] * fftmag[135] + melcoe[266] * fftmag[136] + + melcoe[267] * fftmag[137]; + dout[62] = melcoe[268] * fftmag[133] + melcoe[269] * fftmag[134] + + melcoe[270] * fftmag[135] + melcoe[271] * fftmag[136] + + melcoe[272] * fftmag[137] + melcoe[273] * fftmag[138] + + melcoe[274] * fftmag[139] + melcoe[275] * fftmag[140] + + melcoe[276] * fftmag[141] + melcoe[277] * fftmag[142]; + dout[63] = melcoe[278] * fftmag[138] + melcoe[279] * fftmag[139] + + melcoe[280] * fftmag[140] + melcoe[281] * fftmag[141] + + melcoe[282] * fftmag[142] + melcoe[283] * fftmag[143] + + melcoe[284] * fftmag[144] + melcoe[285] * fftmag[145] + + melcoe[286] * fftmag[146] + melcoe[287] * fftmag[147]; + dout[64] = melcoe[288] * fftmag[143] + melcoe[289] * fftmag[144] + + melcoe[290] * fftmag[145] + melcoe[291] * fftmag[146] + + melcoe[292] * fftmag[147] + melcoe[293] * fftmag[148] + + melcoe[294] * fftmag[149] + melcoe[295] * fftmag[150] + + melcoe[296] * fftmag[151] + melcoe[297] * fftmag[152] + + melcoe[298] * fftmag[153]; + dout[65] = melcoe[299] * fftmag[148] + melcoe[300] * fftmag[149] + + melcoe[301] * fftmag[150] + melcoe[302] * fftmag[151] + + melcoe[303] * fftmag[152] + melcoe[304] * fftmag[153] + + melcoe[305] * fftmag[154] + melcoe[306] * fftmag[155] + + melcoe[307] * fftmag[156] + melcoe[308] * fftmag[157] + + melcoe[309] * fftmag[158]; + dout[66] = melcoe[310] * fftmag[154] + melcoe[311] * fftmag[155] + + melcoe[312] * fftmag[156] + melcoe[313] * fftmag[157] + + melcoe[314] * fftmag[158] + melcoe[315] * fftmag[159] + + melcoe[316] * fftmag[160] + melcoe[317] * fftmag[161] + + melcoe[318] * fftmag[162] + melcoe[319] * fftmag[163] + + melcoe[320] * fftmag[164]; + dout[67] = melcoe[321] * fftmag[159] + melcoe[322] * fftmag[160] + + melcoe[323] * fftmag[161] + melcoe[324] * fftmag[162] + + melcoe[325] * fftmag[163] + melcoe[326] * fftmag[164] + + melcoe[327] * fftmag[165] + melcoe[328] * fftmag[166] + + melcoe[329] * fftmag[167] + melcoe[330] * fftmag[168] + + melcoe[331] * fftmag[169] + melcoe[332] * fftmag[170]; + dout[68] = melcoe[333] * fftmag[165] + melcoe[334] * fftmag[166] + + melcoe[335] * fftmag[167] + melcoe[336] * fftmag[168] + + melcoe[337] * fftmag[169] + melcoe[338] * fftmag[170] + + melcoe[339] * fftmag[171] + melcoe[340] * fftmag[172] + + melcoe[341] * fftmag[173] + melcoe[342] * fftmag[174] + + melcoe[343] * fftmag[175] + melcoe[344] * fftmag[176]; + dout[69] = melcoe[345] * fftmag[171] + melcoe[346] * fftmag[172] + + melcoe[347] * fftmag[173] + melcoe[348] * fftmag[174] + + melcoe[349] * fftmag[175] + melcoe[350] * fftmag[176] + + melcoe[351] * fftmag[177] + melcoe[352] * fftmag[178] + + melcoe[353] * fftmag[179] + melcoe[354] * fftmag[180] + + melcoe[355] * fftmag[181] + melcoe[356] * fftmag[182]; + dout[70] = melcoe[357] * fftmag[177] + melcoe[358] * fftmag[178] + + melcoe[359] * fftmag[179] + melcoe[360] * fftmag[180] + + melcoe[361] * fftmag[181] + melcoe[362] * fftmag[182] + + melcoe[363] * fftmag[183] + melcoe[364] * fftmag[184] + + melcoe[365] * fftmag[185] + melcoe[366] * fftmag[186] + + melcoe[367] * fftmag[187] + melcoe[368] * fftmag[188]; + dout[71] = melcoe[369] * fftmag[183] + melcoe[370] * fftmag[184] + + melcoe[371] * fftmag[185] + melcoe[372] * fftmag[186] + + melcoe[373] * fftmag[187] + melcoe[374] * fftmag[188] + + melcoe[375] * fftmag[189] + melcoe[376] * fftmag[190] + + melcoe[377] * fftmag[191] + melcoe[378] * fftmag[192] + + melcoe[379] * fftmag[193] + melcoe[380] * fftmag[194] + + melcoe[381] * fftmag[195]; + dout[72] = melcoe[382] * fftmag[189] + melcoe[383] * fftmag[190] + + melcoe[384] * fftmag[191] + melcoe[385] * fftmag[192] + + melcoe[386] * fftmag[193] + melcoe[387] * fftmag[194] + + melcoe[388] * fftmag[195] + melcoe[389] * fftmag[196] + + melcoe[390] * fftmag[197] + melcoe[391] * fftmag[198] + + melcoe[392] * fftmag[199] + melcoe[393] * fftmag[200] + + melcoe[394] * fftmag[201] + melcoe[395] * fftmag[202]; + dout[73] = melcoe[396] * fftmag[196] + melcoe[397] * fftmag[197] + + melcoe[398] * fftmag[198] + melcoe[399] * fftmag[199] + + melcoe[400] * fftmag[200] + melcoe[401] * fftmag[201] + + melcoe[402] * fftmag[202] + melcoe[403] * fftmag[203] + + melcoe[404] * fftmag[204] + melcoe[405] * fftmag[205] + + melcoe[406] * fftmag[206] + melcoe[407] * fftmag[207] + + melcoe[408] * fftmag[208] + melcoe[409] * fftmag[209]; + dout[74] = melcoe[410] * fftmag[203] + melcoe[411] * fftmag[204] + + melcoe[412] * fftmag[205] + melcoe[413] * fftmag[206] + + melcoe[414] * fftmag[207] + melcoe[415] * fftmag[208] + + melcoe[416] * fftmag[209] + melcoe[417] * fftmag[210] + + melcoe[418] * fftmag[211] + melcoe[419] * fftmag[212] + + melcoe[420] * fftmag[213] + melcoe[421] * fftmag[214] + + melcoe[422] * fftmag[215] + melcoe[423] * fftmag[216]; + dout[75] = melcoe[424] * fftmag[210] + melcoe[425] * fftmag[211] + + melcoe[426] * fftmag[212] + melcoe[427] * fftmag[213] + + melcoe[428] * fftmag[214] + melcoe[429] * fftmag[215] + + melcoe[430] * fftmag[216] + melcoe[431] * fftmag[217] + + melcoe[432] * fftmag[218] + melcoe[433] * fftmag[219] + + melcoe[434] * fftmag[220] + melcoe[435] * fftmag[221] + + melcoe[436] * fftmag[222] + melcoe[437] * fftmag[223]; + dout[76] = melcoe[438] * fftmag[217] + melcoe[439] * fftmag[218] + + melcoe[440] * fftmag[219] + melcoe[441] * fftmag[220] + + melcoe[442] * fftmag[221] + melcoe[443] * fftmag[222] + + melcoe[444] * fftmag[223] + melcoe[445] * fftmag[224] + + melcoe[446] * fftmag[225] + melcoe[447] * fftmag[226] + + melcoe[448] * fftmag[227] + melcoe[449] * fftmag[228] + + melcoe[450] * fftmag[229] + melcoe[451] * fftmag[230] + + melcoe[452] * fftmag[231]; + dout[77] = melcoe[453] * fftmag[224] + melcoe[454] * fftmag[225] + + melcoe[455] * fftmag[226] + melcoe[456] * fftmag[227] + + melcoe[457] * fftmag[228] + melcoe[458] * fftmag[229] + + melcoe[459] * fftmag[230] + melcoe[460] * fftmag[231] + + melcoe[461] * fftmag[232] + melcoe[462] * fftmag[233] + + melcoe[463] * fftmag[234] + melcoe[464] * fftmag[235] + + melcoe[465] * fftmag[236] + melcoe[466] * fftmag[237] + + melcoe[467] * fftmag[238] + melcoe[468] * fftmag[239]; + dout[78] = melcoe[469] * fftmag[232] + melcoe[470] * fftmag[233] + + melcoe[471] * fftmag[234] + melcoe[472] * fftmag[235] + + melcoe[473] * fftmag[236] + melcoe[474] * fftmag[237] + + melcoe[475] * fftmag[238] + melcoe[476] * fftmag[239] + + melcoe[477] * fftmag[240] + melcoe[478] * fftmag[241] + + melcoe[479] * fftmag[242] + melcoe[480] * fftmag[243] + + melcoe[481] * fftmag[244] + melcoe[482] * fftmag[245] + + melcoe[483] * fftmag[246] + melcoe[484] * fftmag[247]; + dout[79] = melcoe[485] * fftmag[240] + melcoe[486] * fftmag[241] + + melcoe[487] * fftmag[242] + melcoe[488] * fftmag[243] + + melcoe[489] * fftmag[244] + melcoe[490] * fftmag[245] + + melcoe[491] * fftmag[246] + melcoe[492] * fftmag[247] + + melcoe[493] * fftmag[248] + melcoe[494] * fftmag[249] + + melcoe[495] * fftmag[250] + melcoe[496] * fftmag[251] + + melcoe[497] * fftmag[252] + melcoe[498] * fftmag[253] + + melcoe[499] * fftmag[254] + melcoe[500] * fftmag[255]; + global_cmvn(dout); +} + +void Audio::global_cmvn(float *din) +{ + const float *std = (const float *)global_cmvn_std_hex; + const float *mean = (const float *)global_cmvn_mean_hex; + int i; + for (i = 0; i < 80; i++) { + float tmp = din[i] < 1e-7 ? 1e-7 : din[i]; + tmp = log(tmp); + din[i] = (tmp - mean[i]) / std[i]; + } +} diff --git a/src/Audio.h b/src/Audio.h new file mode 100644 index 0000000..ed1f011 --- /dev/null +++ b/src/Audio.h @@ -0,0 +1,27 @@ + +#ifndef AUDIO_H +#define AUDIO_H + + +#include + +#include "Tensor.h" + +class Audio { + private: + int16_t *speech; + int speech_len; + int16_t sample_rate; + + void loadwav(const char *filename); + void audio2feature(); + void melspect(float *din, float *dout); + void global_cmvn(float *din); + + public: + Tensor *fbank_feature; + Audio(const char *filename); + ~Audio(); +}; + +#endif diff --git a/src/CTCDecode.cpp b/src/CTCDecode.cpp new file mode 100644 index 0000000..472132e --- /dev/null +++ b/src/CTCDecode.cpp @@ -0,0 +1,196 @@ +#include "CTCDecode.h" +#include "util.h" +#include +#include +#include +#include +#include +#include +using namespace std; + +#define vocab_size 5537 + +CTCdecode::CTCdecode(float *ctc_weight, float *ctc_bias) + : ctc_weight(ctc_weight), ctc_bias(ctc_bias) +{ +} + +CTCdecode::~CTCdecode() +{ +} + +float log_add(float *din, int len) +{ + float sum = 0; + int i; + for (i = 0; i < len; i++) { + sum = sum + exp(din[i]); + } + return log(sum); +} + +auto char_cmp = [](CharProb a, CharProb b) { return a.prob < b.prob; }; +auto path_cmp = [](PathProb a, PathProb b) { return a.prob < b.prob; }; + +void topk(float *din, int len, set &s) +{ + int i; + for (i = 0; i < 10; i++) { + CharProb tmp; + tmp.char_idx = i; + tmp.prob = din[i]; + s.insert(tmp); + } + + float min = s.begin()->prob; + + for (; i < len; i++) { + if (din[i] > min) { + s.erase(s.begin()); + CharProb tmp; + tmp.char_idx = i; + tmp.prob = din[i]; + s.insert(tmp); + min = s.begin()->prob; + } + } +} + +void ctc_beam_search(Tensor *din, deque &hyps) +{ + int tmax = din->size[2]; + int beam_size = 10; + int i; + + set curr_hyps_set(path_cmp); + PathProb tmp; + tmp.pb = 0; + tmp.prob = 0; + curr_hyps_set.insert(tmp); + + for (i = 0; i < tmax; i++) { + set char_set(char_cmp); + topk(din->buff + i * vocab_size, vocab_size, char_set); + map, PathProb> next_next_map; + for (auto char_it = char_set.begin(); char_it != char_set.end(); + ++char_it) { + int char_idx = char_it->char_idx; + float char_prob = char_it->prob; + for (auto hyps_it = curr_hyps_set.begin(); + hyps_it != curr_hyps_set.end(); hyps_it++) { + int last = -1; + if (hyps_it->prefix.size() > 0) { + int ii = hyps_it->prefix.size() - 1; + last = hyps_it->prefix[ii]; + } + vector curr_prefix(hyps_it->prefix); + vector next_prefix(hyps_it->prefix); + next_prefix.push_back(char_idx); + + if (char_idx == 0) { + auto next_hyps = next_next_map[curr_prefix]; + float tmp[] = {next_hyps.pb, hyps_it->pb + char_prob, + hyps_it->pnb + char_prob}; + next_hyps.pb = log_add(tmp, 3); + next_hyps.prefix = curr_prefix; + next_next_map[curr_prefix] = next_hyps; + } else if (last == char_idx) { + { + auto next_hyps = next_next_map[curr_prefix]; + float tmp[] = {next_hyps.pnb, hyps_it->pnb + char_prob}; + next_hyps.pnb = log_add(tmp, 2); + next_hyps.prefix = curr_prefix; + next_next_map[curr_prefix] = next_hyps; + } + + { + auto next_hyps = next_next_map[next_prefix]; + float tmp[] = {next_hyps.pnb, hyps_it->pb + char_prob}; + next_hyps.pnb = log_add(tmp, 2); + next_hyps.prefix = next_prefix; + next_next_map[next_prefix] = next_hyps; + } + } else { + auto next_hyps = next_next_map[next_prefix]; + float tmp[] = {next_hyps.pnb, hyps_it->pb + char_prob, + hyps_it->pnb + char_prob}; + next_hyps.pnb = log_add(tmp, 3); + next_hyps.prefix = next_prefix; + next_next_map[next_prefix] = next_hyps; + } + } + } + // kaishi + float min = -9999999; + int ii = 0; + curr_hyps_set.clear(); + for (auto map_it = next_next_map.begin(); map_it != next_next_map.end(); + map_it++) { + float tmp[] = {map_it->second.pb, map_it->second.pnb}; + map_it->second.prob = log_add(tmp, 2); + if (ii < 10) { + curr_hyps_set.insert(map_it->second); + min = curr_hyps_set.begin()->prob; + ii++; + } else { + if (min < map_it->second.prob) { + curr_hyps_set.insert(map_it->second); + curr_hyps_set.erase(curr_hyps_set.begin()); + min = curr_hyps_set.begin()->prob; + } + } + } + } + + for (auto hyps_it = curr_hyps_set.begin(); hyps_it != curr_hyps_set.end(); + hyps_it++) { + hyps.push_front(*hyps_it); + // int mm = hyps_it->prefix.size(); + // cout << hyps_it->prefix.size() << endl; + // for (i = 0; i < mm; i++) { + // printf("%d ", hyps_it->prefix[i]); + // } + // printf("\n"); + // printf("%f %f %f\n\n", hyps_it->pb, hyps_it->pnb, hyps_it->prob); + } +} + +void CTCdecode::show_hyps(deque hyps) +{ + for (auto hyps_it = hyps.begin(); hyps_it != hyps.end(); hyps_it++) { + int mm = hyps_it->prefix.size(); + int i; + printf("prefix len is %d, val is [", mm); + for (i = 0; i < mm - 1; i++) { + printf("%d,", hyps_it->prefix[i]); + } + printf("%d]\n", hyps_it->prefix[i]); + printf("pb is %f, pnb is %f, prob is %f\n", hyps_it->pb, hyps_it->pnb, + hyps_it->prob); + } +} + +void CTCdecode::forward(Tensor *din, deque &hyps) +{ + + int mm = din->size[2]; + Tensor ctcin(mm, vocab_size); + int i; + for (i = 0; i < mm; i++) { + int offset = i * vocab_size; + memcpy(ctcin.buff + offset, ctc_bias, sizeof(float) * vocab_size); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, vocab_size, 512, + 1, din->buff, 512, ctc_weight, vocab_size, 1, ctcin.buff, + vocab_size); + + for (i = 0; i < mm; i++) { + int offset = i * vocab_size; + log_softmax(ctcin.buff + offset, vocab_size); + } + + ctc_beam_search(&ctcin, hyps); + // show_hyps(hyps); + +} diff --git a/src/CTCDecode.h b/src/CTCDecode.h new file mode 100644 index 0000000..c3608b9 --- /dev/null +++ b/src/CTCDecode.h @@ -0,0 +1,39 @@ + +#ifndef CTCDECODE_H +#define CTCDECODE_H + +#include +#include +#include +#include + +#include "Tensor.h" +#include "WenetParams.h" + +using namespace std; + +struct CharProb { + int char_idx; + float prob; +}; + +struct PathProb { + vector prefix; + float pb = -INFINITY; + float pnb = -INFINITY; + float prob = -INFINITY; +}; + +class CTCdecode { + private: + float *ctc_weight; + float *ctc_bias; + + public: + CTCdecode(float *ctc_weight, float *ctc_bias); + ~CTCdecode(); + void forward(Tensor *din, deque &hyps); + void show_hyps(deque hyps); +}; + +#endif diff --git a/src/ConvModule.cpp b/src/ConvModule.cpp new file mode 100644 index 0000000..74c6c34 --- /dev/null +++ b/src/ConvModule.cpp @@ -0,0 +1,80 @@ +#include "ConvModule.h" +#include "util.h" +#include +#include +#include + +ConvModule::ConvModule(EncConvParams *params) : params(params) +{ + norm = new LayerNorm(¶ms->norm, 1e-5f); +} + +ConvModule::~ConvModule() +{ +} + +void glu(Tensor *din, Tensor *dout) +{ + int mm = din->buff_size / 1024; + int i, j; + for (i = 0; i < mm; i++) { + for (j = 0; j < 512; j++) { + int in_off = i * 1024 + j; + int out_off = i * 512 + j; + float a = din->buff[in_off]; + float b = din->buff[in_off + 512]; + dout->buff[out_off] = a / (1 + exp(-b)); + } + } +} + +void ConvModule::forward(Tensor *din) +{ + int mm = din->size[2]; + Tensor tmp(mm, 1024); + int i, j; + for (i = 0; i < mm; i++) { + int offset = i * 1024; + memcpy(tmp.buff + offset, params->pointwise_conv1_bias, + sizeof(float) * 1024); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, mm, 1024, 512, 1, + din->buff, 512, params->pointwise_conv1_weight, 512, 1, + tmp.buff, 1024); + glu(&tmp, din); + + Tensor conv_in(1, mm + 14); + Tensor blasin(mm, 15); + conv_in.zeros(); + + for (i = 0; i < 512; i++) { + for (j = 0; j < mm; j++) { + int ii = j * 512 + i; + conv_in.buff[j + 7] = din->buff[ii]; + din->buff[ii] = params->depthwise_conv_bias[i]; + } + for (j = 0; j < mm; j++) { + int offset = j * 15; + memcpy(blasin.buff + offset, conv_in.buff + j, 15 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 1, 15, 1, + blasin.buff, 15, params->depthwise_conv_weight + i * 15, 1, + 1, din->buff + i, 512); + } + + norm->forward(din); + swish(din); + + Tensor tmp2(din); + for (i = 0; i < mm; i++) { + int offset = i * 512; + memcpy(din->buff + offset, params->pointwise_conv2_bias, + sizeof(float) * 512); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, mm, 512, 512, 1, + tmp2.buff, 512, params->pointwise_conv2_weight, 512, 1, + din->buff, 512); +} diff --git a/src/ConvModule.h b/src/ConvModule.h new file mode 100644 index 0000000..1c9d5a4 --- /dev/null +++ b/src/ConvModule.h @@ -0,0 +1,24 @@ + + +#ifndef CONVMODULE_H +#define CONVMODULE_H + +#include + +#include "FeedForward.h" +#include "LayerNorm.h" +#include "Tensor.h" +#include "WenetParams.h" + +class ConvModule { + private: + LayerNorm *norm; + EncConvParams *params; + + public: + ConvModule(EncConvParams *params); + ~ConvModule(); + void forward(Tensor *din); +}; + +#endif diff --git a/src/DecEmbedLayer.cpp b/src/DecEmbedLayer.cpp new file mode 100644 index 0000000..8d5a168 --- /dev/null +++ b/src/DecEmbedLayer.cpp @@ -0,0 +1,22 @@ +#include "DecEmbedLayer.h" +#include +#include + +DecEmbedLayer::DecEmbedLayer(float *params) : params(params) +{ +} +DecEmbedLayer::~DecEmbedLayer() +{ +} +void DecEmbedLayer::forward(Tensor *din, Tensor *&dout) +{ + dout = new Tensor(din->size[2], din->size[3], 512); + + int mm = din->buff_size; + int i; + for (i = 0; i < mm; i++) { + int offset = i * 512; + int offset2 = din->buff[i] * 512; + memcpy(dout->buff + offset, params + offset2, sizeof(float) * 512); + } +} diff --git a/src/DecEmbedLayer.h b/src/DecEmbedLayer.h new file mode 100644 index 0000000..838eb0b --- /dev/null +++ b/src/DecEmbedLayer.h @@ -0,0 +1,22 @@ + + +#ifndef DECEMBEDLAYER_H +#define DECEMBEDLAYER_H + +#include + +#include "PositionEncoding.h" +#include "Tensor.h" +#include "WenetParams.h" + +class DecEmbedLayer { + private: + float *params; + + public: + DecEmbedLayer(float *params); + ~DecEmbedLayer(); + void forward(Tensor *din, Tensor *&dout); +}; + +#endif diff --git a/src/DecSelfAttn.cpp b/src/DecSelfAttn.cpp new file mode 100644 index 0000000..41fe367 --- /dev/null +++ b/src/DecSelfAttn.cpp @@ -0,0 +1,96 @@ +#include "DecSelfAttn.h" +#include "util.h" +#include + +DecSelfAttn::DecSelfAttn(SelfAttnParams *params) : params(params) +{ +} + +DecSelfAttn::~DecSelfAttn() +{ +} + +extern void linear_forward(Tensor *din, Tensor *dout, + float *weight, float *bias); +// { +// int mm = din->buff_size / 512; +// int i; +// if (bias != 0) { +// for (i = 0; i < mm; i++) { +// int offset = i * 512; +// memcpy(dout->buff + offset, bias, sizeof(float) * 512); +// } +// } else { +// dout->zeros(); +// } + +// cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 512, 512, 1, +// din->buff, 512, weight, 512, 1, dout->buff, 512); +// } + +void DecSelfAttn::forward(Tensor *query, Tensor *key, + Tensor *value, Tensor *mask, + Tensor *dout) +{ + + Tensor q(query->size[1], query->size[2], 8, query->size[3] / 8); + Tensor k(key->size[1], key->size[2], 8, key->size[3] / 8); + Tensor v(value->size[1], value->size[2], 8, value->size[3] / 8); + + linear_forward(query, &q, params->linear_q_weight, params->linear_q_bias); + + linear_forward(key, &k, params->linear_k_weight, params->linear_k_bias); + + linear_forward(value, &v, params->linear_v_weight, params->linear_v_bias); + + // SaveDataFile("/home/ck/matlab/conformer/test.bin", v.buff, + // v.buff_size * sizeof(float)); + + int n_batch = q.size[0]; + int n_head = 8; + int n_query = q.size[1]; + int n_key = k.size[1]; + + Tensor attn(1, 1, n_query, n_key); + // attn.shape(); + dout->zeros(); + + int i, j; + for (i = 0; i < n_batch; i++) { + for (j = 0; j < n_head; j++) { + int q_offset = i * q.size[1] * q.size[2] * q.size[3] + j * 64; + int k_offset = i * k.size[1] * k.size[2] * k.size[3] + j * 64; + + attn.zeros(); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, q.size[1], + k.size[1], q.size[3], 1, q.buff + q_offset, 512, + k.buff + k_offset, 512, 1, attn.buff, n_key); + int ii, jj; + for (ii = 0; ii < attn.buff_size; ii++) { + attn.buff[ii] = attn.buff[ii] / 8; + } + + for (ii = 0; ii < n_query; ii++) { + int offset = ii * n_key; + softmax(attn.buff + offset, mask->buff[i * n_query + ii], + n_key); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, attn.size[2], + v.size[3], attn.size[3], 1, attn.buff, attn.size[3], + v.buff + k_offset, 512, 1, dout->buff + q_offset, 512); + } + } + + Tensor linear_in(dout); + int mm = dout->buff_size / 512; + for (i = 0; i < mm; i++) { + int offset = i * 512; + memcpy(dout->buff + offset, params->linear_out_bias, + sizeof(float) * 512); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 512, 512, 1, + linear_in.buff, 512, params->linear_out_weight, 512, 1, + dout->buff, 512); +} diff --git a/src/DecSelfAttn.h b/src/DecSelfAttn.h new file mode 100644 index 0000000..bfc108a --- /dev/null +++ b/src/DecSelfAttn.h @@ -0,0 +1,23 @@ + +#ifndef DECSELFATTN_H +#define DECSELFATTN_H + +#include + +#include "FeedForward.h" +#include "LayerNorm.h" +#include "Tensor.h" +#include "WenetParams.h" + +class DecSelfAttn { + private: + SelfAttnParams *params; + + public: + DecSelfAttn(SelfAttnParams *params); + ~DecSelfAttn(); + void forward(Tensor *query, Tensor *key, Tensor *value, + Tensor *mask, Tensor *dout); +}; + +#endif diff --git a/src/Decoder.cpp b/src/Decoder.cpp new file mode 100644 index 0000000..651f73c --- /dev/null +++ b/src/Decoder.cpp @@ -0,0 +1,67 @@ +#include "Decoder.h" +#include "util.h" +#include +#include + +Decoder::Decoder(DecoderParams *params, PositionEncoding *pos_enc) + : params(params), pos_enc(pos_enc) +{ + embed = new DecEmbedLayer(params->embed_weight); + int i; + for (i = 0; i < 6; i++) { + sub_decoder[i] = new SubDecoder(¶ms->sub_decoder[i]); + } + norm_after = new LayerNorm(¶ms->after_norm, 1e-12f); +} + +Decoder::~Decoder() +{ +} + +void Decoder::forward(Tensor *&hyps_pad, Tensor *&hyps_mask, + Tensor *&encoder_out, Tensor *&encoder_mask, + Tensor *&dout) +{ + // printf("Decoder!!!!\n"); + + int tmp = 0x41b504f3; + float scale = *((float *)&tmp); + + Tensor *embed_out; + Tensor *pos_code; + embed->forward(hyps_pad, embed_out); + pos_enc->fetch(embed_out->size[2], pos_code); + int mm = embed_out->size[1]; + int i, j; + for (i = 0; i < mm; i++) { + int offset = pos_code->buff_size * i; + float *buff = embed_out->buff + offset; + for (j = 0; j < pos_code->buff_size; j++) { + buff[j] = scale * buff[j] + pos_code->buff[j]; + } + } + + for (i = 0; i < 6; i++) { + sub_decoder[i]->forward(embed_out, hyps_mask, encoder_out, + encoder_mask); + } + + + norm_after->forward(embed_out); + dout = new Tensor(embed_out->size[1], embed_out->size[2], 5537); + + mm = dout->buff_size / 5537; + for (i = 0; i < mm; i++) { + int offset = i * 5537; + memcpy(dout->buff + offset, params->output_bias, 5537 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 5537, 512, 1, + embed_out->buff, 512, params->output_weight, 5537, 1, + dout->buff, 5537); + + for (i = 0; i < mm; i++) { + int offset = i * 5537; + log_softmax(dout->buff + offset, 5537); + } +} diff --git a/src/Decoder.h b/src/Decoder.h new file mode 100644 index 0000000..4cb7614 --- /dev/null +++ b/src/Decoder.h @@ -0,0 +1,30 @@ + +#ifndef DECODER_H +#define DECODER_H + +#include + +#include "DecEmbedLayer.h" +#include "PositionEncoding.h" +#include "SubDecoder.h" +#include "SubEncoder.h" +#include "Tensor.h" +#include "WenetParams.h" + +class Decoder { + private: + DecoderParams *params; + DecEmbedLayer *embed; + PositionEncoding *pos_enc; + SubDecoder *sub_decoder[6]; + LayerNorm *norm_after; + + public: + Decoder(DecoderParams *params, PositionEncoding *pos_enc); + ~Decoder(); + void forward(Tensor *&hyps_pad, Tensor *&pad_len, + Tensor *&encoder_out, Tensor *&encoder_mask, + Tensor *&dout); +}; + +#endif diff --git a/src/EmbedLayer.cpp b/src/EmbedLayer.cpp new file mode 100644 index 0000000..573d5f2 --- /dev/null +++ b/src/EmbedLayer.cpp @@ -0,0 +1,215 @@ +#include "EmbedLayer.h" +#include "util.h" +#include +#include +#include +#include +#include + +using namespace std; + +EmbedLayer::EmbedLayer(EncEmbedParams *params) : params(params) +{ +} + +EmbedLayer::~EmbedLayer() +{ +} + +void EmbedLayer::forward(Tensor *din, Tensor *&dout) +{ + Tensor *conv0_out; + Tensor *conv1_out; + + conv0_forward(din, conv0_out); + conv1_forward(conv0_out, conv1_out); + linear_out_forward(conv1_out, dout); + // cout << "test4" << endl; + + // dout_h = conv1_column; + // dout_w = conv1_row; + + + // cout << "test5" << endl; +} + +// void EmbedLayer::init(int width, int hight) +// { +// get_conv_ind(1, row, column, 3, 2, conv0_row, conv0_column, conv0_idxs); +// get_conv_ind(0, conv0_row, conv0_column, 3, 2, conv1_row, conv1_column, +// conv1_idxs); +// conv0_size = conv0_row * conv0_column; +// conv1_size = conv1_row * conv1_column; +// } + +void EmbedLayer::get_conv_ind(int trans, int in_row, int in_column, int kernel, + int stride, int &out_row, int &out_column, + int *&out_idxs) +{ + + out_row = (in_row - kernel + stride) / stride; + out_column = (in_column - kernel + stride) / stride; + if (trans) { + int tmp = out_row; + out_row = out_column; + out_column = tmp; + } + out_idxs = (int *)malloc(sizeof(int) * out_row * out_column * 9); + int i, j; + int mm = 0; + if (trans) { + for (i = 0; i < in_column - kernel + 1; i = i + stride) { + for (j = 0; j < in_row - kernel + 1; j = j + stride) { + int nn1 = (j * in_column + i); + int nn2 = nn1 + in_column; + int nn3 = nn2 + in_column; + + out_idxs[mm++] = nn1++; + out_idxs[mm++] = nn2++; + out_idxs[mm++] = nn3++; + + out_idxs[mm++] = nn1++; + out_idxs[mm++] = nn2++; + out_idxs[mm++] = nn3++; + + out_idxs[mm++] = nn1++; + out_idxs[mm++] = nn2++; + out_idxs[mm++] = nn3++; + } + } + } else { + for (i = 0; i < in_row - kernel + 1; i = i + stride) { + for (j = 0; j < in_column - kernel + 1; j = j + stride) { + int nn = (i * in_column + j); + out_idxs[mm++] = nn; + out_idxs[mm++] = nn + 1; + out_idxs[mm++] = nn + 2; + + nn = nn + in_column; + out_idxs[mm++] = nn; + out_idxs[mm++] = nn + 1; + out_idxs[mm++] = nn + 2; + + nn = nn + in_column; + out_idxs[mm++] = nn; + out_idxs[mm++] = nn + 1; + out_idxs[mm++] = nn + 2; + } + } + } +} + +void EmbedLayer::conv0_forward(Tensor *din, Tensor *&dout) +{ + + int row = din->size[2]; + int column = din->size[3]; + + int conv0_row, conv0_column; + int *conv0_idxs; + + get_conv_ind(1, row, column, 3, 2, conv0_row, conv0_column, conv0_idxs); + + int conv0_size = conv0_row * conv0_column; + + Tensor blas_in(conv0_size, 9); + Tensor blas_out(conv0_size, 512); + + dout = new Tensor(512, conv0_row, conv0_column); + + int i; + for (i = 0; i < blas_in.buff_size; i++) { + int ii = conv0_idxs[i]; + blas_in.buff[i] = din->buff[ii]; + } + + for (i = 0; i < conv0_size; i++) { + int offset = i * 512; + memcpy(blas_out.buff + offset, params->conv0_bias, 512 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, conv0_size, 512, 9, + 1, blas_in.buff, 9, params->conv0_weight, 512, 1, blas_out.buff, + 512); + + for (i = 0; i < blas_out.buff_size; i++) { + int ii = i % 512; + int jj = i / 512; + int kk = ii * conv0_size + jj; + dout->buff[kk] = blas_out.buff[i] > 0 ? blas_out.buff[i] : 0; + } + + free(conv0_idxs); +} + +void EmbedLayer::conv1_forward(Tensor *din, Tensor *&dout) +{ + + int row = din->size[2]; + int column = din->size[3]; + + int conv_row, conv_column; + int *conv_idxs; + + get_conv_ind(0, row, column, 3, 2, conv_row, conv_column, conv_idxs); + int conv_size = conv_row * conv_column; + + Tensor blas_in(conv_size, 9); + Tensor blas_out(conv_size, 512); + + int i; + + for (i = 0; i < conv_size; i++) { + int offset = i * 512; + memcpy(blas_out.buff + offset, params->conv1_bias, 512 * sizeof(float)); + } + + for (i = 0; i < 512; i++) { + int in_offset = i * row * column; + int weight_offset = i * 512 * 9; + float *sub_conv_in = din->buff + in_offset; + float *sub_weight = params->conv1_weight + weight_offset; + + int mm; + for (mm = 0; mm < blas_in.buff_size; mm++) { + int ii = conv_idxs[mm]; + blas_in.buff[mm] = sub_conv_in[ii]; + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, conv_size, 512, + 9, 1, blas_in.buff, 9, sub_weight, 512, 1, blas_out.buff, + 512); + } + + dout = new Tensor(512, conv_row, conv_column); + + for (i = 0; i < blas_out.buff_size; i++) { + int ii = i / (conv_column * 512); + int jj = (i % (conv_column * 512)) / 512; + int kk = i % 512; + int mm = jj * conv_row * 512 + conv_row * kk + ii; + dout->buff[mm] = blas_out.buff[i] > 0 ? blas_out.buff[i] : 0; + } +} + +void EmbedLayer::linear_out_forward(Tensor *din, Tensor *&dout) +{ + + int Tmax = din->size[3]; + int Fmax = din->size[2]; + + int tmp = 0x41b504f3; + float scale = *((float *)&tmp); + + dout = new Tensor(Tmax, 512); + + int i; + for (i = 0; i < Tmax; i++) { + int offset = i * 512; + memcpy(dout->buff + offset, params->out0_bias, 512 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, Tmax, 512, + 512 * Fmax, scale, din->buff, 512 * Fmax, params->out0_weight, + 512, scale, dout->buff, 512); +} diff --git a/src/EmbedLayer.h b/src/EmbedLayer.h new file mode 100644 index 0000000..67e1abd --- /dev/null +++ b/src/EmbedLayer.h @@ -0,0 +1,26 @@ + +#ifndef EMBEDLAYER_H +#define EMBEDLAYER_H + +#include + +#include "Tensor.h" +#include "WenetParams.h" + +class EmbedLayer { + private: + EncEmbedParams *params; + void get_conv_ind(int trans, int in_row, int in_column, int kernel, + int stride, int &out_row, int &out_column, + int *&out_idxs); + void conv0_forward(Tensor *din, Tensor *&dout); + void conv1_forward(Tensor *din, Tensor *&dout); + void linear_out_forward(Tensor *din, Tensor *&dout); + + public: + EmbedLayer(EncEmbedParams *params); + ~EmbedLayer(); + void forward(Tensor *din, Tensor *&dout); +}; + +#endif diff --git a/src/EncSelfAttn.cpp b/src/EncSelfAttn.cpp new file mode 100644 index 0000000..e8a669f --- /dev/null +++ b/src/EncSelfAttn.cpp @@ -0,0 +1,115 @@ +#include +#include +#include + +#include "EncSelfAttn.h" +#include "util.h" + +EncSelfAttn::EncSelfAttn(EncSelfAttnParams *params) : params(params) +{ +} + +EncSelfAttn::~EncSelfAttn() +{ +} + +void linear_forward(Tensor *din, Tensor *dout, float *weight, + float *bias) +{ + int mm = din->buff_size / 512; + int i; + if (bias != 0) { + for (i = 0; i < mm; i++) { + int offset = i * 512; + memcpy(dout->buff + offset, bias, sizeof(float) * 512); + } + } else { + dout->zeros(); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 512, 512, 1, + din->buff, 512, weight, 512, 1, dout->buff, 512); +} + +void EncSelfAttn::forward(Tensor *query, Tensor *key, + Tensor *value, Tensor *pe, + Tensor *dout) +{ + Tensor q(query->size[2], 8, query->size[3] / 8); + Tensor k(key->size[2], 8, key->size[3] / 8); + Tensor v(value->size[2], 8, value->size[3] / 8); + Tensor p(pe->size[2], 8, pe->size[3] / 8); + + linear_forward(query, &q, params->linear0.linear_q_weight, + params->linear0.linear_q_bias); + + linear_forward(key, &k, params->linear0.linear_k_weight, + params->linear0.linear_k_bias); + + linear_forward(value, &v, params->linear0.linear_v_weight, + params->linear0.linear_v_bias); + + linear_forward(pe, &p, params->linear_pos_weight, NULL); + + Tensor q_with_bias_u(&q); + Tensor q_with_bias_v(&q); + + int i, j; + for (i = 0; i < q.buff_size; i++) { + int ii = i % 512; + q_with_bias_u.buff[i] += params->pos_bias_u[ii]; + q_with_bias_v.buff[i] += params->pos_bias_v[ii]; + } + + Tensor scores(q.size[1], 8, k.size[1]); + + scores.zeros(); + + for (i = 0; i < 8; i++) { + int offset1 = q.size[3] * i; + int offset2 = scores.size[3] * i; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, q.size[1], + k.size[1], q.size[3], 1, q_with_bias_u.buff + offset1, 512, + k.buff + offset1, 512, 1, scores.buff + offset2, + k.size[1] * 8); + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, q.size[1], + k.size[1], q.size[3], 1, q_with_bias_v.buff + offset1, 512, + p.buff + offset1, 512, 1, scores.buff + offset2, + k.size[1] * 8); + } + + for (i = 0; i < scores.buff_size; i++) { + scores.buff[i] = scores.buff[i] / 8; + } + int mm = scores.buff_size / scores.size[3]; + + for (i = 0; i < mm; i++) { + int offset = i * scores.size[3]; + softmax(scores.buff + offset, scores.size[3], scores.size[3]); + } + + Tensor tmp(dout->size[2], dout->size[3]); + tmp.zeros(); + + for (i = 0; i < 8; i++) { + int offset1 = scores.size[3] * i; + int offset2 = v.size[3] * i; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, scores.size[1], + v.size[3], v.size[1], 1, scores.buff + offset1, + scores.size[3] * 8, v.buff + offset2, 512, 1, + tmp.buff + offset2, 512); + } + + for (i = 0; i < dout->size[2]; i++) { + int offset = i * 512; + memcpy(dout->buff + offset, params->linear0.linear_out_bias, + 512 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, dout->size[2], 512, + 512, 1, tmp.buff, 512, params->linear0.linear_out_weight, 512, + 1, dout->buff, 512); +} diff --git a/src/EncSelfAttn.h b/src/EncSelfAttn.h new file mode 100644 index 0000000..ffcbe23 --- /dev/null +++ b/src/EncSelfAttn.h @@ -0,0 +1,24 @@ + + +#ifndef ENCSELFATTN_H +#define ENCSELFATTN_H + +#include + +#include "FeedForward.h" +#include "LayerNorm.h" +#include "Tensor.h" +#include "WenetParams.h" + +class EncSelfAttn { + private: + EncSelfAttnParams *params; + + public: + EncSelfAttn(EncSelfAttnParams *params); + ~EncSelfAttn(); + void forward(Tensor *query, Tensor *key, Tensor *value, Tensor *pe, + Tensor *dout); +}; + +#endif diff --git a/src/Encoder.cpp b/src/Encoder.cpp new file mode 100644 index 0000000..f13a8a8 --- /dev/null +++ b/src/Encoder.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include + +#include "Encoder.h" +#include "FeedForward.h" +#include "LayerNorm.h" +#include "Tensor.h" +#include "util.h" + +using namespace std; + +Encoder::Encoder(EncoderParams *params, PositionEncoding *pos_enc) + : params(params), pos_enc(pos_enc) +{ + embed = new EmbedLayer(¶ms->embed); + int i; + for (i = 0; i < 12; i++) { + subencoder[i] = new SubEncoder(¶ms->sub_encoder[i]); + } + after_norm = new LayerNorm(¶ms->after_norm, 1e-12f); +} + +Encoder::~Encoder() +{ + delete embed; +} + +void Encoder::forward(Tensor *din, Tensor *&dout) +{ + + embed->forward(din, dout); + Tensor *pe_code; + // dout->shape(); + pos_enc->fetch(dout->size[2], pe_code); + int i; + for (i = 0; i < 12; i++) { + subencoder[i]->forward(dout, pe_code); + } + after_norm->forward(dout); +} diff --git a/src/Encoder.h b/src/Encoder.h new file mode 100644 index 0000000..4bc86c2 --- /dev/null +++ b/src/Encoder.h @@ -0,0 +1,27 @@ + +#ifndef ENCODER_H +#define ENCODER_H + +#include + +#include "EmbedLayer.h" +#include "PositionEncoding.h" +#include "SubEncoder.h" +#include "Tensor.h" +#include "WenetParams.h" + +class Encoder { + private: + EncoderParams *params; + EmbedLayer *embed; + PositionEncoding *pos_enc; + SubEncoder *subencoder[12]; + LayerNorm *after_norm; + + public: + Encoder(EncoderParams *params, PositionEncoding *pos_enc); + ~Encoder(); + void forward(Tensor *din, Tensor *&dout); +}; + +#endif diff --git a/src/FeedForward.cpp b/src/FeedForward.cpp new file mode 100644 index 0000000..c2a93ec --- /dev/null +++ b/src/FeedForward.cpp @@ -0,0 +1,50 @@ + + +#include +#include +#include +#include + +#include "FeedForward.h" +#include "util.h" + + +FeedForward::FeedForward(FeedForwardParams *params, int active_type) + : params(params) +{ + if (active_type == 0) { + activate = &relu; + } else { + activate = &swish; + } +} + +FeedForward::~FeedForward() +{ +} + +void FeedForward::forward(Tensor *din) +{ + int nn = din->size[3]; + int mm = din->buff_size / nn; + int i; + Tensor tmp(mm, 2048); + + for (i = 0; i < mm; i++) { + int offset = i * 2048; + memcpy(tmp.buff + offset, params->w1_bias, 2048 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 2048, 512, 1, + din->buff, 512, params->w1_weight, 2048, 1, tmp.buff, 2048); + + activate(&tmp); + + for (i = 0; i < mm; i++) { + int offset = i * 512; + memcpy(din->buff + offset, params->w2_bias, 512 * sizeof(float)); + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mm, 512, 2048, 1, + tmp.buff, 2048, params->w2_weight, 512, 1, din->buff, 512); +} diff --git a/src/FeedForward.h b/src/FeedForward.h new file mode 100644 index 0000000..63537e7 --- /dev/null +++ b/src/FeedForward.h @@ -0,0 +1,22 @@ + + +#ifndef FEEDFORWARD_H +#define FEEDFORWARD_H + +#include + +#include "Tensor.h" +#include "WenetParams.h" + +class FeedForward { + private: + FeedForwardParams *params; + void (*activate)(Tensor *din); + + public: + FeedForward(FeedForwardParams *params, int active_type); + ~FeedForward(); + void forward(Tensor *din); +}; + +#endif diff --git a/src/LayerNorm.cpp b/src/LayerNorm.cpp new file mode 100644 index 0000000..0d8a4e4 --- /dev/null +++ b/src/LayerNorm.cpp @@ -0,0 +1,53 @@ +#include +#include + +#include "LayerNorm.h" + +LayerNorm::LayerNorm(NormParams *params, float error) + : params(params), error(error) +{ +} + +LayerNorm::~LayerNorm() +{ +} + +void LayerNorm::mean_var(float *din, float &mean, float &var) +{ + int i; + mean = 0; + for (i = 0; i < 512; i++) { + mean += din[i]; + } + mean = mean / 512; + + var = 0; + for (i = 0; i < 512; i++) { + float tmp = din[i] - mean; + var += tmp * tmp; + } + var = var / 512; +} + +void LayerNorm::norm(float *din, float mean, float var) +{ + int i; + float dd = sqrt(var + error); + for (i = 0; i < 512; i++) { + din[i] = (din[i] - mean) / dd; + din[i] = din[i] * params->weight[i] + params->bias[i]; + } +} + +void LayerNorm::forward(Tensor *din) +{ + int mm = din->buff_size / 512; + int i; + for (i = 0; i < mm; i++) { + int offset = i * 512; + float mean, var; + float *buff = din->buff + offset; + mean_var(buff, mean, var); + norm(buff, mean, var); + } +} diff --git a/src/LayerNorm.h b/src/LayerNorm.h new file mode 100644 index 0000000..9870c47 --- /dev/null +++ b/src/LayerNorm.h @@ -0,0 +1,24 @@ + + +#ifndef LAYERNORM_H +#define LAYERNORM_H + +#include + +#include "Tensor.h" +#include "WenetParams.h" + +class LayerNorm { + private: + NormParams *params; + float error; + void mean_var(float *din, float &mean, float &var); + void norm(float *din, float mean, float var); + + public: + LayerNorm(NormParams *params, float error); + ~LayerNorm(); + void forward(Tensor *din); +}; + +#endif diff --git a/src/Model.cpp b/src/Model.cpp new file mode 100644 index 0000000..d39be8d --- /dev/null +++ b/src/Model.cpp @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "CTCDecode.h" +#include "EmbedLayer.h" +#include "Encoder.h" +#include "Model.h" +#include "Tensor.h" +#include "Vocab.h" +#include "predefine_coe.h" +#include "util.h" + +using namespace std; + +void disp_params(float *din, int size) +{ + int i; + for (i = 0; i < size; i++) { + printf("%f ", din[i]); + } + printf("\n"); +} +Model::Model() +{ + loadparams("wenet_params.bin"); + + vocab = new Vocab("vocab.txt"); + pos_enc = new PositionEncoding(5000); + encoder = new Encoder(¶ms->encoder, pos_enc); + ctc = new CTCdecode(params->ctc_weight, params->ctc_bias); + decoder = new Decoder(¶ms->decoder, pos_enc); + // disp_params(params->decoder.sub_decoder[0].self_attn.linear_q_bias, 10); +} + +Model::~Model() +{ + delete encoder; + delete ctc; +} + +void hyps_process(deque hyps, Tensor *din, + Tensor *&hyps_pad, Tensor *&hyps_mask, + Tensor *&encoder_out, Tensor *&encoder_mask) +{ + int mm = hyps.size(); + + int i = 0; + int j = 0; + int max = 0; + for (auto hyps_it = hyps.begin(); hyps_it != hyps.end(); hyps_it++) { + int ll = hyps_it->prefix.size() + 1; + max = max > ll ? max : ll; + i++; + } + + hyps_pad = new Tensor(mm, max); + hyps_mask = new Tensor(mm, max); + + encoder_out = new Tensor(mm, din->size[2], din->size[3]); + + i = 0; + for (auto hyps_it = hyps.begin(); hyps_it != hyps.end(); hyps_it++) { + int ll = hyps_it->prefix.size() + 1; + hyps_pad->buff[i * max] = 5536; + hyps_mask->buff[i * max] = 1; + for (j = 1; j < max; j++) { + int ii = i * max + j; + hyps_pad->buff[ii] = j < ll ? hyps_it->prefix[j - 1] : 5536; + hyps_mask->buff[ii] = j < ll ? j + 1 : ll; + } + + int offset = i * din->buff_size; + memcpy(encoder_out->buff + offset, din->buff, + sizeof(float) * din->buff_size); + i++; + } + + encoder_mask = new Tensor(mm, max); + for (i = 0; i < encoder_mask->buff_size; i++) { + encoder_mask->buff[i] = din->size[2]; + } +} + +void rescoring(deque hyps, Tensor *decoder_out, + Tensor *scorce) +{ + + int i = 0; + scorce->zeros(); + for (auto hyps_it = hyps.begin(); hyps_it != hyps.end(); hyps_it++) { + int j; + int ll = hyps_it->prefix.size(); + float *prob; + + for (j = 0; j < ll; j++) { + int char_idx = hyps_it->prefix[j]; + prob = decoder_out->buff + + (i * decoder_out->buff_size / 10 + j * 5537); + scorce->buff[i] = scorce->buff[i] + prob[char_idx]; + } + + prob = decoder_out->buff + (i * decoder_out->buff_size / 10 + j * 5537); + scorce->buff[i] = scorce->buff[i] + prob[5536]; + i++; + } +} + +string Model::forward(Tensor *din, Tensor *dout) +{ + encoder->forward(din, dout); + deque hyps; + ctc->forward(dout, hyps); + // ctc->show_hyps(hyps); + Tensor *hyps_pad; + Tensor *hyps_mask; + Tensor *encoder_out; + Tensor *encoder_mask; + hyps_process(hyps, dout, hyps_pad, hyps_mask, encoder_out, encoder_mask); + // printf("hyps size is %ld\n", hyps.size()); + Tensor *decoder_out; + decoder->forward(hyps_pad, hyps_mask, encoder_out, encoder_mask, + decoder_out); + + Tensor scorce(1, 10); + rescoring(hyps, decoder_out, &scorce); + + int i = 0; + float max = -INFINITY; + vector result; + for (auto hyps_it = hyps.begin(); hyps_it != hyps.end(); hyps_it++) { + float tmp_scorce = 0.5 * hyps_it->prob + scorce.buff[i]; + if (tmp_scorce > max) { + max = tmp_scorce; + result = hyps_it->prefix; + } + scorce.buff[i] = tmp_scorce; + i++; + } + // cout << vocab->vector2string(result) << endl; + return vocab->vector2string(result); +} + +void Model::loadparams(const char *filename) +{ + + FILE *fp; + fp = fopen(filename, "rb"); + fseek(fp, 0, SEEK_END); + uint32_t nFileLen = ftell(fp); + fseek(fp, 0, SEEK_SET); + + params = (WenetParams *)memalign(32, nFileLen); + int n = fread(params, 1, nFileLen, fp); + + fclose(fp); +} diff --git a/src/Model.h b/src/Model.h new file mode 100644 index 0000000..fc15017 --- /dev/null +++ b/src/Model.h @@ -0,0 +1,32 @@ + +#ifndef MODEL_H +#define MODEL_H + +#include +#include + +#include "CTCDecode.h" +#include "Decoder.h" +#include "Encoder.h" +#include "Tensor.h" +#include "Vocab.h" +#include "WenetParams.h" + +class Model { + private: + WenetParams *params; + Encoder *encoder; + Decoder *decoder; + CTCdecode *ctc; + PositionEncoding *pos_enc; + Vocab *vocab; + + void loadparams(const char *filename); + + public: + Model(); + ~Model(); + string forward(Tensor *din, Tensor *dout); +}; + +#endif diff --git a/src/PositionEncoding.cpp b/src/PositionEncoding.cpp new file mode 100644 index 0000000..e8a9b67 --- /dev/null +++ b/src/PositionEncoding.cpp @@ -0,0 +1,34 @@ + +#include +#include + +#include "PositionEncoding.h" +#include "Tensor.h" +#include "predefine_coe.h" + +PositionEncoding::PositionEncoding(int max) +{ + pos_enc = new Tensor(max, 512); + float *div = (float *)pos_enc_coe_hex; + int i, j; + for (i = 0; i < max; i++) { + int offset = i * 512; + float *buff = pos_enc->buff + offset; + for (j = 0; j < 256; j++) { + float tmp = i / div[j]; + buff[2 * j] = sin(tmp); + buff[2 * j + 1] = cos(tmp); + } + } +} + +PositionEncoding::~PositionEncoding() +{ + delete pos_enc; +} + +void PositionEncoding::fetch(int size, Tensor *&out) +{ + out = new Tensor(size, 512); + memcpy(out->buff, pos_enc->buff, out->buff_size * sizeof(float)); +} diff --git a/src/PositionEncoding.h b/src/PositionEncoding.h new file mode 100644 index 0000000..93dc453 --- /dev/null +++ b/src/PositionEncoding.h @@ -0,0 +1,20 @@ + + +#ifndef POSITIONENCODING_H +#define POSITIONENCODING_H + +#include +#include "Tensor.h" + +class PositionEncoding { + private: + Tensor *pos_enc; + + public: + PositionEncoding(int max); + ~PositionEncoding(); + void fetch(int size, Tensor *&out); + +}; + +#endif diff --git a/src/SubDecoder.cpp b/src/SubDecoder.cpp new file mode 100644 index 0000000..7ce7c76 --- /dev/null +++ b/src/SubDecoder.cpp @@ -0,0 +1,38 @@ +#include "SubDecoder.h" +#include "DecSelfAttn.h" +#include "util.h" + +SubDecoder::SubDecoder(SubDecoderParams *params) : params(params) +{ + self_attn = new DecSelfAttn(¶ms->self_attn); + src_attn = new DecSelfAttn(¶ms->src_attn); + feedforward = new FeedForward(¶ms->feedward, 0); + norm1 = new LayerNorm(¶ms->norm1, 1e-12); + norm2 = new LayerNorm(¶ms->norm2, 1e-12); + norm3 = new LayerNorm(¶ms->norm3, 1e-12); +} +SubDecoder::~SubDecoder() +{ +} + +void SubDecoder::forward(Tensor *din, Tensor *din_mask, + Tensor *encoder_out, Tensor *encoder_mask) +{ + Tensor residual(din); + norm1->forward(din); + + self_attn->forward(din, din, din, din_mask, din); + + din->add(1, &residual); + residual.reload(din); + norm2->forward(din); + + src_attn->forward(din, encoder_out, encoder_out, encoder_mask, din); + // encoder_mask->disp(); + + din->add(1, &residual); + residual.reload(din); + norm3->forward(din); + feedforward->forward(din); + din->add(1, &residual); +} diff --git a/src/SubDecoder.h b/src/SubDecoder.h new file mode 100644 index 0000000..bda1b98 --- /dev/null +++ b/src/SubDecoder.h @@ -0,0 +1,34 @@ + +#ifndef SUBDECODER_H +#define SUBDECODER_H + +#include + +#include "ConvModule.h" +#include "DecSelfAttn.h" +#include "FeedForward.h" +#include "LayerNorm.h" +#include "Tensor.h" +#include "WenetParams.h" + +class SubDecoder { + private: + SubDecoderParams *params; + // ConvModule *conv_module; + // + DecSelfAttn *self_attn; + DecSelfAttn *src_attn; + FeedForward *feedforward; + // FeedForward *feedforward_macron; + LayerNorm *norm1; + LayerNorm *norm2; + LayerNorm *norm3; + + public: + SubDecoder(SubDecoderParams *params); + ~SubDecoder(); + void forward(Tensor *din, Tensor *din_mask, + Tensor *encoder_out, Tensor *encoder_mask); +}; + +#endif diff --git a/src/SubEncoder.cpp b/src/SubEncoder.cpp new file mode 100644 index 0000000..f5a5e38 --- /dev/null +++ b/src/SubEncoder.cpp @@ -0,0 +1,44 @@ + +#include "SubEncoder.h" +#include "EncSelfAttn.h" +#include "FeedForward.h" +#include "LayerNorm.h" + +SubEncoder::SubEncoder(SubEncoderParams *params) : params(params) +{ + + norm_macaron = new LayerNorm(¶ms->norm_macaron, 1e-12f); + norm_mha = new LayerNorm(¶ms->norm_mha, 1e-12f); + norm_conv = new LayerNorm(¶ms->norm_conv, 1e-12f); + norm_ff = new LayerNorm(¶ms->norm_ff, 1e-12f); + norm_final = new LayerNorm(¶ms->norm_final, 1e-12f); + feedforward_macron = new FeedForward(¶ms->feedforward_macaron, 1); + feedforward = new FeedForward(¶ms->feedforward, 1); + self_attn = new EncSelfAttn(¶ms->self_attn); + conv_module = new ConvModule(¶ms->conv_module); +} + +SubEncoder::~SubEncoder() +{ +} + +void SubEncoder::forward(Tensor *din, Tensor *pe) +{ + Tensor residual(din); + norm_macaron->forward(&residual); + feedforward_macron->forward(&residual); + din->add(0.5, &residual); + residual.reload(din); + norm_mha->forward(din); + self_attn->forward(din, din, din, pe, din); + din->add(1, &residual); + residual.reload(din); + norm_conv->forward(din); + conv_module->forward(din); + din->add(1, &residual); + residual.reload(din); + norm_ff->forward(&residual); + feedforward->forward(&residual); + din->add(0.5, &residual); + norm_final->forward(din); +} diff --git a/src/SubEncoder.h b/src/SubEncoder.h new file mode 100644 index 0000000..30a631c --- /dev/null +++ b/src/SubEncoder.h @@ -0,0 +1,34 @@ + + +#ifndef SUBENCODER_H +#define SUBENCODER_H + +#include + +#include "EncSelfAttn.h" +#include "FeedForward.h" +#include "LayerNorm.h" +#include "Tensor.h" +#include "WenetParams.h" +#include "ConvModule.h" + +class SubEncoder { + private: + SubEncoderParams *params; + ConvModule *conv_module; + EncSelfAttn *self_attn; + FeedForward *feedforward; + FeedForward *feedforward_macron; + LayerNorm *norm_ff; + LayerNorm *norm_mha; + LayerNorm *norm_macaron; + LayerNorm *norm_conv; + LayerNorm *norm_final; + + public: + SubEncoder(SubEncoderParams *params); + ~SubEncoder(); + void forward(Tensor *din, Tensor *pe); +}; + +#endif diff --git a/src/Tensor.h b/src/Tensor.h new file mode 100644 index 0000000..0964775 --- /dev/null +++ b/src/Tensor.h @@ -0,0 +1,111 @@ + + +#ifndef TENSOR_H +#define TENSOR_H + +#include +#include +#include +#include + +using namespace std; + +template class Tensor { + private: + void alloc_buff(); + void free_buff(); + + public: + T *buff; + int size[4]; + int buff_size; + Tensor(Tensor *in); + Tensor(int a); + Tensor(int a, int b); + Tensor(int a, int b, int c); + Tensor(int a, int b, int c, int d); + ~Tensor(); + void zeros(); + void shape(); + void disp(); + void add(float coe, Tensor *in); + void reload(Tensor *in); +}; + +template Tensor::Tensor(int a) : size{1, 1, 1, a} +{ + alloc_buff(); +} + +template Tensor::Tensor(int a, int b) : size{1, 1, a, b} +{ + alloc_buff(); +} + +template Tensor::Tensor(int a, int b, int c) : size{1, a, b, c} +{ + + alloc_buff(); +} + +template +Tensor::Tensor(int a, int b, int c, int d) : size{a, b, c, d} +{ + alloc_buff(); +} + +template Tensor::Tensor(Tensor *in) +{ + memcpy(size, in->size, 4 * sizeof(int)); + alloc_buff(); + memcpy(buff, in->buff, in->buff_size * sizeof(T)); +} + +template Tensor::~Tensor() +{ + free_buff(); +} + +template void Tensor::alloc_buff() +{ + buff_size = size[0] * size[1] * size[2] * size[3]; + buff = (T *)memalign(32, buff_size * sizeof(T)); +} + +template void Tensor::free_buff() +{ + free(buff); +} + +template void Tensor::zeros() +{ + memset(buff, 0, buff_size * sizeof(T)); +} + +template void Tensor::shape() +{ + printf("(%d,%d,%d,%d)\n", size[0], size[1], size[2], size[3]); +} + +template void Tensor::add(float coe, Tensor *in) +{ + int i; + for (i = 0; i < buff_size; i++) { + buff[i] = buff[i] + coe * in->buff[i]; + } +} + +template void Tensor::reload(Tensor *in) +{ + memcpy(buff, in->buff, in->buff_size * sizeof(T)); +} + +template void Tensor::disp() +{ + int i; + for (i = 0; i < buff_size; i++) { + cout << buff[i] << " "; + } + cout << endl; +} +#endif diff --git a/src/Vocab.cpp b/src/Vocab.cpp new file mode 100644 index 0000000..b1fe4f4 --- /dev/null +++ b/src/Vocab.cpp @@ -0,0 +1,39 @@ +#include "Vocab.h" + +#include +#include +#include +#include + +using namespace std; + +Vocab::Vocab(const char *filename) +{ + ifstream in(filename); + string line; + + if (in) // 有该文件 + { + while (getline(in, line)) // line中不包括每行的换行符 + { + vocab.push_back(line); + } + // cout << vocab[1719] << endl; + } else // 没有该文件 + { + cout << "no such file" << endl; + } +} +Vocab::~Vocab() +{ +} + +string Vocab::vector2string(vector in) +{ + int i; + stringstream ss; + for (auto it = in.begin(); it != in.end(); it++) + ss << vocab[*it]; + + return ss.str(); +} diff --git a/src/Vocab.h b/src/Vocab.h new file mode 100644 index 0000000..18ed446 --- /dev/null +++ b/src/Vocab.h @@ -0,0 +1,20 @@ + +#ifndef VOCAB_H +#define VOCAB_H + +#include +#include +#include +using namespace std; + +class Vocab { + private: + vector vocab; + + public: + Vocab(const char *filename); + ~Vocab(); + string vector2string(vector in); +}; + +#endif diff --git a/src/WenetParams.cpp b/src/WenetParams.cpp new file mode 100644 index 0000000..9ee8410 --- /dev/null +++ b/src/WenetParams.cpp @@ -0,0 +1,36 @@ +#include "WenetParams.h" +#include +#include +#include + +// void ModelParams::load(const char *filename) +// { + +// FILE *fp; +// fp = fopen(filename, "rb"); +// fseek(fp, 0, SEEK_END); +// uint32_t nFileLen = ftell(fp); +// fseek(fp, 0, SEEK_SET); +// printf("nFileLen is %d\n", nFileLen); + +// params = (WenetParams *)memalign(32, nFileLen); +// int n = fread(params, 1, nFileLen, fp); +// printf("n is %d\n", n); + +// fclose(fp); +// } + +// ModelParams::ModelParams(const char *filename) +// { +// load(filename); +// int i = 0; +// for (i = 0; i < 10; i++) { +// float a = params->encoder.embed.conv0_bias[i]; +// printf("%f ", a); +// } +// printf("\n"); +// } + +// ModelParams::~ModelParams() +// { +// } diff --git a/src/WenetParams.h b/src/WenetParams.h new file mode 100644 index 0000000..ded6743 --- /dev/null +++ b/src/WenetParams.h @@ -0,0 +1,109 @@ + +#ifndef WENETPARAMS_H +#define WENETPARAMS_H +// #pragma pack(1) +typedef struct { + float conv0_weight[512 * 9]; + float conv0_bias[512]; + + float conv1_weight[512 * 512 * 9]; + float conv1_bias[512]; + + float out0_weight[9728 * 512]; + float out0_bias[512]; + +} EncEmbedParams; + +typedef struct { + float linear_q_weight[512 * 512]; + float linear_q_bias[512]; + float linear_k_weight[512 * 512]; + float linear_k_bias[512]; + float linear_v_weight[512 * 512]; + float linear_v_bias[512]; + float linear_out_weight[512 * 512]; + float linear_out_bias[512]; +} SelfAttnParams; + +typedef struct { + SelfAttnParams linear0; + float linear_pos_weight[512 * 512]; + float pos_bias_u[512]; + float pos_bias_v[512]; + +} EncSelfAttnParams; + +typedef struct { + float w1_weight[512 * 2048]; + float w1_bias[2048]; + float w2_weight[2048 * 512]; + float w2_bias[512]; +} FeedForwardParams; + +typedef struct { + float weight[512]; + float bias[512]; +} NormParams; + +typedef struct { + float pointwise_conv1_weight[1024 * 512]; + float pointwise_conv1_bias[1024]; + + float depthwise_conv_weight[512 * 15]; + float depthwise_conv_bias[512]; + + float pointwise_conv2_weight[512 * 512]; + float pointwise_conv2_bias[512]; + NormParams norm; +} EncConvParams; + +typedef struct { + EncSelfAttnParams self_attn; + FeedForwardParams feedforward; + FeedForwardParams feedforward_macaron; + EncConvParams conv_module; + NormParams norm_ff; + NormParams norm_mha; + NormParams norm_macaron; + NormParams norm_conv; + NormParams norm_final; + // float concat_weight[1024 * 512]; + // float concat_bias[512]; +} SubEncoderParams; + +typedef struct { + EncEmbedParams embed; + SubEncoderParams sub_encoder[12]; + NormParams after_norm; +} EncoderParams; + +typedef struct { + SelfAttnParams self_attn; + SelfAttnParams src_attn; + FeedForwardParams feedward; + NormParams norm1; + NormParams norm2; + NormParams norm3; + // float concat_weight1[1024 * 512]; + // float concat_bias1[512]; + // float concat_weight2[1024 * 512]; + // float concat_bias2[512]; +} SubDecoderParams; + +typedef struct { + float embed_weight[5537 * 512]; + SubDecoderParams sub_decoder[6]; + NormParams after_norm; + float output_weight[5537 * 512]; + float output_bias[5537]; +} DecoderParams; + +typedef struct { + EncoderParams encoder; + float ctc_weight[512 * 5537]; + float ctc_bias[5537]; + DecoderParams decoder; +} WenetParams; + +// #pragma pack() +#endif diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..8814c0b --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include +#include +#include // for gettimeofday() + +#include "Audio.h" +#include "Model.h" +#include +// #include "WenetParams.h" +#include "Tensor.h" +#include "Vocab.h" +#include "util.h" + +using namespace std; + +int main(int argc, char *argv[]) +{ + float *audio_in; + Audio cc(argv[1]); + Model mm; + Tensor *dout; + + struct timeval start, end; + gettimeofday(&start, NULL); + string result = mm.forward(cc.fbank_feature, dout); + gettimeofday(&end, NULL); + + long seconds = (end.tv_sec - start.tv_sec); + long micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); + cout << "result: \"" << result << "\"" << endl; + + printf("inference time is %lf s.\n", (double)micros / 1000000); +} diff --git a/src/predefine_coe.h b/src/predefine_coe.h new file mode 100644 index 0000000..807efd8 --- /dev/null +++ b/src/predefine_coe.h @@ -0,0 +1,243 @@ +#ifndef PREDEFINE_COE_H +#define PREDEFINE_COE_H + +#include + +const int32_t melcoe_hex[] = { + + 0x3f01050c, 0x3e0afb11, 0x3f5d413c, 0x3f547fd0, 0x3e2e00c1, 0x3f132970, + 0x3ed9ad21, 0x3ebb8bb9, 0x3f223a24, 0x3e4de6f8, 0x3f4c8642, 0x3d9c0424, + 0x3f6c7f7c, 0x3f7d295a, 0x3c35a961, 0x3f6fd497, 0x3d815b45, 0x3f6af197, + 0x3da87344, 0x3f6dfce9, 0x3d9018b9, 0x3f787ebc, 0x3d2098fe, 0x3cf02873, + 0x3f75f670, 0x3e08e423, 0x3f5dc6f7, 0x3e8161eb, 0x3f3f4f0b, 0x3eca38e2, + 0x3f1ae38f, 0x3f0f2d23, 0x3ee1a5ba, 0x3f3e9a98, 0x3e82cad1, 0x3f7321ac, + 0x3e321028, 0x3d4de548, 0x3f537bf6, 0x3ed50f76, 0x3f157845, 0x3f2cf6bc, + 0x3ea61288, 0x3f739ea7, 0x3e794186, 0x3d461590, 0x3f41af9f, 0x3f0cdfd4, + 0x3ee64058, 0x3f5f23aa, 0x3e53d467, 0x3e037156, 0x3f4b0ae6, 0x3f0e2fac, + 0x3ee3a0a8, 0x3f6ab111, 0x3e94b1ed, 0x3daa7774, 0x3f35a70a, 0x3f2d08dc, + 0x3d951fb4, 0x3ea5ee48, 0x3f6d5c09, 0x3ef61e1a, 0x3f04f0f3, 0x3f66305c, + 0x3ea7def9, 0x3dce7d20, 0x3f2c1083, 0x3f44354b, 0x3e5baf49, 0x3e6f2ad2, + 0x3f49142e, 0x3f2bfe35, 0x3e0d627b, 0x3ea80396, 0x3f5ca761, 0x3f1ce830, + 0x3dc4d786, 0x3ec62fa0, 0x3f67650f, 0x3f165fc0, 0x3db1323f, 0x3ed34080, + 0x3f69d9b8, 0x3f17def1, 0x3ddbd6b6, 0x3ed0421e, 0x3f648529, 0x3f20ebbd, + 0x3e20901a, 0x3ebe2886, 0x3f57dbf9, 0x3f3116ac, 0x3e6edcc6, 0x3e9dd2a9, + 0x3f4448ce, 0x3f47f9a3, 0x3eaba511, 0x3e601974, 0x3f2a2d77, 0x3f6536e2, + 0x3eec3842, 0x3d0781f6, 0x3dd648ed, 0x3f09e3df, 0x3f7787e1, 0x3f1c411f, + 0x3e45b702, 0x3ec77dc2, 0x3f4e9240, 0x3f47f500, 0x3ebf9c61, 0x3e602c00, + 0x3f2031d0, 0x3f78f0f7, 0x3f135547, 0x3e3bcd78, 0x3ce1e12a, 0x3ed95573, + 0x3f510ca2, 0x3f4bc3c2, 0x3ed37e77, 0x3d0ded37, 0x3e50f0f8, 0x3f1640c5, + 0x3f77212d, 0x3f291bd1, 0x3e94df6c, 0x3eadc85e, 0x3f35904a, 0x3f6cd43b, + 0x3f104351, 0x3e52dc63, 0x3d995e26, 0x3edf795f, 0x3f4b48e7, 0x3f5a29e7, + 0x3f00963d, 0x3e1fdb2f, 0x3e175865, 0x3efed385, 0x3f580934, 0x3f50466d, + 0x3ef30046, 0x3e0e7c6b, 0x3e3ee64e, 0x3f067fdd, 0x3f5c60e5, 0x3f4e9ea4, + 0x3ef4f46a, 0x3e1cb596, 0x3e45856f, 0x3f0585cb, 0x3f58d29b, 0x3f54b3ef, + 0x3f0309ad, 0x3e48aa5b, 0x3e2d3042, 0x3ef9eca6, 0x3f4dd569, 0x3f6212c4, + 0x3f12be68, 0x3e8853a3, 0x3def69e0, 0x3eda8330, 0x3f3bd62e, 0x3f76516a, + 0x3f2931b5, 0x3eb98e9b, 0x3d88773c, 0x3d1ae95c, 0x3ead9c96, 0x3f2338b2, + 0x3f6ef119, 0x3f46054d, 0x3ef74eba, 0x3e47c83a, 0x3e67eace, 0x3f0458a3, + 0x3f4e0df1, 0x3f68e26b, 0x3f207590, 0x3eb1515d, 0x3d8bc852, 0x3db8eca9, + 0x3ebf14e0, 0x3f275751, 0x3f6e86f6, 0x3f4ae3f8, 0x3f04e6de, 0x3e7dfcce, + 0x3e547020, 0x3ef63244, 0x3f4080cd, 0x3f7aaa80, 0x3f366659, 0x3ee560cb, + 0x3e3e1967, 0x3caab00e, 0x3e93334e, 0x3f0d4f9a, 0x3f5079a6, 0x3f6ce5f8, + 0x3f2acd10, 0x3ed272ff, 0x3e20a4c5, 0x3d98d042, 0x3eaa65e0, 0x3f16c680, + 0x3f57d6cf, 0x3f679a1b, 0x3f278a40, 0x3ecfef5c, 0x3e2381fd, 0x3dc32f28, + 0x3eb0eb80, 0x3f180852, 0x3f571f81, 0x3f6a42d8, 0x3f2c1ce8, 0x3edcd9d1, + 0x3e44c475, 0x3dade93f, 0x3ea7c630, 0x3f119318, 0x3f4ecee3, 0x3f7467d4, + 0x3f380f62, 0x3ef84c54, 0x3e815525, 0x3cb361d7, 0x3d3982c8, 0x3e8fe13b, + 0x3f03d9d6, 0x3f3f556d, 0x3f7a64f1, 0x3f4af618, 0x3f10ba30, 0x3eadcbc5, + 0x3debbe02, 0x3e5427a0, 0x3ede8b9f, 0x3f291a1d, 0x3f628840, 0x3f646e63, + 0x3f2bc86b, 0x3ee70902, 0x3e6e854e, 0x3c83b300, 0x3ddc8cea, 0x3ea86f2a, + 0x3f0c7b7f, 0x3f445eac, 0x3f7be268, 0x3f4cf80b, 0x3f162f6e, 0x3ebf8516, + 0x3e26c0c2, 0x3e4c1fd5, 0x3ed3a124, 0x3f203d75, 0x3f564fd0, 0x3f73f733, + 0x3f3e966d, 0x3f098cbf, 0x3ea9b21c, 0x3e01e917, 0x3d408cd1, 0x3e82d326, + 0x3eece682, 0x3f2b26f2, 0x3f5f85ba, 0x3f6c6f56, 0x3f38b733, 0x3f0550d9, + 0x3ea47689, 0x3dfbabd7, 0x3d9c8552, 0x3e8e919a, 0x3ef55e4f, 0x3f2dc4bb, + 0x3f608a85, 0x3f6cfe84, 0x3f3ad56c, 0x3f08f945, 0x3eaed247, 0x3e189086, + 0x3d980be2, 0x3e8a5528, 0x3eee0d76, 0x3f2896dc, 0x3f59dbde, 0x3f75295d, + 0x3f4477f6, 0x3f140f14, 0x3ec7dbbd, 0x3e504e0f, 0x3c8fe67e, 0x3d2d6a38, + 0x3e6e2028, 0x3ed7e1d9, 0x3f1c1221, 0x3f4bec7c, 0x3f7b80cc, 0x3f553023, + 0x3f262589, 0x3eeebd40, 0x3e91b54d, 0x3dd4c6f8, 0x3e2b3f74, 0x3eb3b4ef, + 0x3f08a160, 0x3f372559, 0x3f656721, 0x3f6c988d, 0x3f3ed8f9, 0x3f11596d, + 0x3ec83270, 0x3e5c5ea4, 0x3d254149, 0x3d9b3b97, 0x3e824e0e, 0x3edd4d25, + 0x3f1be6c8, 0x3f48e857, 0x3f75abeb, 0x3f5dcdd1, 0x3f318436, 0x3f0576a0, + 0x3eb348db, 0x3e3833fb, 0x3c2bedc9, 0x3e08c8be, 0x3e9cf794, 0x3ef512c0, + 0x3f265b92, 0x3f51f301, 0x3f7d5049, 0x3f578bfc, 0x3f2ca136, 0x3f01eecf, + 0x3eaee867, 0x3e34c34c, 0x3c490794, 0x3e21d00f, 0x3ea6bd94, 0x3efc2262, + 0x3f288bcc, 0x3f52cf2d, 0x3f7cdbe2, 0x3f594d89, 0x3f2fac87, 0x3f064092, + 0x3eba1245, 0x3e5016cd, 0x3d335c27, 0x3e1ac9dd, 0x3ea0a6f1, 0x3ef37edc, + 0x3f22f6de, 0x3f4bfa4d, 0x3f74ca3e, 0x3f6298cf, 0x3f3a2e5b, 0x3f11f5e8, + 0x3ed3ddf9, 0x3e84323c, 0x3dd39eaa, 0x3deb3986, 0x3e8ba34a, 0x3edc142f, + 0x3f161103, 0x3f3de6e2, 0x3f658c2b, 0x3f72feac, 0x3f4bb92e, 0x3f24a2e9, + 0x3efb76d9, 0x3eae048f, 0x3e41dc34, 0x3d219509, 0x3d50153e, 0x3e511b46, + 0x3eb6ba2d, 0x3f024494, 0x3f28fdb9, 0x3f4f88f3, 0x3f75e6af, 0x3f63e8a7, + 0x3f3de4a8, 0x3f180cea, 0x3ee4c20e, 0x3e99c134, 0x3e1e2cfc, 0x3c1824f4, + 0x3de0bac6, 0x3e8436b1, 0x3ecfe62d, 0x3f0d9ef9, 0x3f331f66, 0x3f5874c1, + 0x3f7d9f6c, 0x3f5d6037, 0x3f3889c9, 0x3f13dcea, 0x3edeb27d, 0x3e95fcd3, + 0x3e1b303e, 0x3c3075cb, 0x3e0a7f24, 0x3e8eec6e, 0x3ed8462b, 0x3f10a6c1, + 0x3f350197, 0x3f5933f1, 0x3f7d3e29, 0x3f5edf68, 0x3f3b246a, 0x3f179088, + 0x3ee846d8, 0x3ea1b983, 0x3e36f0d8, 0x3d2c1773, 0x3e048260, 0x3e89b72b, + 0x3ed0def0, 0x3f0bdc94, 0x3f2f233e, 0x3f5243ca, 0x3f753e89, 0x3f67ec34, + 0x3f453c1d, 0x3f22b0e2, 0x3f004a36, 0x3ebc0f98, 0x3e6fa55d, 0x3dcf7467, + 0x3dc09e5f, 0x3e6b0f8d, 0x3eba9e3c, 0x3eff6b94, 0x3f21f834, 0x3f4416a9, + 0x3f661173, 0x3f781723, 0x3f5662cf, 0x3f34d14a, 0x3f13624c, 0x3ee42b1a, + 0x3ea1d591, 0x3e3f86e3, 0x3d6fa1a1, 0x3cfd1ba9, 0x3e2674c3, 0x3e965d6c, + 0x3ed93b69, 0x3f0dea73, 0x3f2f1538, 0x3f501e47, 0x3f7105e6, 0x3f6e33a9, + 0x3f4d8e22, 0x3f2d0944, 0x3f0ca4cd, 0x3ed8c0fd, 0x3e98782f, 0x3e30dd66, + 0x3d452061, 0x3d8e62bc, 0x3e49c779, 0x3ea5ed78, 0x3ee6b665, 0x3f139f81, + 0x3f33c3e8, 0x3f53c8a7, 0x3f73adfa, 0x3f6c8be0, 0x3f4ce4ab, 0x3f2d5c2a, + 0x3f0df223, 0x3edd4cb5, 0x3e9ef12d, 0x3e41a276, 0x3d8bb1ba, 0x3d9ba0ff, + 0x3e4c6d54, 0x3ea547ab, 0x3ee41bba, 0x3f1159a6, 0x3f30876a, 0x3f4f9762, + 0x3f6e89c9, 0x3f72a12b, 0x3f53e942, 0x3f354e46, 0x3f16cffe, 0x3ef0dc6f, + 0x3eb45177, 0x3e6ffd59, 0x3def8e9c + +}; + +const int32_t window_hex[] = { + 0x00000000, 0x398b03f6, 0x3a61d1c5, 0x3ae0ee32, 0x3b37623a, 0x3b85f871, + 0x3bb69d19, 0x3bed453b, 0x3c14d40b, 0x3c35c45b, 0x3c59595d, 0x3c7f7c1d, + 0x3c940c13, 0x3ca98d81, 0x3cc039eb, 0x3cd8098d, 0x3cf0f52e, 0x3d057b06, + 0x3d1302e6, 0x3d210f33, 0x3d2f9d0e, 0x3d3ea9ba, 0x3d4e3293, 0x3d5e3510, + 0x3d6eaebd, 0x3d7f9d38, 0x3d887f19, 0x3d9167b5, 0x3d9a8756, 0x3da3dce9, + 0x3dad675d, 0x3db725ab, 0x3dc116cc, 0x3dcb39bf, 0x3dd58d86, 0x3de01126, + 0x3deac3a7, 0x3df5a413, 0x3e0058bb, 0x3e05f571, 0x3e0ba7b2, 0x3e116f08, + 0x3e174afe, 0x3e1d3b1c, 0x3e233ef0, 0x3e295605, 0x3e2f7fe7, 0x3e35bc23, + 0x3e3c0a46, 0x3e4269de, 0x3e48da79, 0x3e4f5ba5, 0x3e55ecf2, 0x3e5c8ded, + 0x3e633e26, 0x3e69fd2c, 0x3e70ca8f, 0x3e77a5de, 0x3e7e8eaa, 0x3e82c241, + 0x3e86437c, 0x3e89cacd, 0x3e8d57fc, 0x3e90ead3, 0x3e948319, 0x3e982097, + 0x3e9bc316, 0x3e9f6a5d, 0x3ea31636, 0x3ea6c66a, 0x3eaa7ac0, 0x3eae3303, + 0x3eb1eefa, 0x3eb5ae6f, 0x3eb9712a, 0x3ebd36f6, 0x3ec0ff9b, 0x3ec4cae2, + 0x3ec89895, 0x3ecc687d, 0x3ed03a64, 0x3ed40e13, 0x3ed7e354, 0x3edbb9f2, + 0x3edf91b5, 0x3ee36a69, 0x3ee743d7, 0x3eeb1dca, 0x3eeef80c, 0x3ef2d267, + 0x3ef6aca8, 0x3efa8698, 0x3efe6002, 0x3f011c59, 0x3f03083a, 0x3f04f389, + 0x3f06de2d, 0x3f08c80b, 0x3f0ab10a, 0x3f0c990f, 0x3f0e8001, 0x3f1065c6, + 0x3f124a45, 0x3f142d65, 0x3f160f0c, 0x3f17ef21, 0x3f19cd8b, 0x3f1baa32, + 0x3f1d84fb, 0x3f1f5dd0, 0x3f213498, 0x3f230939, 0x3f24db9d, 0x3f26abaa, + 0x3f28794a, 0x3f2a4464, 0x3f2c0ce1, 0x3f2dd2a9, 0x3f2f95a6, 0x3f3155bf, + 0x3f3312e0, 0x3f34ccef, 0x3f3683d8, 0x3f383784, 0x3f39e7dd, 0x3f3b94cc, + 0x3f3d3e3c, 0x3f3ee418, 0x3f40864a, 0x3f4224bd, 0x3f43bf5c, 0x3f455613, + 0x3f46e8cc, 0x3f487774, 0x3f4a01f6, 0x3f4b883f, 0x3f4d0a3b, 0x3f4e87d6, + 0x3f5000fe, 0x3f5175a0, 0x3f52e5a9, 0x3f545106, 0x3f55b7a5, 0x3f571975, + 0x3f587664, 0x3f59ce60, 0x3f5b2158, 0x3f5c6f3b, 0x3f5db7f9, 0x3f5efb80, + 0x3f6039c2, 0x3f6172af, 0x3f62a636, 0x3f63d448, 0x3f64fcd6, 0x3f661fd3, + 0x3f673d2e, 0x3f6854db, 0x3f6966ca, 0x3f6a72ef, 0x3f6b793d, 0x3f6c79a5, + 0x3f6d741d, 0x3f6e6896, 0x3f6f5706, 0x3f703f5f, 0x3f712198, 0x3f71fda4, + 0x3f72d379, 0x3f73a30c, 0x3f746c52, 0x3f752f43, 0x3f75ebd4, 0x3f76a1fc, + 0x3f7751b2, 0x3f77faee, 0x3f789da6, 0x3f7939d4, 0x3f79cf6e, 0x3f7a5e6f, + 0x3f7ae6cf, 0x3f7b6886, 0x3f7be38f, 0x3f7c57e4, 0x3f7cc57f, 0x3f7d2c5b, + 0x3f7d8c72, 0x3f7de5bf, 0x3f7e3840, 0x3f7e83ee, 0x3f7ec8c7, 0x3f7f06c7, + 0x3f7f3deb, 0x3f7f6e31, 0x3f7f9795, 0x3f7fba17, 0x3f7fd5b4, 0x3f7fea6b, + 0x3f7ff83b, 0x3f7fff23, 0x3f7fff23, 0x3f7ff83b, 0x3f7fea6b, 0x3f7fd5b4, + 0x3f7fba17, 0x3f7f9795, 0x3f7f6e31, 0x3f7f3deb, 0x3f7f06c7, 0x3f7ec8c7, + 0x3f7e83ee, 0x3f7e3840, 0x3f7de5bf, 0x3f7d8c72, 0x3f7d2c5b, 0x3f7cc57f, + 0x3f7c57e4, 0x3f7be38f, 0x3f7b6886, 0x3f7ae6cf, 0x3f7a5e6f, 0x3f79cf6e, + 0x3f7939d4, 0x3f789da6, 0x3f77faee, 0x3f7751b2, 0x3f76a1fc, 0x3f75ebd4, + 0x3f752f43, 0x3f746c52, 0x3f73a30c, 0x3f72d379, 0x3f71fda4, 0x3f712198, + 0x3f703f5f, 0x3f6f5706, 0x3f6e6896, 0x3f6d741d, 0x3f6c79a5, 0x3f6b793d, + 0x3f6a72ef, 0x3f6966ca, 0x3f6854db, 0x3f673d2e, 0x3f661fd3, 0x3f64fcd6, + 0x3f63d448, 0x3f62a636, 0x3f6172af, 0x3f6039c2, 0x3f5efb80, 0x3f5db7f9, + 0x3f5c6f3b, 0x3f5b2158, 0x3f59ce60, 0x3f587664, 0x3f571975, 0x3f55b7a5, + 0x3f545106, 0x3f52e5a9, 0x3f5175a0, 0x3f5000fe, 0x3f4e87d6, 0x3f4d0a3b, + 0x3f4b883f, 0x3f4a01f6, 0x3f487774, 0x3f46e8cc, 0x3f455613, 0x3f43bf5c, + 0x3f4224bd, 0x3f40864a, 0x3f3ee418, 0x3f3d3e3c, 0x3f3b94cc, 0x3f39e7dd, + 0x3f383784, 0x3f3683d8, 0x3f34ccef, 0x3f3312e0, 0x3f3155bf, 0x3f2f95a6, + 0x3f2dd2a9, 0x3f2c0ce1, 0x3f2a4464, 0x3f28794a, 0x3f26abaa, 0x3f24db9d, + 0x3f230939, 0x3f213498, 0x3f1f5dd0, 0x3f1d84fb, 0x3f1baa32, 0x3f19cd8b, + 0x3f17ef21, 0x3f160f0c, 0x3f142d65, 0x3f124a45, 0x3f1065c6, 0x3f0e8001, + 0x3f0c990f, 0x3f0ab10a, 0x3f08c80b, 0x3f06de2d, 0x3f04f389, 0x3f03083a, + 0x3f011c59, 0x3efe6002, 0x3efa8698, 0x3ef6aca8, 0x3ef2d267, 0x3eeef80c, + 0x3eeb1dca, 0x3ee743d7, 0x3ee36a69, 0x3edf91b5, 0x3edbb9f2, 0x3ed7e354, + 0x3ed40e13, 0x3ed03a64, 0x3ecc687d, 0x3ec89895, 0x3ec4cae2, 0x3ec0ff9b, + 0x3ebd36f6, 0x3eb9712a, 0x3eb5ae6f, 0x3eb1eefa, 0x3eae3303, 0x3eaa7ac0, + 0x3ea6c66a, 0x3ea31636, 0x3e9f6a5d, 0x3e9bc316, 0x3e982097, 0x3e948319, + 0x3e90ead3, 0x3e8d57fc, 0x3e89cacd, 0x3e86437c, 0x3e82c241, 0x3e7e8eaa, + 0x3e77a5de, 0x3e70ca8f, 0x3e69fd2c, 0x3e633e26, 0x3e5c8ded, 0x3e55ecf2, + 0x3e4f5ba5, 0x3e48da79, 0x3e4269de, 0x3e3c0a46, 0x3e35bc23, 0x3e2f7fe7, + 0x3e295605, 0x3e233ef0, 0x3e1d3b1c, 0x3e174afe, 0x3e116f08, 0x3e0ba7b2, + 0x3e05f571, 0x3e0058bb, 0x3df5a413, 0x3deac3a7, 0x3de01126, 0x3dd58d86, + 0x3dcb39bf, 0x3dc116cc, 0x3db725ab, 0x3dad675d, 0x3da3dce9, 0x3d9a8756, + 0x3d9167b5, 0x3d887f19, 0x3d7f9d38, 0x3d6eaebd, 0x3d5e3510, 0x3d4e3293, + 0x3d3ea9ba, 0x3d2f9d0e, 0x3d210f33, 0x3d1302e6, 0x3d057b06, 0x3cf0f52e, + 0x3cd8098d, 0x3cc039eb, 0x3ca98d81, 0x3c940c13, 0x3c7f7c1d, 0x3c59595d, + 0x3c35c45b, 0x3c14d40b, 0x3bed453b, 0x3bb69d19, 0x3b85f871, 0x3b37623a, + 0x3ae0ee32, 0x3a61d1c5, 0x398b03f6, 0x00000000 + +}; + +const int global_cmvn_mean_hex[] = { + 0x413d6566, 0x4147923f, 0x4156ab15, 0x41613d12, 0x416b155b, 0x41722783, + 0x4176cd05, 0x4178532a, 0x417aa3c3, 0x417aed19, 0x417d4d2c, 0x417e6abb, + 0x41805848, 0x418122ab, 0x41812b23, 0x418161a8, 0x41810ef9, 0x4180863a, + 0x41815d8f, 0x417ff8b2, 0x417de2aa, 0x4180a5f2, 0x417e8bd1, 0x418041ac, + 0x417f2d60, 0x4180487f, 0x417eb835, 0x418018d8, 0x417ef8c1, 0x417ea302, + 0x417f30cf, 0x417ea0bb, 0x417ebac2, 0x417faab6, 0x417fca4d, 0x41805e45, + 0x4180e308, 0x4180ef3e, 0x418109fc, 0x4180afa3, 0x418113e2, 0x4180c915, + 0x41819f86, 0x418190bf, 0x418220bd, 0x4182f2e5, 0x4183e1c7, 0x41843eec, + 0x4184b066, 0x418574db, 0x41852611, 0x4184fc81, 0x41851b2a, 0x4185a1c7, + 0x41861152, 0x41868c28, 0x41871930, 0x41871f83, 0x41868893, 0x4185d919, + 0x4185664b, 0x418480a6, 0x41840e3a, 0x41836ace, 0x4182b217, 0x4181cb79, + 0x4180fb13, 0x418098b9, 0x41805ded, 0x417ff69a, 0x417f49bd, 0x417ecef8, + 0x417e286c, 0x417d9135, 0x417cfff4, 0x417ca8f7, 0x417b2e8f, 0x41773788, + 0x4170b095, 0x4167417f}; + +const int global_cmvn_std_hex[] = { + 0x4040335e, 0x405235d3, 0x40589be4, 0x4054261f, 0x40544ba2, 0x40575418, + 0x405b6528, 0x40617999, 0x40605fcf, 0x405c9c6d, 0x40590796, 0x405899fc, + 0x405810b8, 0x40587c40, 0x40592b5e, 0x4057fb12, 0x4057028b, 0x405515d7, + 0x4053d714, 0x405418c7, 0x405536bc, 0x4052f54e, 0x4052d382, 0x4051201d, + 0x4050a8d2, 0x4050857f, 0x404ffe85, 0x4050a0da, 0x40517a8a, 0x40508862, + 0x40504f68, 0x404f3159, 0x404f0930, 0x404e8a2e, 0x404e7383, 0x404eb185, + 0x404edaa9, 0x404efed2, 0x404ea8f4, 0x404f6d0d, 0x404ee9d9, 0x404f4cca, + 0x404fb13f, 0x405051c5, 0x40503f5e, 0x4050df6e, 0x4052974e, 0x4053d421, + 0x40544d48, 0x40544ec8, 0x40550e57, 0x40558287, 0x4055d122, 0x4056b22a, + 0x4058ea5c, 0x405acbc3, 0x405a89e7, 0x405a88ed, 0x405afadb, 0x405a1c60, + 0x405a6f46, 0x405b0a24, 0x405b5f44, 0x405cc0a9, 0x405d984b, 0x405ef9b8, + 0x4061178a, 0x406262bf, 0x40644904, 0x40660b20, 0x4067f7f1, 0x406a35e5, + 0x406c1e97, 0x406e16a9, 0x406eadb1, 0x406d0cba, 0x406d9ca0, 0x406f5a14, + 0x406e84a7, 0x406cd985}; + +const int pos_enc_coe_hex[] = { + 0x3f800000, 0x3f84b063, 0x3f898cc0, 0x3f8e96b2, 0x3f93cfe5, 0x3f993a15, + 0x3f9ed70c, 0x3fa4a8a8, 0x3faab0d5, 0x3fb0f193, 0x3fb76cf5, 0x3fbe2520, + 0x3fc51c50, 0x3fcc54d2, 0x3fd3d10c, 0x3fdb9378, 0x3fe39ea9, 0x3febf549, + 0x3ff49a1b, 0x3ffd8ffe, 0x40036cf4, 0x40083d78, 0x400d3b22, 0x40126799, + 0x4017c496, 0x401d53df, 0x4023174b, 0x402910c4, 0x402f4244, 0x4035adda, + 0x403c55a4, 0x40433bd9, 0x404a62c2, 0x4051ccbd, 0x40597c3f, 0x406173d4, + 0x4069b621, 0x407245e2, 0x407b25ed, 0x40822c9a, 0x4086f161, 0x408be2e0, + 0x409102bc, 0x409652a6, 0x409bd461, 0x40a189c1, 0x40a774aa, 0x40ad9711, + 0x40b3f300, 0x40ba8a92, 0x40c15ff6, 0x40c8756f, 0x40cfcd58, 0x40d76a1e, + 0x40df4e48, 0x40e77c73, 0x40eff755, 0x40f8c1be, 0x4100ef4c, 0x4105a873, + 0x410a8de6, 0x410fa144, 0x4114e43b, 0x411a588a, 0x41200000, 0x4125dc7c, + 0x412beff0, 0x41323c5f, 0x4138c3df, 0x413f889a, 0x41468cd0, 0x414dd2d2, + 0x41555d0a, 0x415d2df7, 0x41654832, 0x416dae69, 0x41766364, 0x417f6a07, + 0x418462a7, 0x41893c2b, 0x418e432a, 0x4193794e, 0x4198e051, 0x419e79ff, + 0x41a44831, 0x41aa4cd6, 0x41b089ea, 0x41b70180, 0x41bdb5bc, 0x41c4a8d7, + 0x41cbdd1e, 0x41d354f5, 0x41db12d6, 0x41e31950, 0x41eb6b0d, 0x41f40ad0, + 0x41fcfb72, 0x42031ff6, 0x4207eda7, 0x420ce865, 0x421211d5, 0x42176bad, + 0x421cf7b4, 0x4222b7c0, 0x4228adb9, 0x422edb98, 0x4235436b, 0x423be74f, + 0x4242c979, 0x4249ec31, 0x425151d4, 0x4258fcd6, 0x4260efc0, 0x42692d37, + 0x4271b7f3, 0x427a92cb, 0x4281e057, 0x4286a253, 0x428b90ed, 0x4290adc8, + 0x4295fa95, 0x429b7917, 0x42a12b1f, 0x42a71290, 0x42ad3160, 0x42b38995, + 0x42ba1d4a, 0x42c0eead, 0x42c80000, 0x42cf539b, 0x42d6ebec, 0x42decb76, + 0x42e6f4d6, 0x42ef6ac1, 0x42f83003, 0x4300a3c3, 0x43055a26, 0x430a3cbb, + 0x430f4d1f, 0x43148d01, 0x4319fe1e, 0x431fa244, 0x43257b51, 0x432b8b36, + 0x4331d3f4, 0x433857a1, 0x433f1865, 0x4346187e, 0x434d5a3e, 0x4354e00b, + 0x435cac64, 0x4364c1e0, 0x436d232b, 0x4375d30c, 0x437ed466, 0x43841519, + 0x4388ebc5, 0x438defd2, 0x439322e8, 0x439886c2, 0x439e1d27, 0x43a3e7f3, + 0x43a9e911, 0x43b0227e, 0x43b6964a, 0x43bd4698, 0x43c435a1, 0x43cb65b0, + 0x43d2d927, 0x43da927e, 0x43e29445, 0x43eae123, 0x43f37bd8, 0x43fc673e, + 0x4402d325, 0x44079e06, 0x440c95d8, 0x4411bc42, 0x441712f8, 0x441c9bbf, + 0x4422586d, 0x44284ae8, 0x442e7528, 0x4434d93a, 0x443b793b, 0x4442575d, + 0x444975e6, 0x4450d734, 0x44587db7, 0x44606bfa, 0x4468a49c, 0x44712a58, + 0x447a0000, 0x44819441, 0x44865373, 0x448b3f2a, 0x44905906, 0x4495a2b9, + 0x449b1e02, 0x44a0ccb4, 0x44a6b0b0, 0x44accbe9, 0x44b32067, 0x44b9b042, + 0x44c07da6, 0x44c78ad5, 0x44ceda26, 0x44d66e03, 0x44de48f1, 0x44e66d89, + 0x44eede7f, 0x44f79e9e, 0x45005867, 0x45050c07, 0x4509ebbf, 0x450ef92c, + 0x451435fb, 0x4519a3e8, 0x451f44bf, 0x45251a60, 0x452b26b7, 0x45316bc7, + 0x4537eba3, 0x453ea872, 0x4545a471, 0x454ce1f0, 0x45546355, 0x455c2b1d, + 0x45643bdc, 0x456c983e, 0x45754309, 0x457e3f1c, 0x4583c7b8, 0x45889b8f, + 0x458d9cab, 0x4592ccb6, 0x45982d67, 0x459dc087, 0x45a387ee, 0x45a98587, + 0x45afbb4e, 0x45b62b53, 0x45bcd7b6, 0x45c3c2af, 0x45caee88, 0x45d25da1, + 0x45da1272, 0x45e20f88, 0x45ea5789, 0x45f2ed34, 0x45fbd360, 0x46028680, + 0x46074e93, 0x460c437c, 0x461166e2, 0x4616ba77 +}; +#endif diff --git a/src/util.cpp b/src/util.cpp new file mode 100644 index 0000000..59816f8 --- /dev/null +++ b/src/util.cpp @@ -0,0 +1,71 @@ + +#include +#include +#include + +#include "Tensor.h" +void SaveDataFile(const char *filename, void *data, uint32_t len) +{ + FILE *fp; + fp = fopen(filename, "wb+"); + fwrite(data, 1, len, fp); + fclose(fp); +} + +void relu(Tensor *din) +{ + int i; + for (i = 0; i < din->buff_size; i++) { + float val = din->buff[i]; + din->buff[i] = val < 0 ? 0 : val; + } +} + +void swish(Tensor *din) +{ + int i; + for (i = 0; i < din->buff_size; i++) { + float val = din->buff[i]; + din->buff[i] = val / (1 + exp(-val)); + } +} + +void softmax(float *din, int mask, int len) +{ + float *tmp = (float *)malloc(mask * sizeof(float)); + int i; + float sum = 0; + float max = -INFINITY; + + for (i = 0; i < mask; i++) { + max = max < din[i] ? din[i] : max; + } + max = max * 0.9; + + for (i = 0; i < mask; i++) { + tmp[i] = exp(din[i] - max); + sum += tmp[i]; + } + for (i = 0; i < mask; i++) { + din[i] = tmp[i] / sum; + } + free(tmp); + for (i = mask; i < len; i++) { + din[i] = 0; + } +} + +void log_softmax(float *din, int len) +{ + float *tmp = (float *)malloc(len * sizeof(float)); + int i; + float sum = 0; + for (i = 0; i < len; i++) { + tmp[i] = exp(din[i]); + sum += tmp[i]; + } + for (i = 0; i < len; i++) { + din[i] = log(tmp[i] / sum); + } + free(tmp); +} diff --git a/src/util.h b/src/util.h new file mode 100644 index 0000000..7cfd354 --- /dev/null +++ b/src/util.h @@ -0,0 +1,15 @@ + + +#ifndef UTIL_H +#define UTIL_H +#include "Tensor.h" +extern void SaveDataFile(const char *filename, void *data, uint32_t len); + +extern void relu(Tensor *din); +extern void swish(Tensor *din); + +extern void softmax(float *din, int mask, int len); + +extern void log_softmax(float *din, int len); + +#endif diff --git a/vocab.txt b/vocab.txt new file mode 100644 index 0000000..890df44 --- /dev/null +++ b/vocab.txt @@ -0,0 +1,5537 @@ + + +▁ +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丢 +两 +严 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丽 +举 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +亓 +五 +井 +亘 +亚 +些 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +仡 +代 +令 +以 +仨 +仪 +们 +仰 +仲 +仵 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +众 +优 +伙 +会 +伞 +伟 +传 +伢 +伤 +伥 +伦 +伧 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +位 +低 +住 +佐 +佑 +体 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佣 +佤 +佩 +佬 +佯 +佰 +佳 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +侉 +例 +侍 +侏 +侗 +供 +依 +侠 +侣 +侥 +侦 +侧 +侨 +侩 +侬 +侮 +侯 +侵 +便 +促 +俄 +俅 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +信 +俣 +俦 +俨 +俩 +俪 +俭 +修 +俯 +俱 +俳 +俶 +俸 +俺 +俾 +個 +倌 +倍 +倏 +倒 +倔 +倘 +候 +倚 +倜 +借 +倡 +倦 +倨 +倩 +倪 +倭 +债 +值 +倾 +偃 +假 +偈 +偌 +偎 +偏 +偕 +做 +停 +健 +偶 +偷 +偻 +偿 +傀 +傅 +傈 +傍 +傣 +傥 +储 +催 +傲 +傻 +像 +僖 +僚 +僧 +僭 +僮 +僳 +僵 +僻 +儆 +儋 +儒 +儡 +儿 +兀 +允 +元 +兄 +充 +兆 +先 +光 +克 +免 +兑 +兔 +兖 +党 +兜 +兢 +入 +全 +八 +公 +六 +兮 +兰 +共 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +冈 +冉 +册 +再 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +净 +凄 +准 +凇 +凉 +凋 +凌 +减 +凑 +凛 +凝 +几 +凡 +凤 +凫 +凭 +凯 +凰 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +刨 +利 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +刿 +剁 +剂 +剃 +削 +剌 +前 +剐 +剑 +剔 +剖 +剜 +剥 +剧 +剩 +剪 +副 +割 +剽 +剿 +劈 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +劾 +势 +勃 +勇 +勉 +勋 +勐 +勒 +勖 +勘 +募 +勤 +勰 +勺 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匹 +区 +医 +匾 +匿 +十 +千 +卅 +升 +午 +卉 +半 +华 +协 +卑 +卒 +卓 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卺 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厮 +去 +县 +参 +又 +叉 +及 +友 +双 +反 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吲 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吾 +呀 +呃 +呆 +呈 +告 +呋 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咯 +咱 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哄 +哆 +哇 +哈 +哉 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哙 +哚 +哝 +哟 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唢 +唤 +唧 +唪 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唻 +唾 +啃 +啄 +商 +啉 +啊 +啐 +啕 +啖 +啜 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啭 +啮 +啰 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喏 +喑 +喒 +喔 +喘 +喙 +喜 +喝 +喟 +喧 +喱 +喳 +喵 +喷 +喹 +喻 +喽 +嗅 +嗑 +嗒 +嗓 +嗔 +嗖 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗫 +嗬 +嗯 +嗲 +嗳 +嗷 +嗽 +嘀 +嘁 +嘈 +嘉 +嘌 +嘎 +嘘 +嘚 +嘛 +嘞 +嘟 +嘣 +嘤 +嘬 +嘭 +嘱 +嘲 +嘴 +嘶 +嘹 +嘻 +嘿 +噌 +噎 +噔 +噗 +噙 +噜 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噶 +噻 +噼 +嚅 +嚎 +嚏 +嚓 +嚣 +嚯 +嚷 +嚼 +囊 +囍 +囔 +囚 +四 +回 +因 +囡 +团 +囤 +囧 +囫 +园 +困 +囱 +围 +囵 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +圜 +土 +圣 +在 +圩 +圪 +圭 +地 +圳 +场 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坷 +坻 +垂 +垃 +垄 +垅 +型 +垒 +垓 +垛 +垠 +垡 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +埂 +埃 +埋 +城 +埔 +埙 +域 +埠 +埭 +培 +基 +堂 +堆 +堇 +堑 +堕 +堡 +堤 +堪 +堰 +堵 +塌 +塑 +塔 +塘 +塞 +填 +塾 +墁 +境 +墅 +墉 +墎 +墒 +墓 +墙 +增 +墟 +墨 +墩 +壁 +壅 +壑 +壕 +壤 +士 +壬 +壮 +声 +壳 +壶 +壹 +处 +备 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +女 +奴 +奶 +奸 +她 +好 +妁 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妗 +妙 +妞 +妤 +妥 +妨 +妩 +妪 +妮 +妯 +妲 +妹 +妻 +妾 +姆 +姊 +始 +姐 +姑 +姒 +姓 +委 +姗 +姘 +姚 +姜 +姝 +姣 +姥 +姨 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娌 +娑 +娓 +娘 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婆 +婉 +婊 +婕 +婚 +婢 +婧 +婪 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媪 +媲 +媳 +媾 +嫁 +嫂 +嫉 +嫌 +嫔 +嫖 +嫘 +嫡 +嫣 +嫦 +嫩 +嬅 +嬉 +嬛 +嬴 +嬷 +孀 +子 +孑 +孔 +孕 +字 +存 +孙 +孚 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孬 +孰 +孱 +孳 +孵 +孺 +孽 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寝 +寞 +察 +寡 +寤 +寥 +寨 +寮 +寰 +寸 +对 +寺 +寻 +导 +寿 +封 +射 +将 +尉 +尊 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +居 +屈 +屉 +届 +屋 +屌 +屎 +屏 +屐 +屑 +展 +属 +屠 +屡 +履 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岚 +岛 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +岿 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峰 +峻 +崂 +崃 +崆 +崇 +崎 +崔 +崖 +崛 +崤 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵘 +嵩 +嵬 +嶂 +嶙 +巅 +巍 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巽 +巾 +币 +市 +布 +帅 +帆 +师 +希 +帏 +帐 +帔 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +带 +帧 +席 +帮 +帷 +常 +帼 +帽 +幂 +幄 +幅 +幌 +幔 +幕 +幡 +幢 +干 +平 +年 +并 +幸 +幺 +幻 +幼 +幽 +广 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +度 +座 +庭 +庵 +庶 +康 +庸 +庾 +廉 +廊 +廓 +廖 +廪 +延 +廷 +建 +廿 +开 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弓 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +弹 +强 +弼 +彀 +归 +当 +录 +彗 +彝 +形 +彤 +彦 +彧 +彩 +彪 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徒 +徕 +得 +徘 +徙 +徜 +御 +徨 +循 +徭 +微 +徵 +德 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忾 +忿 +怀 +态 +怂 +怄 +怅 +怆 +怍 +怎 +怏 +怒 +怔 +怕 +怖 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怩 +怪 +怯 +怵 +总 +怼 +怿 +恁 +恃 +恋 +恍 +恐 +恒 +恕 +恙 +恢 +恣 +恤 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恶 +恸 +恹 +恺 +恻 +恼 +恽 +恿 +悄 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悬 +悯 +悱 +悲 +悴 +悸 +悻 +悼 +情 +惆 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +想 +惴 +惶 +惹 +惺 +愁 +愈 +愉 +愍 +愎 +意 +愕 +愚 +感 +愠 +愣 +愤 +愧 +愫 +愿 +慈 +慌 +慎 +慑 +慕 +慢 +慧 +慨 +慰 +慵 +慷 +憋 +憎 +憔 +憧 +憨 +憩 +憬 +憷 +憾 +懂 +懈 +懊 +懋 +懑 +懒 +懦 +懵 +懿 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +戗 +战 +戚 +戛 +戟 +截 +戬 +戮 +戳 +戴 +户 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +抔 +投 +抖 +抗 +折 +抚 +抛 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +抻 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挺 +挽 +捂 +捅 +捆 +捉 +捋 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捭 +据 +捯 +捱 +捶 +捷 +捺 +捻 +掀 +掂 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掠 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掮 +掰 +掳 +掷 +掸 +掺 +掼 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揠 +握 +揣 +揩 +揪 +揭 +援 +揶 +揽 +搀 +搁 +搂 +搅 +搏 +搐 +搓 +搔 +搜 +搞 +搡 +搪 +搬 +搭 +携 +搽 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摧 +摩 +摸 +摹 +撂 +撅 +撇 +撑 +撒 +撕 +撞 +撤 +撩 +撬 +播 +撮 +撰 +撵 +撷 +撸 +撺 +撼 +擀 +擂 +擅 +操 +擎 +擒 +擞 +擢 +擤 +擦 +攀 +攒 +攘 +攥 +攫 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敏 +救 +敕 +敖 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敷 +文 +斋 +斌 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +断 +斯 +新 +方 +於 +施 +旁 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旯 +旰 +旱 +时 +旷 +旺 +昀 +昂 +昃 +昆 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昵 +昶 +昼 +显 +晁 +時 +晃 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晟 +晤 +晦 +晨 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暑 +暖 +暗 +暝 +暧 +暨 +暮 +暴 +暹 +曙 +曜 +曝 +曦 +曰 +曲 +曳 +更 +曹 +曼 +曾 +替 +最 +月 +有 +朊 +朋 +服 +朔 +朕 +朗 +望 +朝 +期 +朦 +木 +未 +末 +本 +札 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杪 +杭 +杯 +杰 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +析 +枕 +林 +枚 +果 +枝 +枞 +枢 +枣 +枥 +枪 +枫 +枭 +枯 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柿 +栀 +栅 +标 +栈 +栉 +栋 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桷 +梁 +梅 +梆 +梏 +梓 +梗 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +检 +棂 +棉 +棋 +棍 +棒 +棕 +棘 +棚 +棠 +棣 +森 +棱 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +椟 +椭 +椰 +椴 +椽 +椿 +楂 +楔 +楚 +楞 +楠 +楣 +楫 +楷 +楹 +楼 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榛 +榜 +榨 +榫 +榭 +榴 +榷 +榻 +槁 +槃 +槊 +槌 +槐 +槛 +槟 +槽 +槿 +樊 +樓 +樟 +模 +樨 +横 +樯 +樱 +樵 +樽 +樾 +橄 +橇 +橐 +橘 +橙 +橛 +橡 +橱 +橹 +檀 +檄 +檐 +檬 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +款 +歃 +歆 +歇 +歉 +歌 +歙 +止 +正 +此 +步 +武 +歧 +歪 +歹 +死 +歼 +殂 +殃 +殄 +殆 +殇 +殉 +殊 +残 +殍 +殒 +殓 +殖 +殚 +殡 +殴 +段 +殷 +殿 +毁 +毂 +毅 +毋 +母 +每 +毒 +毓 +比 +毕 +毖 +毗 +毙 +毛 +毡 +毫 +毯 +毽 +氅 +氏 +氐 +民 +氓 +气 +氙 +氛 +氟 +氢 +氤 +氦 +氧 +氨 +氩 +氪 +氮 +氯 +氰 +氲 +水 +永 +汀 +汁 +求 +汆 +汇 +汉 +汊 +汐 +汕 +汗 +汛 +汜 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +汽 +汾 +沁 +沂 +沃 +沅 +沆 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沔 +沙 +沛 +沟 +没 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +泄 +泉 +泊 +泌 +泓 +泔 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洇 +洋 +洌 +洒 +洗 +洙 +洛 +洞 +津 +洪 +洮 +洱 +洲 +洵 +洹 +活 +洼 +洽 +派 +流 +浃 +浅 +浆 +浇 +浊 +测 +济 +浏 +浐 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浠 +浣 +浥 +浦 +浩 +浪 +浮 +浴 +海 +浸 +涂 +涅 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淖 +淘 +淙 +淝 +淞 +淡 +淤 +淦 +淫 +淬 +淮 +深 +淳 +混 +淹 +添 +淼 +清 +渊 +渌 +渍 +渎 +渐 +渑 +渔 +渗 +渚 +渝 +渠 +渡 +渣 +渤 +渥 +温 +渭 +港 +渲 +渴 +游 +渺 +湃 +湄 +湍 +湎 +湓 +湖 +湘 +湛 +湟 +湮 +湾 +湿 +溃 +溅 +溉 +溏 +源 +溘 +溜 +溟 +溢 +溥 +溧 +溪 +溯 +溴 +溶 +溺 +滁 +滂 +滇 +滋 +滏 +滑 +滓 +滔 +滕 +滚 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滴 +滹 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漩 +漪 +漫 +漯 +漱 +漳 +漾 +潇 +潋 +潍 +潘 +潜 +潞 +潢 +潦 +潭 +潮 +潴 +潸 +潺 +潼 +澄 +澈 +澍 +澎 +澜 +澡 +澧 +澳 +澶 +澹 +激 +濂 +濑 +濒 +濛 +濞 +濠 +濡 +濮 +濯 +瀑 +瀚 +瀛 +瀣 +灌 +灏 +灞 +火 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +灾 +灿 +炀 +炅 +炉 +炊 +炎 +炒 +炕 +炖 +炙 +炜 +炝 +炫 +炬 +炭 +炮 +炯 +炳 +炴 +炷 +炸 +点 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焦 +焯 +焰 +焱 +然 +煅 +煊 +煌 +煎 +煜 +煞 +煤 +煦 +照 +煨 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熔 +熙 +熟 +熠 +熨 +熬 +熵 +熹 +燃 +燊 +燎 +燕 +燚 +燥 +燧 +燮 +爆 +爪 +爬 +爱 +爵 +父 +爷 +爸 +爹 +爻 +爽 +片 +版 +牌 +牍 +牒 +牙 +牛 +牟 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牵 +特 +牺 +犀 +犁 +犄 +犊 +犍 +犒 +犟 +犬 +犯 +状 +犷 +犸 +犹 +狂 +狄 +狈 +狍 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狲 +狸 +狼 +猎 +猕 +猖 +猛 +猜 +猝 +猢 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猷 +猾 +猿 +獐 +獒 +獗 +獠 +獭 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +班 +珮 +珲 +珺 +球 +琅 +理 +琉 +琊 +琏 +琐 +琚 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琳 +琴 +琵 +琶 +琼 +瑁 +瑄 +瑕 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑭 +瑰 +瑶 +瑾 +璀 +璃 +璇 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +瓒 +瓜 +瓠 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甘 +甙 +甚 +甜 +生 +甥 +用 +甩 +甫 +甬 +甭 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畅 +畈 +界 +畏 +畔 +留 +畜 +略 +畦 +番 +畲 +畴 +畸 +畿 +疆 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疠 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疴 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痛 +痞 +痢 +痣 +痤 +痦 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痼 +痿 +瘀 +瘁 +瘆 +瘘 +瘙 +瘟 +瘠 +瘢 +瘤 +瘦 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +癌 +癔 +癖 +癜 +癞 +癣 +癫 +癸 +登 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皌 +皎 +皑 +皓 +皖 +皙 +皮 +皱 +皲 +皿 +盂 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盟 +盥 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +眙 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +着 +睁 +睐 +睑 +睚 +睛 +睡 +睢 +督 +睥 +睦 +睨 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞌 +瞎 +瞑 +瞒 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞿 +矍 +矗 +矛 +矜 +矢 +矣 +知 +矩 +矫 +矬 +短 +矮 +石 +矶 +矾 +矿 +砀 +码 +砂 +砌 +砍 +砒 +研 +砖 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +破 +砷 +砸 +砺 +砾 +础 +硅 +硌 +硒 +硕 +硝 +硫 +硬 +确 +硼 +碉 +碌 +碍 +碎 +碑 +碗 +碘 +碚 +碜 +碟 +碣 +碧 +碰 +碱 +碳 +碴 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磡 +磨 +磬 +磴 +磷 +磺 +礁 +礴 +示 +礼 +社 +祀 +祁 +祇 +祈 +祉 +祎 +祐 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +禀 +禁 +禄 +禅 +福 +禧 +禹 +禺 +离 +禽 +禾 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秫 +秭 +积 +称 +秸 +移 +秽 +稀 +程 +稍 +税 +稔 +稗 +稚 +稞 +稠 +稣 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穆 +穗 +穰 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窣 +窥 +窦 +窨 +窸 +窿 +立 +竖 +站 +竞 +竟 +章 +竣 +童 +竭 +端 +竹 +竺 +竽 +竿 +笃 +笄 +笆 +笈 +笋 +笏 +笑 +笔 +笕 +笙 +笛 +笞 +笠 +笤 +符 +笨 +第 +笳 +笸 +笺 +笼 +等 +筋 +筏 +筐 +筑 +筒 +答 +策 +筚 +筛 +筝 +筠 +筱 +筵 +筷 +筹 +签 +简 +箍 +箐 +箔 +箕 +算 +箜 +管 +箩 +箫 +箭 +箱 +箴 +箸 +篁 +篆 +篇 +篌 +篑 +篓 +篙 +篝 +篡 +篦 +篮 +篱 +篷 +篾 +簇 +簋 +簌 +簟 +簧 +簪 +簸 +簿 +籁 +籍 +米 +类 +籽 +粉 +粑 +粒 +粕 +粗 +粘 +粝 +粟 +粤 +粥 +粪 +粮 +粱 +粲 +粳 +粹 +粼 +粽 +精 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糟 +糠 +糯 +系 +紊 +素 +索 +紧 +紫 +累 +絮 +綦 +繁 +繇 +纂 +纠 +红 +纣 +纤 +纥 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绉 +绊 +绌 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绺 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缑 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缛 +缜 +缝 +缟 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缫 +缭 +缮 +缰 +缱 +缴 +缶 +缸 +缺 +罂 +罄 +罐 +网 +罔 +罕 +罗 +罘 +罚 +罡 +罢 +罩 +罪 +置 +署 +罹 +羁 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +群 +羧 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聘 +聚 +聩 +聪 +聿 +肃 +肄 +肆 +肇 +肉 +肋 +肌 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胗 +胚 +胛 +胜 +胞 +胡 +胤 +胥 +胧 +胪 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脆 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脯 +脱 +脸 +脾 +腆 +腈 +腊 +腋 +腌 +腐 +腑 +腓 +腔 +腕 +腚 +腥 +腧 +腩 +腭 +腮 +腰 +腱 +腴 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膛 +膜 +膝 +膦 +膨 +膳 +膺 +膻 +臀 +臂 +臃 +臆 +臊 +臣 +臧 +自 +臬 +臭 +至 +致 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +舌 +舍 +舐 +舒 +舔 +舛 +舜 +舞 +舟 +舢 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +艇 +艘 +艮 +良 +艰 +色 +艳 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芨 +芩 +芪 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芽 +芾 +苁 +苇 +苋 +苍 +苎 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茆 +茉 +茌 +茎 +茏 +茔 +茕 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +荇 +草 +荏 +荐 +荒 +荔 +荚 +荜 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荷 +荸 +荻 +荼 +莅 +莆 +莉 +莎 +莒 +莓 +莘 +莜 +莞 +莠 +莨 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +菁 +菅 +菇 +菊 +菌 +菏 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +菪 +菱 +菲 +萁 +萃 +萄 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萱 +萸 +萼 +落 +葆 +著 +葚 +葛 +葡 +董 +葩 +葫 +葬 +葭 +葱 +葳 +葵 +葺 +蒂 +蒋 +蒙 +蒜 +蒡 +蒲 +蒸 +蒹 +蒺 +蒽 +蒿 +蓁 +蓄 +蓉 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓿 +蔑 +蔓 +蔗 +蔚 +蔡 +蔫 +蔬 +蔷 +蔺 +蔻 +蔼 +蔽 +蕃 +蕉 +蕊 +蕖 +蕙 +蕞 +蕤 +蕨 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薏 +薛 +薪 +薯 +薰 +藁 +藉 +藏 +藐 +藓 +藕 +藜 +藤 +藩 +藻 +藿 +蘅 +蘑 +蘸 +蘼 +虎 +虏 +虐 +虑 +虔 +虚 +虞 +虢 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚡 +蚣 +蚤 +蚩 +蚪 +蚬 +蚯 +蚱 +蚴 +蛀 +蛆 +蛇 +蛉 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛾 +蜀 +蜂 +蜃 +蜇 +蜈 +蜉 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜮 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝗 +蝙 +蝠 +蝣 +蝰 +蝴 +蝶 +蝼 +蝾 +螂 +螃 +螅 +螈 +融 +螨 +螯 +螳 +螺 +蟀 +蟆 +蟋 +蟑 +蟒 +蟠 +蟥 +蟹 +蟾 +蠕 +蠡 +蠢 +蠹 +血 +衅 +行 +衍 +衔 +街 +衙 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袤 +被 +袭 +袱 +裁 +裂 +装 +裆 +裔 +裕 +裘 +裙 +裟 +裢 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +褂 +褊 +褐 +褒 +褓 +褚 +褛 +褡 +褥 +褪 +褴 +褶 +襁 +襄 +襟 +襦 +西 +要 +覃 +覆 +见 +观 +规 +觅 +视 +觇 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +言 +詹 +誉 +誊 +誓 +謇 +警 +譬 +计 +订 +讣 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诂 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +诌 +词 +诏 +译 +诒 +诓 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诨 +诩 +诫 +诬 +语 +诮 +误 +诰 +诱 +诲 +诳 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谙 +谚 +谛 +谜 +谟 +谡 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豉 +豌 +豚 +象 +豢 +豪 +豫 +豸 +豹 +豺 +貂 +貅 +貉 +貌 +貔 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贶 +贷 +贸 +费 +贺 +贻 +贼 +贽 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赉 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赟 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +趄 +超 +越 +趋 +趔 +趟 +趣 +足 +趴 +趵 +趸 +趾 +趿 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跤 +跨 +跪 +跬 +路 +跳 +践 +跶 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踵 +踹 +踽 +蹁 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹦 +蹩 +蹬 +蹭 +蹰 +蹲 +蹴 +蹶 +蹼 +蹿 +躁 +躇 +躏 +身 +躬 +躯 +躲 +躺 +车 +轧 +轨 +轩 +转 +轮 +软 +轰 +轱 +轲 +轳 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辋 +辍 +辎 +辐 +辑 +输 +辔 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辨 +辩 +辫 +辰 +辱 +边 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迷 +迸 +迹 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +逑 +递 +途 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逡 +逢 +逮 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遇 +遍 +遏 +遐 +遑 +遒 +道 +遗 +遛 +遢 +遣 +遥 +遨 +遭 +遮 +遴 +遵 +遽 +避 +邀 +邂 +邃 +還 +邈 +邋 +邑 +邓 +邕 +邗 +邙 +邛 +邝 +邡 +邢 +那 +邦 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郇 +郊 +郎 +郑 +郓 +郜 +郝 +郡 +郢 +郦 +郧 +部 +郫 +郭 +郯 +郴 +郸 +都 +郾 +鄂 +鄙 +鄞 +鄢 +鄯 +鄱 +酉 +酊 +酋 +酌 +配 +酎 +酒 +酗 +酚 +酝 +酞 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酽 +酿 +醅 +醇 +醉 +醋 +醌 +醍 +醐 +醒 +醚 +醛 +醪 +醮 +醴 +醺 +采 +釉 +释 +里 +重 +野 +量 +金 +釜 +鉴 +銮 +錾 +鎏 +鏖 +鑫 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铍 +铎 +铐 +铖 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铱 +铲 +铳 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锃 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锒 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锨 +锭 +键 +锯 +锰 +锱 +锲 +锵 +锷 +锹 +锻 +镀 +镁 +镂 +镆 +镇 +镉 +镊 +镌 +镍 +镐 +镑 +镒 +镕 +镖 +镛 +镜 +镢 +镣 +镭 +镯 +镰 +镳 +镶 +长 +门 +闩 +闪 +闫 +闭 +问 +闯 +闰 +闱 +闲 +闳 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闼 +闽 +闾 +阀 +阁 +阂 +阄 +阅 +阆 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阗 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陉 +陋 +陌 +降 +限 +陕 +陛 +陟 +陡 +院 +除 +陨 +险 +陪 +陲 +陵 +陶 +陷 +隅 +隆 +隋 +隍 +随 +隐 +隔 +隗 +隘 +隙 +障 +隧 +隶 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雌 +雍 +雎 +雏 +雕 +雨 +雪 +雯 +雳 +零 +雷 +雹 +雾 +需 +霁 +霄 +霆 +震 +霉 +霍 +霎 +霏 +霓 +霖 +霜 +霞 +霭 +霰 +露 +霸 +霹 +霾 +青 +靓 +靖 +静 +靛 +非 +靠 +靡 +面 +靥 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞑 +鞘 +鞠 +鞣 +鞭 +韦 +韧 +韩 +韪 +韫 +韬 +韭 +音 +韵 +韶 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颀 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颏 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颜 +额 +颞 +颠 +颢 +颤 +颦 +颧 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飞 +食 +餐 +餮 +饕 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馊 +馋 +馍 +馏 +馐 +馑 +馒 +馔 +馕 +首 +馗 +香 +馥 +馨 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驽 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骝 +骞 +骠 +骡 +骢 +骤 +骥 +骨 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髌 +髓 +高 +髡 +髦 +髯 +髻 +鬃 +鬄 +鬓 +鬟 +鬣 +鬶 +鬻 +鬼 +魁 +魂 +魃 +魄 +魅 +魇 +魉 +魍 +魏 +魑 +魔 +鱼 +鱿 +鲀 +鲁 +鲅 +鲇 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲠 +鲢 +鲤 +鲧 +鲨 +鲫 +鲲 +鲳 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳏 +鳐 +鳕 +鳖 +鳗 +鳜 +鳝 +鳞 +鳟 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸩 +鸪 +鸬 +鸭 +鸮 +鸯 +鸱 +鸳 +鸵 +鸷 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹗 +鹘 +鹚 +鹜 +鹞 +鹤 +鹦 +鹧 +鹩 +鹪 +鹫 +鹬 +鹭 +鹰 +鹳 +鹿 +麋 +麒 +麓 +麝 +麟 +麦 +麴 +麸 +麻 +麼 +麽 +麾 +黄 +黍 +黎 +黏 +黑 +黔 +默 +黛 +黜 +黝 +黠 +黢 +黩 +黯 +鼎 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齉 +齐 +齿 +龄 +龅 +龇 +龈 +龊 +龋 +龌 +龙 +龚 +龛 +龟 +