修改语言配置与官方一致,修改搜索token生成机制

This commit is contained in:
dingtalk_dxneyp 2024-01-31 12:31:03 +08:00
parent 0f12973881
commit 51e6fe7952

View File

@ -15,6 +15,7 @@
{ {
encode: false, encode: false,
tokenize: function(str) { tokenize: function(str) {
return str.replace(/[\x00-\x7F]/g, '').split(''); // 只去除空白行,标点符号保留、表情符号保留、数字保留、英文保留
return str.replace(/\n*\s*\r*/g, '').split('');
} }
} }