[FIX] Local database searches using non-latin characters (#2462)

* [FIX] Local database searches using non-latin characters

* Add tests

app/containers/MessageBox/index.js

@@ -46,6 +46,7 @@ import CommandsPreview from './CommandsPreview';
 import { getUserSelector } from '../../selectors/login';
 import Navigation from '../../lib/Navigation';
 import { withActionSheet } from '../ActionSheet';
+import { sanitizeLikeString } from '../../lib/database/utils';
 
 const imagePickerConfig = {
 	cropping: true,
@@ -491,8 +492,9 @@ class MessageBox extends Component {
 		const db = database.active;
 		if (keyword) {
 			const customEmojisCollection = db.collections.get('custom_emojis');
+			const likeString = sanitizeLikeString(keyword);
 			let customEmojis = await customEmojisCollection.query(
-				Q.where('name', Q.like(`${ Q.sanitizeLikeString(keyword) }%`))
+				Q.where('name', Q.like(`${ likeString }%`))
 			).fetch();
 			customEmojis = customEmojis.slice(0, MENTIONS_COUNT_TO_DISPLAY);
 			const filteredEmojis = emojis.filter(emoji => emoji.indexOf(keyword) !== -1).slice(0, MENTIONS_COUNT_TO_DISPLAY);
@@ -504,8 +506,9 @@ class MessageBox extends Component {
 	getSlashCommands = debounce(async(keyword) => {
 		const db = database.active;
 		const commandsCollection = db.collections.get('slash_commands');
+		const likeString = sanitizeLikeString(keyword);
 		const commands = await commandsCollection.query(
-			Q.where('id', Q.like(`${ Q.sanitizeLikeString(keyword) }%`))
+			Q.where('id', Q.like(`${ likeString }%`))
 		).fetch();
 		this.setState({ mentions: commands || [] });
 	}, 300)
@@ -734,8 +737,9 @@ class MessageBox extends Component {
 			const db = database.active;
 			const commandsCollection = db.collections.get('slash_commands');
 			const command = message.replace(/ .*/, '').slice(1);
+			const likeString = sanitizeLikeString(command);
 			const slashCommand = await commandsCollection.query(
-				Q.where('id', Q.like(`${ Q.sanitizeLikeString(command) }%`))
+				Q.where('id', Q.like(`${ likeString }%`))
 			).fetch();
 			if (slashCommand.length > 0) {
 				logEvent(events.COMMAND_RUN);

app/lib/database/utils.js

@@ -1 +1,7 @@
+import XRegExp from 'xregexp';
+
+// Matches letters from any alphabet and numbers
+const likeStringRegex = new XRegExp('[^\\p{L}\\p{Nd}]', 'g');
+export const sanitizeLikeString = str => str.replace(likeStringRegex, '_');
+
 export const sanitizer = r => r;

app/lib/database/utils.test.js

@@ -0,0 +1,37 @@
+/* eslint-disable no-undef */
+import * as utils from './utils';
+
+describe('sanitizeLikeStringTester', () => {
+	// example chars that shouldn't return
+	const disallowedChars = ',./;[]!@#$%^&*()_-=+~';
+
+	const sanitizeLikeStringTester = str => expect(utils.sanitizeLikeString(`${ str }${ disallowedChars }`)).toBe(`${ str }${ '_'.repeat(disallowedChars.length) }`);
+
+	// Testing a couple of different alphabets
+	test('render test (latin)', () => {
+		sanitizeLikeStringTester('test123');
+	});
+
+	test('render test (arabic)', () => {
+		sanitizeLikeStringTester('اختبار123');
+	});
+
+	test('render test (russian)', () => {
+		sanitizeLikeStringTester('тест123');
+	});
+
+	test('render test (chinese trad)', () => {
+		sanitizeLikeStringTester('測試123');
+	});
+
+	test('render test (japanese)', () => {
+		sanitizeLikeStringTester('テスト123');
+	});
+});
+
+describe('sanitizer', () => {
+	test('render the same result', () => {
+		const content = { a: true };
+		expect(utils.sanitizer(content)).toBe(content);
+	});
+});

app/lib/rocketchat.js

@@ -56,6 +56,7 @@ import { useSsl } from '../utils/url';
 import UserPreferences from './userPreferences';
 import { Encryption } from './encryption';
 import EventEmitter from '../utils/events';
+import { sanitizeLikeString } from './database/utils';
 
 const TOKEN_KEY = 'reactnativemeteor_usertoken';
 const CURRENT_SERVER = 'currentServer';
@@ -517,8 +518,12 @@ const RocketChat = {
 		}
 
 		const db = database.active;
+		const likeString = sanitizeLikeString(searchText);
 		let data = await db.collections.get('subscriptions').query(
-			Q.where('name', Q.like(`%${ Q.sanitizeLikeString(searchText) }%`))
+			Q.or(
+				Q.where('name', Q.like(`%${ likeString }%`)),
+				Q.where('fname', Q.like(`%${ likeString }%`))
+			)
 		).fetch();
 
 		if (filterUsers && !filterRooms) {

app/views/NewServerView/index.js

@@ -30,6 +30,7 @@ import { CloseModalButton } from '../../containers/HeaderButton';
 import { showConfirmationAlert } from '../../utils/info';
 import database from '../../lib/database';
 import ServerInput from './ServerInput';
+import { sanitizeLikeString } from '../../lib/database/utils';
 
 const styles = StyleSheet.create({
 	title: {
@@ -138,10 +139,11 @@ class NewServerView extends React.Component {
 				Q.experimentalSortBy('updated_at', Q.desc),
 				Q.experimentalTake(3)
 			];
+			const likeString = sanitizeLikeString(text);
 			if (text) {
 				whereClause = [
 					...whereClause,
-					Q.where('url', Q.like(`%${ Q.sanitizeLikeString(text) }%`))
+					Q.where('url', Q.like(`%${ likeString }%`))
 				];
 			}
 			const serversHistory = await serversHistoryCollection.query(...whereClause).fetch();

app/views/SearchMessagesView/index.js

@@ -22,6 +22,7 @@ import { getUserSelector } from '../../selectors/login';
 import SafeAreaView from '../../containers/SafeAreaView';
 import { CloseModalButton } from '../../containers/HeaderButton';
 import database from '../../lib/database';
+import { sanitizeLikeString } from '../../lib/database/utils';
 
 class SearchMessagesView extends React.Component {
 	static navigationOptions = ({ navigation, route }) => {
@@ -83,12 +84,13 @@ class SearchMessagesView extends React.Component {
 		if (this.encrypted) {
 			const db = database.active;
 			const messagesCollection = db.collections.get('messages');
+			const likeString = sanitizeLikeString(searchText);
 			return messagesCollection
 				.query(
 					// Messages of this room
 					Q.where('rid', this.rid),
 					// Message content is like the search text
-					Q.where('msg', Q.like(`%${ Q.sanitizeLikeString(searchText) }%`))
+					Q.where('msg', Q.like(`%${ likeString }%`))
 				)
 				.fetch();
 		}

app/views/ShareListView/index.js

@@ -26,6 +26,7 @@ import { animateNextTransition } from '../../utils/layoutAnimation';
 import { withTheme } from '../../theme';
 import SafeAreaView from '../../containers/SafeAreaView';
 import RocketChat from '../../lib/rocketchat';
+import { sanitizeLikeString } from '../../lib/database/utils';
 
 const permission = {
 	title: I18n.t('Read_External_Permission'),
@@ -195,13 +196,14 @@ class ShareListView extends React.Component {
 			Q.experimentalSortBy('room_updated_at', Q.desc)
 		];
 		if (text) {
+			const likeString = sanitizeLikeString(text);
 			return db.collections
 				.get('subscriptions')
 				.query(
 					...defaultWhereClause,
 					Q.or(
-						Q.where('name', Q.like(`%${ Q.sanitizeLikeString(text) }%`)),
+						Q.where('name', Q.like(`%${ likeString }%`)),
-						Q.where('fname', Q.like(`%${ Q.sanitizeLikeString(text) }%`))
+						Q.where('fname', Q.like(`%${ likeString }%`))
 					)
 				).fetch();
 		}

package.json

@@ -123,7 +123,8 @@
     "semver": "7.3.2",
     "ua-parser-js": "^0.7.21",
     "url-parse": "^1.4.7",
-    "use-deep-compare-effect": "^1.3.1"
+    "use-deep-compare-effect": "^1.3.1",
+    "xregexp": "^4.3.0"
   },
   "devDependencies": {
     "@babel/core": "^7.8.4",

yarn.lock

@@ -16109,7 +16109,7 @@ xregexp@4.1.1:
   resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-4.1.1.tgz#eb8a032aa028d403f7b1b22c47a5f16c24b21d8d"
   integrity sha512-QJ1gfSUV7kEOLfpKFCjBJRnfPErUzkNKFMso4kDSmGpp3x6ZgkyKf74inxI7PnnQCFYq5TqYJCd7DrgDN8Q05A==
 
-xregexp@^4.2.4:
+xregexp@^4.2.4, xregexp@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-4.3.0.tgz#7e92e73d9174a99a59743f67a4ce879a04b5ae50"
   integrity sha512-7jXDIFXh5yJ/orPn4SXjuVrWWoi4Cr8jfV1eHv9CixKSbU+jY4mxfrBwAuDvupPNKpMUY+FeIqsVw/JLT9+B8g==