Making tests more reliable

e2e/data.ts

@@ -37,10 +37,20 @@ const data = {
 			password: '123',
 			email: `mobile+profileChanges${value}@rocket.chat`
 		},
+		encryption: {
+			username: `userencryption${value}`,
+			password: '123',
+			email: `mobile+encryption${value}@rocket.chat`
+		},
 		existing: {
 			username: `existinguser${value}`,
 			password: '123',
 			email: `mobile+existing${value}@rocket.chat`
+		},
+		inapp: {
+			username: `inappuser${value}`,
+			password: '123',
+			email: `mobile+inapp${value}@rocket.chat`
 		}
 	},
 	channels: {

e2e/helpers/app.ts

@@ -109,36 +109,6 @@ async function mockMessage(message: string, isThread = false) {
 		.tap();
 }
 
-async function starMessage(message: string) {
-	const deviceType = device.getPlatform();
-	const { textMatcher } = platformTypes[deviceType];
-	const messageLabel = `${data.random}${message}`;
-	await element(by[textMatcher](messageLabel)).atIndex(0).longPress();
-	await expect(element(by.id('action-sheet'))).toExist();
-	await expect(element(by.id('action-sheet-handle'))).toBeVisible();
-	await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
-	await element(by[textMatcher]('Star')).atIndex(0).tap();
-	await waitFor(element(by.id('action-sheet')))
-		.not.toExist()
-		.withTimeout(5000);
-}
-
-async function pinMessage(message: string) {
-	const deviceType = device.getPlatform();
-	const { textMatcher } = platformTypes[deviceType];
-	const messageLabel = `${data.random}${message}`;
-	await waitFor(element(by[textMatcher](messageLabel)).atIndex(0)).toExist();
-	await element(by[textMatcher](messageLabel)).atIndex(0).tap();
-	await element(by[textMatcher](messageLabel)).atIndex(0).longPress();
-	await expect(element(by.id('action-sheet'))).toExist();
-	await expect(element(by.id('action-sheet-handle'))).toBeVisible();
-	await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
-	await element(by[textMatcher]('Pin')).atIndex(0).tap();
-	await waitFor(element(by.id('action-sheet')))
-		.not.toExist()
-		.withTimeout(5000);
-}
-
 async function dismissReviewNag() {
 	const deviceType = device.getPlatform();
 	const { textMatcher } = platformTypes[deviceType];
@@ -150,6 +120,7 @@ async function dismissReviewNag() {
 
 async function tapBack() {
 	await element(by.id('header-back')).atIndex(0).tap();
+	await sleep(300); // Wait for animation to finish
 }
 
 async function searchRoom(room: string) {
@@ -173,10 +144,10 @@ async function navigateToRoom(room: string) {
 async function tryTapping(
 	theElement: Detox.IndexableNativeElement | Detox.NativeElement,
 	timeout: number,
-	longtap = false
+	longPress = false
 ): Promise<void> {
 	try {
-		if (longtap) {
+		if (longPress) {
 			await theElement.tap();
 			await theElement.longPress();
 		} else {
@@ -193,11 +164,16 @@ async function tryTapping(
 async function tapAndWaitFor(
 	elementToTap: Detox.IndexableNativeElement | Detox.NativeElement,
 	elementToWaitFor: Detox.IndexableNativeElement | Detox.NativeElement,
-	timeout: number
+	timeout: number,
+	longPress = false
 ) {
 	try {
-		await elementToTap.tap();
+		if (longPress) {
-		await waitFor(elementToWaitFor).toBeVisible().withTimeout(200);
+			elementToTap.longPress();
+		} else {
+			await elementToTap.tap();
+		}
+		await waitFor(elementToWaitFor).toBeVisible().withTimeout(1000);
 	} catch (e) {
 		if (timeout <= 0) {
 			throw e;
@@ -231,6 +207,26 @@ const checkServer = async (server: string) => {
 		.withTimeout(10000);
 };
 
+// Useful to get rid of `Too many requests` alert on register
+async function expectValidRegisterOrRetry(platform: keyof typeof platformTypes, retries = 3) {
+	if (retries === 0) {
+		throw new Error('Too many retries');
+	}
+	try {
+		await waitFor(element(by.id('rooms-list-view')))
+			.toBeVisible()
+			.withTimeout(60000);
+	} catch (error) {
+		/**
+		 * We can't use regex to properly match by label, so we assume [error-too-many-requests] is visible.
+		 * We don't need to wait for another 60 seconds, because we already did above, so we just try again.
+		 *  */
+		await element(by[platformTypes[platform].textMatcher]('OK').and(by.type(platformTypes[platform].alertButtonType))).tap();
+		await element(by.id('register-view-submit')).tap();
+		await expectValidRegisterOrRetry(platform, retries - 1);
+	}
+}
+
 export {
 	navigateToWorkspace,
 	navigateToLogin,
@@ -238,8 +234,6 @@ export {
 	login,
 	logout,
 	mockMessage,
-	starMessage,
-	pinMessage,
 	dismissReviewNag,
 	tapBack,
 	sleep,
@@ -249,5 +243,6 @@ export {
 	tapAndWaitFor,
 	checkRoomTitle,
 	checkServer,
-	platformTypes
+	platformTypes,
+	expectValidRegisterOrRetry
 };

e2e/helpers/data_setup.ts

@@ -183,8 +183,8 @@ const get = (endpoint: string) => {
 	return rocketchat.get(endpoint);
 };
 
-const post = async (endpoint: string, body: any) => {
+const post = async (endpoint: string, body: any, user = data.users.regular) => {
-	await login(data.users.regular.username, data.users.regular.password);
+	await login(user.username, user.password);
 	console.log(`POST /${endpoint} ${JSON.stringify(body)}`);
 	return rocketchat.post(endpoint, body);
 };

e2e/jest.config.js

@@ -3,8 +3,8 @@ module.exports = {
 	rootDir: '..',
 	testSequencer: '<rootDir>/e2e/testSequencer.js',
 	testMatch: ['<rootDir>/e2e/tests/**/*.spec.ts'],
-	testTimeout: 120000,
+	testTimeout: 240000,
-	maxWorkers: 2,
+	maxWorkers: 3,
 	globalSetup: '<rootDir>/e2e/globalSetup.ts',
 	globalTeardown: 'detox/runners/jest/globalTeardown',
 	reporters: ['detox/runners/jest/reporter'],

e2e/tests/assorted/01-e2eencryption.spec.ts

@@ -10,11 +10,12 @@ import {
 	logout,
 	platformTypes,
 	TTextMatcher,
-	tapAndWaitFor
+	tapAndWaitFor,
+	expectValidRegisterOrRetry
 } from '../../helpers/app';
 import data from '../../data';
 
-const testuser = data.users.regular;
+const testuser = data.users.encryption;
 const otheruser = data.users.alternate;
 
 const checkServer = async (server: string) => {
@@ -107,6 +108,7 @@ describe('E2E Encryption', () => {
 
 			it('should tap "Save my password" and close modal', async () => {
 				await element(by.id('e2e-save-password-view-saved-password').and(by.label('I Saved My E2E Password'))).tap();
+				await sleep(300); // wait for animation
 				await waitFor(element(by.id('rooms-list-view')))
 					.toBeVisible()
 					.withTimeout(2000);
@@ -365,10 +367,8 @@ describe('E2E Encryption', () => {
 			await element(by.id('register-view-username')).replaceText(data.registeringUser.username);
 			await element(by.id('register-view-email')).replaceText(data.registeringUser.email);
 			await element(by.id('register-view-password')).replaceText(data.registeringUser.password);
-			await element(by.id('register-view-submit')).tap();
+			await element(by.id('register-view-password')).tapReturnKey();
-			await waitFor(element(by.id('rooms-list-view')))
+			await expectValidRegisterOrRetry(device.getPlatform());
-				.toBeVisible()
-				.withTimeout(60000);
 
 			await checkServer(data.alternateServer);
 		});

e2e/tests/assorted/05-joinpublicroom.spec.ts

@@ -1,7 +1,7 @@
 import { expect } from 'detox';
 
 import data from '../../data';
-import { navigateToLogin, login, mockMessage, tapBack, searchRoom, platformTypes, TTextMatcher } from '../../helpers/app';
+import { navigateToLogin, login, mockMessage, tapBack, searchRoom, platformTypes, TTextMatcher, sleep } from '../../helpers/app';
 
 const testuser = data.users.regular;
 const room = data.channels.detoxpublic.name;
@@ -122,6 +122,9 @@ describe('Join public room', () => {
 	describe('Usage', () => {
 		it('should join room', async () => {
 			await element(by.id('room-view-join-button')).tap();
+			await waitFor(element(by.id('room-view-join-button')))
+				.not.toBeVisible()
+				.withTimeout(2000);
 			await tapBack();
 			await waitFor(element(by.id(`rooms-list-view-item-${room}`)))
 				.toBeVisible()

e2e/tests/assorted/06-status.spec.ts

@@ -52,10 +52,8 @@ describe('Status screen', () => {
 			await element(by.id('sidebar-custom-status-busy')).tap();
 		});
 
-		// TODO: flaky
 		it('should change status text', async () => {
 			await element(by.id('status-view-input')).replaceText('status-text-new');
-			await element(by.id('status-view-online')).tap(); // necessary to receive in-app notifications later
 			await element(by.id('status-view-submit')).tap();
 			await sleep(3000); // Wait until the loading hide
 			await waitFor(element(by.label('status-text-new')))

e2e/tests/assorted/07-changeserver.spec.ts

@@ -1,5 +1,5 @@
 import data from '../../data';
-import { navigateToLogin, login, checkServer, sleep } from '../../helpers/app';
+import { navigateToLogin, login, checkServer, sleep, expectValidRegisterOrRetry } from '../../helpers/app';
 
 const reopenAndCheckServer = async (server: string) => {
 	await device.launchApp({ permissions: { notifications: 'YES' }, newInstance: true });
@@ -65,10 +65,8 @@ describe('Change server', () => {
 		await element(by.id('register-view-username')).replaceText(data.registeringUser2.username);
 		await element(by.id('register-view-email')).replaceText(data.registeringUser2.email);
 		await element(by.id('register-view-password')).replaceText(data.registeringUser2.password);
-		await element(by.id('register-view-submit')).tap();
+		await element(by.id('register-view-password')).tapReturnKey();
-		await waitFor(element(by.id('rooms-list-view')))
+		await expectValidRegisterOrRetry(device.getPlatform());
-			.toBeVisible()
-			.withTimeout(60000);
 
 		await waitFor(element(by.id(`rooms-list-view-item-${data.groups.private.name}`)))
 			.toBeNotVisible()

e2e/tests/assorted/10-deleteserver.spec.ts

@@ -1,5 +1,13 @@
 import data from '../../data';
-import { sleep, navigateToLogin, login, checkServer, platformTypes, TTextMatcher } from '../../helpers/app';
+import {
+	sleep,
+	navigateToLogin,
+	login,
+	checkServer,
+	platformTypes,
+	TTextMatcher,
+	expectValidRegisterOrRetry
+} from '../../helpers/app';
 
 describe('Delete server', () => {
 	let alertButtonType: string;
@@ -41,10 +49,8 @@ describe('Delete server', () => {
 		await element(by.id('register-view-username')).replaceText(data.registeringUser3.username);
 		await element(by.id('register-view-email')).replaceText(data.registeringUser3.email);
 		await element(by.id('register-view-password')).replaceText(data.registeringUser3.password);
-		await element(by.id('register-view-submit')).tap();
+		await element(by.id('register-view-password')).tapReturnKey();
-		await waitFor(element(by.id('rooms-list-view')))
+		await expectValidRegisterOrRetry(device.getPlatform());
-			.toBeVisible()
-			.withTimeout(60000);
 
 		await checkServer(data.alternateServer);
 	});

e2e/tests/assorted/11-deeplinking.spec.ts

@@ -1,7 +1,14 @@
 import EJSON from 'ejson';
 
 import data from '../../data';
-import { tapBack, checkServer, navigateToRegister, platformTypes, TTextMatcher } from '../../helpers/app';
+import {
+	tapBack,
+	checkServer,
+	navigateToRegister,
+	platformTypes,
+	TTextMatcher,
+	expectValidRegisterOrRetry
+} from '../../helpers/app';
 import { get, login, sendMessage } from '../../helpers/data_setup';
 
 const DEEPLINK_METHODS = { AUTH: 'auth', ROOM: 'room' };
@@ -79,11 +86,8 @@ describe('Deep linking', () => {
 			await element(by.id('register-view-username')).replaceText(data.registeringUser4.username);
 			await element(by.id('register-view-email')).replaceText(data.registeringUser4.email);
 			await element(by.id('register-view-password')).replaceText(data.registeringUser4.password);
-			await element(by.type(scrollViewType)).atIndex(0).scrollTo('bottom');
+			await element(by.id('register-view-password')).tapReturnKey();
-			await element(by.id('register-view-submit')).tap();
+			await expectValidRegisterOrRetry(device.getPlatform());
-			await waitFor(element(by.id('rooms-list-view')))
-				.toBeVisible()
-				.withTimeout(10000);
 			await authAndNavigate();
 		});
 	});

e2e/tests/assorted/14-in-app-notification.spec.ts

@@ -6,27 +6,30 @@ const waitForInAppNotificationAnimation = async () => {
 	await sleep(500);
 };
 
+const sender = data.users.alternate;
+const receiver = data.users.inapp;
+
 describe('InApp Notification', () => {
 	let dmCreatedRid: string;
 
 	beforeAll(async () => {
 		await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
 		await navigateToLogin();
-		await login(data.users.regular.username, data.users.regular.password);
+		await login(receiver.username, receiver.password);
-		const result = await post('im.create', { username: data.users.alternate.username });
+		const result = await post('im.create', { username: sender.username }, receiver);
 		dmCreatedRid = result.data.room.rid;
 	});
 
 	describe('receive in RoomsListView', () => {
 		const text = 'Message in DM';
 		it('should tap on InApp Notification', async () => {
-			await sendMessage(data.users.alternate, dmCreatedRid, text);
+			await sendMessage(sender, dmCreatedRid, text);
 			await waitFor(element(by.id(`in-app-notification-${text}`)))
 				.toBeVisible()
 				.withTimeout(2000);
 			await waitForInAppNotificationAnimation();
 			await element(by.id(`in-app-notification-${text}`)).tap();
-			await checkRoomTitle(data.users.alternate.username);
+			await checkRoomTitle(sender.username);
 			await tapBack();
 		});
 	});
@@ -35,13 +38,13 @@ describe('InApp Notification', () => {
 		const text = 'Another msg';
 		it('should receive and tap InAppNotification while in another room', async () => {
 			await navigateToRoom(data.userRegularChannels.detoxpublic.name);
-			await sendMessage(data.users.alternate, dmCreatedRid, text);
+			await sendMessage(sender, dmCreatedRid, text);
 			await waitFor(element(by.id(`in-app-notification-${text}`)))
 				.toExist()
 				.withTimeout(2000);
 			await waitForInAppNotificationAnimation();
 			await element(by.id(`in-app-notification-${text}`)).tap();
-			await checkRoomTitle(data.users.alternate.username);
+			await checkRoomTitle(sender.username);
 		});
 
 		it('should tap back and go back to RoomsListView', async () => {

e2e/tests/onboarding/04-createuser.spec.ts

@@ -1,4 +1,4 @@
-import { navigateToRegister } from '../../helpers/app';
+import { navigateToRegister, expectValidRegisterOrRetry } from '../../helpers/app';
 import data from '../../data';
 
 describe('Create user screen', () => {
@@ -6,7 +6,6 @@ describe('Create user screen', () => {
 		await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
 		await navigateToRegister();
 	});
-
 	describe('Usage', () => {
 		it('should register', async () => {
 			await element(by.id('register-view-name')).replaceText(data.registeringUser.username);
@@ -17,9 +16,8 @@ describe('Create user screen', () => {
 			await element(by.id('register-view-email')).tapReturnKey();
 			await element(by.id('register-view-password')).replaceText(data.registeringUser.password);
 			await element(by.id('register-view-password')).tapReturnKey();
-			await waitFor(element(by.id('rooms-list-view')))
+
-				.toBeVisible()
+			await expectValidRegisterOrRetry(device.getPlatform());
-				.withTimeout(60000);
 		});
 	});
 });

e2e/tests/room/02-room.spec.ts

@@ -241,7 +241,7 @@ describe('Room screen', () => {
 					.withTimeout(4000);
 				await tryTapping(element(by.id(`mention-item-${username}`)), 2000);
 				await expect(element(by.id('messagebox-input'))).toHaveText(`${messageMention} `);
-				await tryTapping(element(by.id('messagebox-input')), 2000);
+				// await tryTapping(element(by.id('messagebox-input')), 2000);
 				if (device.getPlatform() === 'ios') {
 					await element(by.id('messagebox-input')).typeText(message);
 					await element(by.id('messagebox-send-message')).tap();
@@ -342,7 +342,7 @@ describe('Room screen', () => {
 				await waitFor(element(by.id('emoji-picker-tab-emoji')))
 					.toExist()
 					.withTimeout(2000);
-				await element(by.id('action-sheet-handle')).swipe('up', 'fast', 1);
+				// await element(by.id('action-sheet-handle')).swipe('up', 'fast', 1);
 				await element(by.id('emoji-picker-tab-emoji')).tap();
 				await waitFor(element(by.id('emoji-grinning')))
 					.toExist()
@@ -364,7 +364,7 @@ describe('Room screen', () => {
 				await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
 				await element(by.id('add-reaction')).tap();
 				await waitFor(element(by.id('emoji-searchbar-input')))
-					.toExist()
+					.toBeVisible()
 					.withTimeout(2000);
 				await element(by.id('emoji-searchbar-input')).replaceText('laughing');
 				await waitFor(element(by.id('emoji-laughing')))

e2e/tests/room/03-roomactions.spec.ts

@@ -8,15 +8,46 @@ import {
 	sleep,
 	searchRoom,
 	mockMessage,
-	starMessage,
-	pinMessage,
 	platformTypes,
 	TTextMatcher,
-	tapAndWaitFor
+	tapAndWaitFor,
+	tryTapping
 } from '../../helpers/app';
 
 const { sendMessage } = require('../../helpers/data_setup');
 
+async function starMessage(message: string) {
+	const deviceType = device.getPlatform();
+	const { textMatcher } = platformTypes[deviceType];
+	const messageLabel = `${data.random}${message}`;
+	await waitFor(element(by[textMatcher](messageLabel)).atIndex(0)).toExist();
+	await tryTapping(element(by[textMatcher](messageLabel)).atIndex(0), 2000, true);
+	await waitFor(element(by.id('action-sheet')))
+		.toExist()
+		.withTimeout(2000);
+	await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
+	await element(by[textMatcher]('Star')).atIndex(0).tap();
+	await waitFor(element(by.id('action-sheet')))
+		.not.toExist()
+		.withTimeout(5000);
+}
+
+async function pinMessage(message: string) {
+	const deviceType = device.getPlatform();
+	const { textMatcher } = platformTypes[deviceType];
+	const messageLabel = `${data.random}${message}`;
+	await waitFor(element(by[textMatcher](messageLabel)).atIndex(0)).toExist();
+	await tryTapping(element(by[textMatcher](messageLabel)).atIndex(0), 2000, true);
+	await waitFor(element(by.id('action-sheet')))
+		.toExist()
+		.withTimeout(2000);
+	await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
+	await element(by[textMatcher]('Pin')).atIndex(0).tap();
+	await waitFor(element(by.id('action-sheet')))
+		.not.toExist()
+		.withTimeout(5000);
+}
+
 async function navigateToRoomActions(type: string) {
 	let room;
 	if (type === 'd') {

e2e/tests/room/04-discussion.spec.ts

@@ -93,6 +93,7 @@ describe('Discussion', () => {
 
 		it('should create discussion', async () => {
 			const discussionName = `${data.random}message`;
+			await element(by[textMatcher](discussionName)).atIndex(0).tap();
 			await element(by[textMatcher](discussionName)).atIndex(0).longPress();
 			await waitFor(element(by.id('action-sheet')))
 				.toExist()

e2e/tests/room/05-threads.spec.ts

@@ -83,6 +83,7 @@ describe('Threads', () => {
 			const thread = `${data.random}thread`;
 			it('should create thread', async () => {
 				await mockMessage('thread');
+				await element(by[textMatcher](thread)).atIndex(0).tap();
 				await element(by[textMatcher](thread)).atIndex(0).longPress();
 				await expect(element(by.id('action-sheet'))).toExist();
 				await expect(element(by.id('action-sheet-handle'))).toBeVisible();

e2e/tests/room/09-jumptomessage.spec.ts

@@ -168,7 +168,7 @@ describe('Room', () => {
 		await waitFor(element(by[textMatcher]('50')))
 			.toExist()
 			.withTimeout(5000);
-		await element(by.id('room-view-messages')).atIndex(0).swipe('up', 'slow', 0.4);
+		await element(by.id('room-view-messages')).atIndex(0).swipe('up', 'slow', 0.3);
 		await waitFor(element(by[textMatcher]('Load Newer')))
 			.toExist()
 			.withTimeout(5000);

e2e/tests/room/10-ignoreuser.spec.ts

@@ -68,6 +68,7 @@ describe('Ignore/Block User', () => {
 
 			it('should unblock user', async () => {
 				await navigateToInfoView();
+				await sleep(300); // wait for navigation animation
 				await tapAndWaitFor(
 					element(by.id('room-info-view-ignore')),
 					element(by.id('room-info-view-ignore').withDescendant(by[textMatcher]('Block user'))),
@@ -78,7 +79,9 @@ describe('Ignore/Block User', () => {
 					.toBeVisible()
 					.withTimeout(5000);
 				await tapBack();
-				await expect(element(by.id('messagebox'))).toBeVisible();
+				await waitFor(element(by.id('messagebox')))
+					.toBeVisible()
+					.withTimeout(2000);
 				await tapBack();
 			});
 		});

package.json

@@ -22,9 +22,9 @@
 		"e2e:android-build": "yarn detox build -c android.emu.release",
 		"e2e:android-test": "yarn detox test -c android.emu.release --cleanup --headless --take-screenshots failing",
 		"e2e:ios-build": "yarn detox build -c ios.sim.release",
-		"e2e:ios-test": "yarn detox test -c ios.sim.release --cleanup --headless --take-screenshots failing",
+		"e2e:ios-test": "TEST_SESSION=`openssl rand -hex 10` yarn detox test -c ios.sim.release --record-videos failing",
 		"e2e:ios-build-debug": "yarn detox build -c ios.sim.debug",
-		"e2e:ios-test-debug": "TEST_SESSION=`openssl rand -hex 10` yarn detox test -c ios.sim.debug --take-screenshots failing -R"
+		"e2e:ios-test-debug": "TEST_SESSION=`openssl rand -hex 10` yarn detox test -c ios.sim.debug --take-screenshots failing"
 	},
 	"lint-staged": {
 		"*.{js,ts,tsx}": [