diff --git a/CHANGES.rst b/CHANGES.rst
index 4f393ff59..28604cdde 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -9,6 +9,7 @@ Changes to be released in next version
  * Rooms Tab: Remove the directory section (#4521).
  * Notifications: Show decrypted content is enabled by default (#4519).
  * People Tab: Remove the local contacts section (#4523).
+ * Contacts: Delay access to local contacts until they're needed for display (#4616).
 
 🐛 Bugfix
  * Room: Fixed mentioning users from room info member details (#4583)
diff --git a/Riot/Modules/Application/LegacyAppDelegate.m b/Riot/Modules/Application/LegacyAppDelegate.m
index ae55cf538..a6df6cb48 100644
--- a/Riot/Modules/Application/LegacyAppDelegate.m
+++ b/Riot/Modules/Application/LegacyAppDelegate.m
@@ -1854,16 +1854,6 @@ NSString *const AppDelegateUniversalLinkDidChangeNotification = @"AppDelegateUni
             [account addObserver:self forKeyPath:@"enableInAppNotifications" options:0 context:nil];
         }
         
-        // Load the local contacts on first account creation.
-        if ([MXKAccountManager sharedManager].accounts.count == 1)
-        {
-            dispatch_async(dispatch_get_main_queue(), ^{
-                
-                [self refreshLocalContacts];
-                
-            });
-        }
-        
         [self.delegate legacyAppDelegate:self didAddAccount:account];
     }];
     
@@ -1976,14 +1966,6 @@ NSString *const AppDelegateUniversalLinkDidChangeNotification = @"AppDelegateUni
         // during this blocking task.
         dispatch_after(dispatch_walltime(DISPATCH_TIME_NOW, 0.3 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
             [[MXKContactManager sharedManager] addMatrixSession:mxSession];
-
-            // Load the local contacts on first account
-            if ([MXKAccountManager sharedManager].accounts.count == 1)
-            {
-                dispatch_async(dispatch_get_main_queue(), ^{
-                    [self refreshLocalContacts];
-                });
-            }
         });
 
         // Register the session to the widgets manager
@@ -2939,54 +2921,6 @@ NSString *const AppDelegateUniversalLinkDidChangeNotification = @"AppDelegateUni
     }];
 }
 
-- (void)refreshLocalContacts
-{
-    if (!BuildSettings.allowLocalContactsAccess)
-    {
-        return;
-    }
-    
-    // Do not scan local contacts in background if the user has not decided yet about using
-    // an identity server
-    BOOL doRefreshLocalContacts = NO;
-    for (MXSession *session in mxSessionArray)
-    {
-        if (session.hasAccountDataIdentityServerValue)
-        {
-            doRefreshLocalContacts = YES;
-            break;
-        }
-    }
-
-    // Check whether the application is allowed to access the local contacts.
-    if (doRefreshLocalContacts
-        && [CNContactStore authorizationStatusForEntityType:CNEntityTypeContacts] == CNAuthorizationStatusAuthorized)
-    {
-        // Check the user permission for syncing local contacts. This permission was handled independently on previous application version.
-        if (![MXKAppSettings standardAppSettings].syncLocalContacts)
-        {
-            // Check whether it was not requested yet.
-            if (![MXKAppSettings standardAppSettings].syncLocalContactsPermissionRequested)
-            {
-                [MXKAppSettings standardAppSettings].syncLocalContactsPermissionRequested = YES;
-                
-                [MXKContactManager requestUserConfirmationForLocalContactsSyncInViewController:self.presentedViewController completionHandler:^(BOOL granted) {
-                    
-                    if (granted)
-                    {
-                        // Allow local contacts sync in order to discover matrix users.
-                        [MXKAppSettings standardAppSettings].syncLocalContacts = YES;
-                    }
-                    
-                }];
-            }
-        }
-        
-        // Refresh the local contacts list.
-        [[MXKContactManager sharedManager] refreshLocalContacts];
-    }
-}
-
 #pragma mark - Matrix Groups handling
 
 - (void)showGroup:(MXGroup*)group withMatrixSession:(MXSession*)mxSession
diff --git a/Riot/Modules/Contacts/ContactsTableViewController.m b/Riot/Modules/Contacts/ContactsTableViewController.m
index 5437cd149..c9af2e4c5 100644
--- a/Riot/Modules/Contacts/ContactsTableViewController.m
+++ b/Riot/Modules/Contacts/ContactsTableViewController.m
@@ -174,6 +174,15 @@
     [self refreshContactsTable];
 }
 
+- (void)viewDidAppear:(BOOL)animated
+{
+    [super viewDidAppear:animated];
+    
+    // Load the local contacts for display.
+    // In viewDidAppear as it may trigger a request for contacts access.
+    [self refreshLocalContacts];
+}
+
 - (void)viewWillDisappear:(BOOL)animated
 {
     [super viewWillDisappear:animated];
@@ -212,6 +221,54 @@
     }
 }
 
+- (void)refreshLocalContacts
+{
+    if (!BuildSettings.allowLocalContactsAccess)
+    {
+        return;
+    }
+    
+    // Do not scan local contacts in background if the user has not decided yet about using
+    // an identity server
+    BOOL doRefreshLocalContacts = NO;
+    for (MXSession *session in self.mxSessions)
+    {
+        if (session.hasAccountDataIdentityServerValue)
+        {
+            doRefreshLocalContacts = YES;
+            break;
+        }
+    }
+
+    // Check whether the application is allowed to access the local contacts.
+    if (doRefreshLocalContacts
+        && [CNContactStore authorizationStatusForEntityType:CNEntityTypeContacts] == CNAuthorizationStatusAuthorized)
+    {
+        // Check the user permission for syncing local contacts. This permission was handled independently on previous application version.
+        if (![MXKAppSettings standardAppSettings].syncLocalContacts)
+        {
+            // Check whether it was not requested yet.
+            if (![MXKAppSettings standardAppSettings].syncLocalContactsPermissionRequested)
+            {
+                [MXKAppSettings standardAppSettings].syncLocalContactsPermissionRequested = YES;
+                
+                [MXKContactManager requestUserConfirmationForLocalContactsSyncInViewController:self completionHandler:^(BOOL granted) {
+                    
+                    if (granted)
+                    {
+                        // Allow local contacts sync in order to discover matrix users.
+                        [MXKAppSettings standardAppSettings].syncLocalContacts = YES;
+                    }
+                    
+                }];
+            }
+        }
+        
+        // Refresh the local contacts list.
+        [[MXKContactManager sharedManager] refreshLocalContacts];
+    }
+}
+
 - (void)refreshContactsTable
 {
     [self.contactsTableView reloadData];