Convo: fetched collections can be huge - add a slight delay between storing items if we did not fetch them from the network to allow the DB to do its thing

Zotlabs/Daemon/Convo.php

@@ -6,6 +6,7 @@ use Zotlabs\Lib\Activity;
 use Zotlabs\Lib\ActivityStreams;
 use Zotlabs\Lib\ASCollection;
 use Zotlabs\Lib\ASCache;
+use Zotlabs\Lib\Config;
 
 class Convo {
 
@@ -33,6 +34,7 @@ class Convo {
 		}
 
 		$force = $argv[4] ?? false;
+		$interval = Config::Get('queueworker', 'queue_interval', 500000);
 
 		foreach ($channels as $channel_id) {
 			$channel = channelx_by_n($channel_id);
@@ -46,6 +48,8 @@ class Convo {
 			}
 
 			foreach ($messages as $message) {
+				$network_fetch = false;
+
 				if (is_string($message)) {
 					$cached = ASCache::Get($message);
 					if ($cached) {
@@ -54,6 +58,8 @@ class Convo {
 					}
 					else {
 						// logger('convo_fetching: ' . $message);
+						$network_fetch = true;
+
 						$data = Activity::fetch($message, $channel);
 						if ($data) {
 							ASCache::Set($message, $data);
@@ -65,6 +71,12 @@ class Convo {
 					$data = $message;
 				}
 
+				if (!$network_fetch) {
+					// Add some delay so that the DB will not be overwhelmed
+					// Fetched from network will already have a slight delay
+					usleep($interval);
+				}
+
 				$AS = new ActivityStreams($data);
 				if ($AS->is_valid() && is_array($AS->obj)) {
 					$item = Activity::decode_note($AS);