TheRON/core
1
0
Fork 0
mirror of https://framagit.org/hubzilla/core.git synced 2026-06-21 00:52:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add support filesystem storage for xchan profile photos

Browse Source
This commit is contained in:
Max Kostikov
2021-01-24 21:03:34 +01:00
committed by Mario
parent 6b0c61ac6b
commit 552796286e
7 changed files with 167 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
Zotlabs/Lib/Hashpath.php Normal file
View File

@@ -0,0 +1,55 @@
<?php
namespace Zotlabs\Lib;
/*
* Zotlabs\Lib\Hashpath
*
* Creates hashed directory structures for fast access and resistance to overloading any single directory with files.
*
* Takes a $hash which could be any string
* a $prefix which is where to place the hash directory in the filesystem, default is current directory
* use an empty string for $prefix to place hash directories directly off the root directory
* an optional $depth and $slice (default is 2) to indicate the hash level
* $depth = 1, 256 directories, suitable for < 384K records/files
* $depth = 2, 65536 directories, suitable for < 98M records/files
* $depth = 3, 16777216 directories, suitable for < 2.5B records/files
* ...
* The total number of records anticipated divided by the number of hash directories should generally be kept to
* less than 1500 entries for optimum performance though this varies by operating system and filesystem type.
* ext4 uses 32 bit inode numbers (~4B record limit) so use caution or alternative filesystem types with $depth above 3.
* an optional $mkdir (boolean) to recursively create the directory (ignoring errors) before returning
*
* examples: for a $hash of 'abcdefg' and prefix of 'path' the following paths are returned for $depth = 1 and $depth = 3
* path/7d/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a
* path/7d/1a/54/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a
*
* see also: boot.php:os_mkdir() - here we provide the equivalent of mkdir -p with permissions of 770.
*
*/
class Hashpath {
static function path($hash, $prefix = '.', $depth = 1, $slice = 2, $mkdir = true, $alg = false) {
if ($alg)
$hash = hash($alg, $hash);
$start = 0;
if ($depth < 1)
$depth = 1;
$sluglen = $depth * $slice;
do {
$slug = substr($hash, $start, $slice);
$prefix .= '/' . $slug;
$start += $slice;
$sluglen -= $slice;
}
while ($sluglen);
if ($mkdir)
os_mkdir($prefix, STORAGE_DEFAULT_PERMISSIONS, true);
return $prefix . '/' . $hash;
}
}

6
Zotlabs/Module/Photo.php
View File

@@ -194,9 +194,9 @@ class Photo extends \Zotlabs\Web\Controller {
$mimetype = $e[0]['mimetype'];
$modified = strtotime($e[0]['edited'] . 'Z');
if(intval($e[0]['os_storage'])) {
if(intval($e[0]['os_storage']))
$streaming = $data;
}
if($e[0]['allow_cid'] != '' || $e[0]['allow_gid'] != '' || $e[0]['deny_gid'] != '' || $e[0]['deny_gid'] != '')
$prvcachecontrol = 'no-store, no-cache, must-revalidate';
}
@@ -282,7 +282,7 @@ class Photo extends \Zotlabs\Web\Controller {
header("Content-Length: " . (isset($filesize) ? $filesize : strlen($data)));
// If it's a file resource, stream it.
if($streaming && $channel) {
if($streaming) {
if(strpos($streaming,'store') !== false)
$istream = fopen($streaming,'rb');
else

29
Zotlabs/Photo/PhotoDriver.php
View File

@@ -2,6 +2,8 @@
namespace Zotlabs\Photo;
use Zotlabs\Lib\Hashpath;
/**
* @brief Abstract photo driver class.
*
@@ -505,18 +507,25 @@ abstract class PhotoDriver {
* @return boolean
*/
public function storeThumbnail($arr, $scale = 0) {
// We only process thumbnails here
if($scale == 0)
return false;
$arr['imgscale'] = $scale;
if(boolval(get_config('system','filesystem_storage_thumbnails', 0))) {
$channel = channelx_by_n($arr['uid']);
// We only process thumbnails here
if($scale == 0)
return false;
$arr['imgscale'] = $scale;
if(boolval(get_config('system','photo_storage_type', 1))) {
$arr['os_storage'] = 1;
$arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale;
if(! $this->saveImage($arr['os_syspath']))
if (array_key_exists('uid', $arr) && ! in_array($scale, [ PHOTO_RES_PROFILE_300, PHOTO_RES_PROFILE_80, PHOTO_RES_PROFILE_48 ])) {
$channel = channelx_by_n($arr['uid']);
$arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale;
}
else
$arr['os_syspath'] = Hashpath::path($arr['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $scale;
if (! $this->saveImage($arr['os_syspath']))
return false;
}
else

2
doc/hidden_configs.bb
View File

@@ -62,7 +62,7 @@ Options are:
[*= system.email_notify_icon_url ] URL of image (32x32) to display in email notifications (HTML bodies).
[*= system.expire_delivery_reports ] Expiration in days for delivery reports - default 10
[*= system.expire_limit ] Don't expire any more than this number of posts per channel per expiration run to keep from exhausting memory. Default 5000.
[*= system.filesystem_storage_thumbnails ] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2
[*= system.photo_storage_type] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2
[*= system.hidden_version_siteinfo ] If true, do not report the software version on siteinfo pages (system.hide_version also hides the version on these pages, this setting *only* hides the version on siteinfo pages).
[*= system.hide_help ] Don't display help documentation link in nav bar
[*= system.hide_in_statistics ] Tell the red statistics servers to completely hide this hub in hub lists.

2
include/import.php
View File

@@ -1469,7 +1469,7 @@ function sync_files($channel, $files) {
fclose($fp);
// Override remote hub thumbnails storage settings
if(! boolval(get_config('system','filesystem_storage_thumbnails', 0))) {
if(! boolval(get_config('system','photo_storage_type', 1))) {
$p['os_storage'] = 0;
$p['content'] = file_get_contents($stored_image);
@unlink($stored_image);

43
include/photo/photo_driver.php
View File

@@ -205,7 +205,11 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) {
if($thing)
$hash = photo_new_resource();
else {
$r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = 4 LIMIT 1", dbesc($xchan), intval(PHOTO_XCHAN));
$r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = %d LIMIT 1",
dbesc($xchan),
intval(PHOTO_XCHAN),
intval(PHOTO_RES_PROFILE_300)
);
if($r) {
$hash = $r[0]['resource_id'];
$modified = $r[0]['edited'];
@@ -316,26 +320,20 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) {
'filename' => basename($photo),
'album' => $album,
'photo_usage' => $flags,
'imgscale' => 4,
'edited' => $modified,
'description' => (array_key_exists('etag', $hdrs) ? $hdrs['etag'] : ''),
'expires' => gmdate('Y-m-d H:i:s', (isset($expires) ? $expires : time() + 86400))
];
$r = $img->save($p);
if($r === false)
$photo_failure = true;
$r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300);
$img->scaleImage(80);
$p['imgscale'] = 5;
$r = $img->save($p);
if($r === false)
$photo_failure = true;
$r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80);
$img->scaleImage(48);
$p['imgscale'] = 6;
$r = $img->save($p);
if($r === false)
$r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48);
if($r1 === false || $r2 === false || $r3 === false)
$photo_failure = true;
$photo = z_root() . '/photo/' . $hash . '-4';
@@ -424,31 +422,26 @@ function import_channel_photo($photo, $type, $aid, $uid) {
'resource_id' => $hash,
'filename' => $filename,
'album' => t('Profile Photos'),
'photo_usage' => PHOTO_PROFILE,
'imgscale' => 4,
'photo_usage' => PHOTO_PROFILE
];
// photo size
$img->scaleImageSquare(300);
$r = $img->save($p);
if($r === false)
$photo_failure = true;
$r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300);
// thumb size
$img->scaleImage(80);
$p['imgscale'] = 5;
$r = $img->save($p);
if($r === false)
$photo_failure = true;
$r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80);
// micro size
$img->scaleImage(48);
$p['imgscale'] = 6;
$r = $img->save($p);
if($r === false)
$r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48);
if($r1 === false || $r2 === false || $r3 === false)
$photo_failure = true;
} else {
}
else {
logger('Invalid image.');
$photo_failure = true;
}

113
util/storageconv
View File

@@ -18,6 +18,8 @@ require_once('include/cli_startup.php');
cli_startup();
use Zotlabs\Lib\Hashpath;
if($argc == 1) {
usage();
killme();
@@ -25,15 +27,20 @@ if($argc == 1) {
if($argc == 2) {
$storage = (intval(get_config('system','filesystem_storage_thumbnails', 0)) > 0 ? 1 : 0);
$storage = (intval(get_config('system','photo_storage_type', 1)) > 0 ? 1 : 0);
echo 'Current storage set to: ' . ($storage ? 'filesystem' : 'SQL database') . PHP_EOL;
switch($argv[1]) {
case 'stats':
$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
echo 'Local images: ' . $x[0]['qty'] . PHP_EOL;
$x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND imgscale > 0");
echo 'Thumbnails total: ' . $x[0]['qty'] . PHP_EOL;
$x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND os_storage != %d AND imgscale > 0",
$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND imgscale > 0");
echo 'Image thumbnails: ' . $x[0]['qty'] . PHP_EOL;
$xx = intval($x[0]['qty']);
$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage IN (1, 2)");
echo 'Imported profiles thumbnails: ' . $x[0]['qty'] . PHP_EOL;
$xx += intval($x[0]['qty']);
echo 'Thumbnails total: ' . $xx . PHP_EOL;
$x = q("SELECT COUNT(id) AS qty FROM photo WHERE os_storage != %d AND imgscale > 0",
$storage
);
echo 'Thumbnails to convert: ' . $x[0]['qty'] . PHP_EOL;
@@ -41,87 +48,108 @@ if($argc == 2) {
case 'fs':
if($storage == 0) {
echo 'Please set system.filesystem_storage_thumbnails to 1 before move thumbnails to filesystem storage' . PHP_EOL;
echo 'Please set system.photo_storage_type to 1 before move thumbnails to filesystem storage' . PHP_EOL;
break;
}
$x = q("SELECT resource_id, content FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
$cur_id = 0;
$i = 0;
if($x) {
foreach($x as $xx) {
$n = q("SELECT id, imgscale, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0",
dbesc($xx['resource_id']),
$storage
);
$r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 0 AND imgscale > 0");
$max_num = $r[0]['max_num'];
$img_path = dbunescbin($xx['content']);
foreach($n as $nn) {
echo '.';
$filename = $img_path . '-' . $nn['imgscale'];
while ($i < $max_num) {
if(! file_put_contents($filename, dbunescbin($nn['content']))) {
echo 'Failed to save file ' . $filename . PHP_EOL;
$x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 0 AND imgscale > 0 ORDER BY id LIMIT 10",
intval($cur_id)
);
if($x) {
foreach($x as $xx) {
if ($xx['uid'] == 0 || in_array($xx['imgscale'], [4, 5, 6]))
$filename = Hashpath::path($xx['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $xx['imgscale'];
else {
$z = q("SELECT channel_address FROM channel WHERE channel_id = %d",
intval($xx['uid'])
);
$filename = 'store/' . $z[0]['channel_address'] . '/' . $xx['resource_id'] . '-' . $xx['imgscale'];
}
if(! file_put_contents($filename, dbunescbin($xx['content']))) {
echo PHP_EOL . 'Failed to save file ' . $filename . PHP_EOL;
continue;
}
$z = q("UPDATE photo SET content = '%s', os_storage = 1 WHERE id = %d",
dbescbin($filename),
intval($nn['id'])
intval($xx['id'])
);
if(! $z) {
@unlink($filename);
echo 'Failed to update metadata for saved file ' . $filename . PHP_EOL;
echo PHP_EOL . 'Failed to update metadata for saved file ' . $filename . PHP_EOL;
}
$cur_id = $xx['id'];
echo '.';
$i++;
}
}
echo ($i % 100 == 0 ? $i : '');
}
echo $i . PHP_EOL . 'Total thumbnails processed: ' . $i;
break;
case 'db':
if($storage == 1) {
echo 'Please set system.filesystem_storage_thumbnails to 0 before move thumbnails to SQL database storage' . PHP_EOL;
echo 'Please set system.photo_storage_type to 0 before move thumbnails to SQL database storage' . PHP_EOL;
break;
}
$x = q("SELECT resource_id FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
if($x) {
foreach($x as $xx) {
$cur_id = 0;
$i = 0;
$n = q("SELECT id, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0",
dbesc($xx['resource_id']),
$storage
);
foreach($n as $nn) {
$r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 1 AND imgscale > 0");
$max_num = $r[0]['max_num'];
while ($i < $max_num) {
$x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 1 AND imgscale > 0 ORDER BY id LIMIT 10",
intval($cur_id)
);
if($x) {
foreach($x as $xx) {
echo '.';
$filename = dbunescbin($nn['content']);
$filename = dbunescbin($xx['content']);
$content = file_get_contents($filename);
if($content) {
$z = q("UPDATE photo SET content = '%s', os_storage = 0 WHERE id = %d",
dbescbin($content),
intval($nn['id'])
intval($xx['id'])
);
if(! $z) {
echo 'Failed to update stored file metadata ' . $filename . PHP_EOL;
echo PHP_EOL . 'Failed to update stored file metadata ' . $filename . PHP_EOL;
continue;
}
@unlink($filename);
}
else
echo 'Can not read file contents ' . $filename . PHP_EOL;
echo PHP_EOL . 'Can not read file contents ' . $filename . PHP_EOL;
$cur_id = $xx['id'];
echo '.';
$i++;
}
}
echo ($i % 100 == 0 ? $i : '');
}
echo $i . PHP_EOL . 'Total files processed: ' . $i;
break;
default:
@@ -129,6 +157,5 @@ if($argc == 2) {
return;
}
echo PHP_EOL;
}