Yuba/Yuba.php

#!/usr/bin/php
<?php

// Yuba
//                                      //
//////////////////////////////////////////
$version = "0.6.0.1";

ini_set('memory_limit', '4096M');
date_default_timezone_set("America/Los_Angeles");
$time_start = microtime(true);

include('ProgressBar.php');

// Path & application variables
//////////////////////////////////////////

if (in_array("-nohash", $argv)) { $wopt_hash = 0; } else { $wopt_hash = 1; }
if (in_array("-nothumbs", $argv)) { $wopt_thumbs = 0; } else { $wopt_thumbs = 1; }

$zpath = realpath(@$argv[1]);
$bdest = realpath(@$argv[2]);
if (!is_dir($zpath) | !is_dir($bdest)) { echo "Usage: walk <path> <dest>"; die; }

// Check for bundle
if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); }
if (is_writable($zpath)) { $wopt_paranoid = 1; } else { $wopt_paranoid = 0; }

$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle";
if (!is_dir($bpath)) { mkdir($bpath); }
if (!is_dir($bpath."/thumbs")) { mkdir($bpath."/thumbs"); }

$wopt_hash_limit = 1; // don't hash if exceeds in gigs, 0 for unlimited
$wopt_thumb_size = "512";

// Treat these directories as files
$wopt_bundles = array(	"app",
						"bundle", 
						"sparsebundle", 
						"photoslibrary", 
						"aplibrary", 
						"apvault", 
						"abbu", 
						"calendar",
						"framework",
						"plugin",
						"kext", 
						"rtfd"
						);
							
foreach ($wopt_bundles as $bundle) {
	$wopt_nodescend[] = "*.".$bundle;
	}

// Ignore matching files and directories
$wopt_ignore = array(		".DS_Store",
							".DocumentRevisions-V100", 
							".Spotlight-V100",
							".TemporaryItems",
							".apdisk",
							".com.apple.timemachine.donotpresent",
							".fseventsd",
							".metadata-never-index",
							".neofinder.abemeda.volinfo.xml"
							);

$max_label = 50;

// Metadata tools
$bin_gfi = "/Applications/Xcode.app/Contents/Developer/usr/bin/GetFileInfo";
$bin_mediainfo = "/opt/local/bin/mediainfo";
$bin_exiftool = "/opt/local/bin/exiftool";
$bin_tq = "/opt/local/bin/ql-thumbnail";
$bin_tv = "/opt/local/bin/vipsthumbnail";
$bin_tf = "/usr/local/bin/ffmpegthumbnailer";

// Media extensions
//////////////////////////////////////////

$t_files['ffmpeg'] = array(		"mkv",
								"avi",
								"mpeg",
								"mpg",
								"vob",
								"mp4",
								"m4v",
								"m2v",
								"m2ts",
								"asf",
								"wmv",
								"rm",
								"divx",
								"fla",
								"flv",
								"webm"  );
								
$t_files['vips'] = array(		"jpg",
								"jpeg",
								"tif",
								"tiff",
								"gif",
								"psd",
								"png"  );
								
$m_files = array(	"mkv",
					"ogg",
					"avi",
					"wav",
					"mpeg",
					"mpg",
					"vob",
					"mp4",
					"m2v",
					"mp3",
					"asf",
					"wma",
					"wmv",
					"qt",
					"mov",
					"rm",
					"ifo",
					"ac3",
					"dts",
					"aac",
					"ape",
					"flac",
					"aiff", 
					"m2ts"	);

$e_files = array(	"ai",
					"aiff",
					"ape",
					"asf",
					"avi",
					"bmp",
					"divx",
					"dng",
					"doc",
					"docx",
					"eps",
					"epub",
					"exe",
					"exif",
					"fla",
					"flac",
					"flv",
					"gif",
					"icc",
					"iso",
					"jpg",
					"jpeg",
					"m2ts",
					"m4a",
					"m4b",
					"m4v",
					"mkv",
					"mobi",
					"azw",
					"azw3",
					"mov",
					"qt",
					"mp3",
					"mp4",
					"mpeg",
					"mpg",
					"m2v",
					"nef",
					"numbers",
					"ogg",
					"pages",
					"pdf",
					"pict",
					"png",
					"ppm",
					"ppt",
					"psd",
					"psb",
					"qif",
					"raw",
					"rtf",
					"sr2",
					"srf",
					"svg",
					"swf",
					"tiff",
					"tif",
					"torrent",
					"vcf",
					"vob",
					"wav",
					"webm",
					"wma",
					"wmv",
					"xls",
					"xlsx",
					"xmp",
					"zip"  );

foreach ($e_files as $ext) { $e_files[] = strtoupper($ext); }
foreach ($m_files as $ext) { $m_files[] = strtoupper($ext); }
foreach ($t_files['ffmpeg'] as $ext) { $t_files['ffmpeg'][] = strtoupper($ext); }
foreach ($t_files['vips'] as $ext) { $t_files['vips'][] = strtoupper($ext); }

// Functions
//////////////////////////////////////////

/*
function getParents($zpath, $pathname) {
	$path = dirname($pathname);
	$parts = explode("/",trim(substr($path,strlen(basename($zpath))),"/"));
	foreach ($parts as $index => $part) {
		$parents[] = array($part, md5($zpath."/".implode("/",array_slice($parts, 0, $index+1))));
		}
	return $parents;
	}
*/

function shortlabel($pathname, $max) {
	$basename = basename($pathname);
	$suffix = "(...).".pathinfo($basename,PATHINFO_EXTENSION);
	if (strlen($basename) > $max) {
		return substr($basename, 0, ($max-strlen($suffix))).$suffix;
		} else {
		return $basename;
		}
	}

function human_filesize($bytes, $decimals = 2) {
    $size = array('B','kB','MB','GB','TB','PB','EB','ZB','YB');
    $factor = floor((strlen($bytes) - 1) / 3);
    return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$size[$factor];
	}

function stringPrint($string) {
	echo $string.@str_repeat(" ", (10-strlen($string)));
	}

function getWoptString() {
	global $wopt_bundles, $wopt_ignore, $wopt_hash, $wopt_hash_limit, $wopt_mediainfo, $wopt_exiftool, $wopt_thumbs, $wopt_thumb_size, $wopt_paranoid;
	return array(	array("bundles", $wopt_bundles), 
					array("ignore", $wopt_ignore),
					array("hash", $wopt_hash),
					array("wopt_hash_limit", $wopt_hash_limit), 
					array("mediainfo", $wopt_mediainfo),
					array("exiftool", $wopt_exiftool),
					array("thumbs", $wopt_thumbs),
					array("thumb_size", $wopt_thumb_size),
					array("wopt_paranoid", $wopt_paranoid),
					);
	}

class plistParser extends XMLReader {
	public function parseString($string) { $this->XML($string); return $this->process(); }
    private function process() {
    	$this->read();
    	if($this->nodeType !== XMLReader::DOC_TYPE || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 2); }
		if(!$this->next("plist") || $this->nodeType !== XMLReader::ELEMENT || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 3); }
		$plist = array(); while($this->read()) { if($this->nodeType == XMLReader::ELEMENT) { $plist[] = $this->parse_node(); } }
		if(count($plist) == 1 && $plist[0]) { return $plist[0]; } else { return $plist; }
		}
	private function parse_node() {
		if($this->nodeType !== XMLReader::ELEMENT) return;
		switch($this->name) {
			case 'data': return base64_decode($this->getNodeText()); break;
			case 'real': return floatval($this->getNodeText()); break;
			case 'string': return $this->getNodeText(); break;
			case 'integer': return intval($this->getNodeText()); break;
			case 'date': return $this->getNodeText(); break;
			case 'true': return true; break;
			case 'false': return false; break;
			case 'array': return $this->parse_array(); break;
			case 'dict': return $this->parse_dict(); break;
			default: throw new Exception(sprintf("Not a valid plist. %s is not a valid type", $this->name), 4);
			}			
		}
	private function parse_dict() {
		$array = array(); $this->nextOfType(XMLReader::ELEMENT);
		do { if($this->nodeType !== XMLReader::ELEMENT || $this->name !== "key") { if(!$this->next("key")) { return $array; } } $key = $this->getNodeText(); $this->nextOfType(XMLReader::ELEMENT); $array[$key] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); }
		while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "dict")); return $array;
		}
	private function parse_array() {
		$array = array(); $this->nextOfType(XMLReader::ELEMENT);
		do { $array[] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); }
		while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "array")); return $array;
		}
	private function getNodeText() { $string = $this->readString(); $this->nextOfType(XMLReader::END_ELEMENT); return $string; }
	private function nextOfType() { $types = func_get_args(); $this->read(); while($this->nodeType && !(in_array($this->nodeType, $types))) { $this->read(); } }
	private function isNodeOfTypeName($type, $name) { return $this->nodeType === $type && $this->name === $name; }
	}

function parseMediaInfo ($xml) {
	$xml = simplexml_load_string($xml);
    $data = array();
    $data['version'] = (string) $xml['version'];
    foreach ($xml->File->track as $track) {
		$trackType = strtolower($track['type']);
		$trackId = isset($track['streamid']) ? $track['streamid'] : 1;
		$trackId = (string)$trackId;
		$trackData = [];
		foreach ($track as $rawKey => $rawVal) {
			$key = strtolower($rawKey);
			$val = (string)$rawVal;
			if ($key == 'stream_identifier') { continue; }
			if (!array_key_exists($key, $trackData)) {
				$trackData[$key] = array($val);
				} elseif (!in_array($val, $trackData[$key])) {
				$trackData[$key][] = $val;
				}
            }
		if ($trackType == 'general') {
			$data['file']['general'] = $trackData;
			} else {
			$data['file'][$trackType][$trackId] = $trackData;
			}
		}
	return $data;
	}

function bashcolor($str,$fgcolor="white",$bgcolor=null) {
	static $fgcolors = array('black' => '0;30', 'dark gray' => '1;30', 'blue' => '0;34', 'light blue' => '1;34', 'green' => '0;32', 'light green' => '1;32', 'cyan' => '0;36', 'light cyan' => '1;36', 'red' => '0;31', 'light red' => '1;31', 'purple' => '0;35', 'light purple' => '1;35', 'brown' => '0;33', 'yellow' => '1;33', 'light gray' => '0;37', 'white' => '1;37', 'underline' => '4');
	static $bgcolors = array('black' => '40', 'red' => '41', 'green' => '42', 'yellow' => '43', 'blue' => '44', 'magenta' => '45', 'cyan' => '46', 'light gray' => '47');
	$out="";
	if (!isset($fgcolors[$fgcolor])) { $fgcolor='white'; }
	if (!isset($bgcolors[$bgcolor])) { $bgcolor=null; }
	if ($fgcolor) { $out .= "\033[{$fgcolors[$fgcolor]}m"; }
	if ($bgcolor) { $out .= "\033[{$bgcolors[$bgcolor]}m"; }
	$out .= $str."\033[0m";
	return $out;
	}

// Disk info
//////////////////////////////////////////

$host = gethostname();
$disks = shell_exec("diskutil list 2>&1");

if (substr($zpath, 0, 9) != "/Volumes/") {
	$zbase = "/";
	} else {
	$zparts = explode("/", $zpath);
	$zbase = "/Volumes/".$zparts[2];
	}

$diskutil = shell_exec("diskutil info ".$zbase." 2>&1");
$getstats = array(	"Volume Name", 
					"Protocol", 
					"Volume UUID", 
					"Device Location", 
					"Volume Total Space",
					"Volume Available Space",
					"Level Type"
					);
foreach ($getstats as $stat) {
	preg_match("/(".$stat.":)(.*)(\n)/",$diskutil,$matches);
	if (isset($matches[2])) {
		if (substr($stat, -5, 5) == "Space") {
			$pieces = explode(" ", trim($matches[2]));
			$summary = $pieces[0]." ".$pieces[1];
			$stats[$stat] = $summary;
			} else {
			$stats[$stat] = trim($matches[2]);
			}
		}
	}

$dstats = serialize($stats);
if ($zpath == "/") {
	$type = "Startup disk";
	} elseif (strtolower($zpath) == strtolower("/Volumes/".$stats["Volume Name"])) {
	if ($stats["Protocol"] == "Disk Image") {
		$type = "Disk image";
		} else {
		$type = "External disk";
		}
	} else {
	$type = "Folder";
	}

$profile = shell_exec("system_profiler SPHardwareDataType SPStorageDataType SPThunderboltDataType SPUSBDataType 2>&1");
$qlmanage = shell_exec("qlmanage -m 2>&1");
$sysvers = shell_exec("sw_vers 2>&1");

// Database
//////////////////////////////////////////

$stamp = date("Y-m-d_H-i-s", time());

$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite");
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

/*
$dbo->query("PRAGMA page_size = 4096");
$dbo->query("PRAGMA cache_size = 10000");
$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
$dbo->query("PRAGMA synchronous = NORMAL");
$dbo->query("PRAGMA journal_mode = WAL");
*/

$dbo->exec("CREATE TABLE _walkwalk (
				version TEXT, 
				opts TEXT, 
				host TEXT, 
				uid INTEGER, 
				zpath TEXT, 
				bpath TEXT, 
				type TEXT, 
				passed_file INTEGER, 
				passed_dir INTEGER, 
				passed_link INTEGER,
				passed_total INTEGER,  
				nodescended INTEGER, 
				ignored INTEGER, 
				dupes INTEGER, 
				stats TEXT, 
				qlmanage TEXT, 
				sysvers TEXT, 
				diskutil TEXT, 
				disks TEXT, 
				profile TEXT, 
				status TEXT
				)");

$dbo->exec("CREATE TABLE family (
				pid TEXT, 
				fid TEXT, 
				children TEXT
				)");
				
$dbo->exec("CREATE TABLE files (
				pid TEXT, 
				fid TEXT, 
				Pathname TEXT, 
				Path TEXT, 
				Filename TEXT, 
				Extension TEXT, 
				Type TEXT, 
				items INTEGER, 
				newest INTEGER, 
				stat TEXT, 
				LinkTarget TEXT, 
				RealPath TEXT, 
				Inode INTEGER, 
				Size INTEGER, 
				Perms INTEGER, 
				Owner TEXT, 
				ATime INTEGER, 
				MTime INTEGER, 
				CTime INTEGER, 
				gfi_type TEXT, 
				gfi_attr TEXT, 
				gfi_created TEXT, 
				hash TEXT, 
				tinfo TEXT, 
				hasmeta INTEGER, 
				DateAdded TEXT, 
				ContentType TEXT, 
				Creator TEXT, 
				Kind TEXT, 
				UserTags TEXT, 
				FSInvisible INTEGER, 
				PixelWidth INTEGER, 
				PixelHeight INTEGER, 
				spotlight TEXT,
				duration TEXT, 
				mediainfo TEXT, 
				exiftool TEXT
				)");

$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :profile, :status)");
$stmt->BindValue(":version",$version);
$stmt->BindValue(":opts",serialize(getWoptString()));
$stmt->BindValue(":host",$host);
$stmt->BindValue(":uid",posix_getuid());
$stmt->BindValue(":zpath",$zpath);
$stmt->BindValue(":bpath",$bpath);
$stmt->BindValue(":type",$type);
$stmt->BindValue(":stats",$dstats);
$stmt->BindValue(":qlmanage",$qlmanage);
$stmt->BindValue(":sysvers",$sysvers);
$stmt->BindValue(":diskutil",$diskutil);
$stmt->BindValue(":disks",$disks);
$stmt->BindValue(":profile",$profile);
$stmt->BindValue(":status","aborted");
$stmt->execute();

// Iterator
//////////////////////////////////////////

$passed_file = $passed_dir = $passed_link = $nodescended = $ignored = 0;
$files = new RecursiveIteratorIterator(
	new RecursiveCallbackFilterIterator(
		new RecursiveDirectoryIterator(
			// start in parent dir to include self
      		dirname($zpath),
			RecursiveDirectoryIterator::SKIP_DOTS
			),
		function ($current, $key, $iterator) use ($wopt_ignore, $wopt_nodescend) {
			global $nodescended, $ignored, $passed_file, $passed_dir, $passed_link, $zpath;
			$clean = true;
			// ensure we don't traverse zpath siblings
			if ($zpath != "/" && (substr($current->getRealpath(), 0, strlen($zpath)+1) != $zpath."/") && ($current->getRealpath() != $zpath)) {
				$clean = false;
				}
			// filenames to ignore
			if (is_array($wopt_ignore)) {
				foreach ($wopt_ignore as $wildcard) {
					if (fnmatch($wildcard, $current->getFilename())) {
						$clean = false;
						$ignored++;
						//echo "\nSkipping: ".$current->getRealpath()."\n\n";
						}
					}
				}
			// directories to ignore
			if (is_array($wopt_nodescend)) {
				foreach ($wopt_nodescend as $wildcard) {
					if (fnmatch($wildcard, $current->getPath())) {
						$clean = false;
						$nodescended++;
						//echo "\nNodescending: ".$current->getRealpath()."\n\n";
						}
					}
				}
			if ($clean) {
				if ($current->getType() == "file") {
					$passed_file++;
					} elseif ($current->getType() == "dir") {
					$passed_dir++;
					} elseif ($current->getType() == "link") {
					$passed_link++;
					}
				}
			return $clean;
			}
		),
	RecursiveIteratorIterator::SELF_FIRST,
	RecursiveIteratorIterator::CATCH_GET_CHILD
	);

// Banner
//////////////////////////////////////////

echo "Yuba ".$version."\n";
echo "-----------------------------------------------\n";
$banner = $zpath." -> ".$bpath;
echo $banner."\n";
echo str_repeat("-", strlen($banner))."\n";

// Permissions
//////////////////////////////////////////

/*
if (posix_getuid()) {
	
	echo bashcolor("You are not root. Checking file readability: ", "red");
	
	echo "\n";
	
	$oops = 0;
	foreach ($files as $splFileInfo) {
		$path = $splFileInfo->getRealPath();
		if (!is_readable($path)) {
			$oops = 1;
			echo "x";
			} else {
			echo ".";
			}
		}
		
	echo "\n\n";
		
	if ($oops) {
		echo "Some files could not be read. Continue? (Y/n)";
		$line = trim(fgets(fopen("php://stdin","r")));
		$line = $line ?: "y";
		if($line != "y"){
			echo "Exiting!\n"; die;
			}
		}
	
	} else {
	
	echo bashcolor("Running as root. Some QuickLook plugins may not be available.", "red");
	echo "\n\n";
	
	}
*/

$fixatimes = 0;
if ($wopt_paranoid) {

	echo bashcolor("\nFilesystem is writable. You can choose:\n(c) Preserve ctimes (default)\n(a) Preserve atimes\n", "purple");
	$line = trim(fgets(fopen("php://stdin","r"))) ?: "c";
	if ($line == "a") { $fixatimes = 1; }
	
	}

// Prescan
//////////////////////////////////////////

$i = 0;
$family = array();
$fids = array();
foreach ($files as $splFileInfo) {

	$pathname = $splFileInfo->getPathname();
	$path = $splFileInfo->getPath();
	
	$key = md5($pathname);
	$pkey = md5($path);

	if (array_key_exists($key, $family)) {
		echo "Duplicate key on ".$pathname."\n"; die;
		}
	
	$family[$key] = array();
	
	// Path-agnostic Unique File ID (to prevent redundant hashes and thumbs)

	if ($splFileInfo->getType() != "dir" && $splFileInfo->getType() != "link") {

		$fid = md5($splFileInfo->getSize().$splFileInfo->getMtime().$splFileInfo->getBasename());
		$dx[$fid][] = $pathname;
		$fx[] = array($fid, $pathname);
		$family[$key]['fid'] = $fid;
		
		}

	// Parents
	
	//$family[$key]['parents'] = getParents($zpath, $pathname);
	
	// Children
	
	if (strlen($path) > strlen($zpath)-1) { // prevent zpath root orphan
		$family[$pkey]['children'][] = $key;
		}
	
	if ($i % 5000 == 0) {
		echo "\r\033[K\rPrescan: ".$pathname;
		}
	
	$i++;
	
	}

echo "\r\033[K\rPrescan: done\n";

// Debug record of duplicate FIDs

$dupes = array_filter($dx, function($a) { return count($a) > 2; });

ob_start();
var_dump($dupes);
$dxo = ob_get_clean();

if (strlen($dxo)) {
	file_put_contents($bpath."/".$stamp."_dupes.txt",$dxo);
	$dupecount = count($dupes,COUNT_RECURSIVE) - count($dupes);
	echo "\n".bashcolor(floor(($dupecount/$i)*100)." percent of files look like duplicates","green")."\n\n";
	}

// Write family to DB

$message = "Writing family to DB: ";
$message .= $passed_file." files, ";
$message .= $passed_dir." dirs, ";
$message .= $nodescended." bundles, ";
$message .= $passed_link." links, ";
$message .= $ignored." ignored, ";
$message .= ($dupecount ? $dupecount : 0)." dupes";

echo ProgressBar::start($i,$message);

foreach ($family as $key => $item) {

	echo ProgressBar::next();
	$stmt = $dbo->prepare("INSERT INTO family VALUES (:pid, :fid, :children)");
	$stmt->BindValue(":pid",$key);
	if (@$item['fid']) {
		$stmt->BindValue(":fid",$item['fid']);
		}
	if (@$item['children'] && is_array(@$item['children'])) {
		$stmt->BindValue(":children",serialize($item['children']));
		}
	$stmt->execute();

	}

echo ProgressBar::finish();

unset($dx, $dxo, $dupes);

// stats
$stmt = "UPDATE _walkwalk SET ";
$stmt .= "passed_file=".$passed_file.", ";
$stmt .= "passed_dir=".$passed_dir.", ";
$stmt .= "passed_link=".$passed_link.", ";
$stmt .= "passed_total=".$i.", ";
$stmt .= "nodescended=".$nodescended.", ";
$stmt .= "ignored=".$ignored.", ";
$stmt .= "dupes=".($dupecount ? $dupecount : 0).", ";
$stmt .= "status='completed_in_".$seconds."'";
$dbo->exec($stmt);

// Thumbnails
//////////////////////////////////////////

if ($wopt_thumbs) {

	$message = "Generating thumbnails...";
	echo ProgressBar::start(count($fx),$message);

	$tempdir = "/tmp/".$blabel."_".$stamp;
	if (!is_dir($tempdir)) { mkdir($tempdir); }
	
	foreach ($fx as $array) {
		
		$fid = $array[0];
		$pathname = $array[1];
		$ext = pathinfo($pathname,PATHINFO_EXTENSION);
		$tpath = $bpath."/thumbs/".substr($fid, 0, 2);
		$tfile = $tpath."/".$fid.".jpg";
		
		// HACK for ql-thumbnail bug
		$t_skip = array("emlx");
		if (count($t_skip) && in_array($ext, $t_skip)) {
			echo ProgressBar::next(1, "Skipping ".shortlabel(basename($pathname),$max_label));
			continue;
			}
		
		// check to see if a thumb already exists
		if (file_exists($tfile)) {
			echo ProgressBar::next(1, "Thumb found for ".shortlabel(basename($pathname),$max_label));
			continue;
			} else {
			echo ProgressBar::next(1, "Generating thumb for ".shortlabel(basename($pathname),$max_label));
			}
		
		$shellpath = escapeshellarg($pathname);
		$tempfile = $tempdir."/".$fid.".jpg";
			
		// first try to make a thumb with external tools
		$cmd = null;
		if (in_array($ext, $t_files['vips'])) {
			$cmd = $bin_tv." ".$shellpath." -o ".$tempfile."[Q=85,optimize_coding] --size=".$wopt_thumb_size;
			} elseif (in_array($ext, $t_files['ffmpeg'])) {
			$cmd = $bin_tf." -i ".$shellpath." -o ".$tempfile." -s ".$wopt_thumb_size." -c jpg -q 9";
			}
			
		if ($cmd) { shell_exec($cmd." 2>&1"); }
		
		// if those tools failed, try quicklook
		if (!@filesize($tempfile)) {
			$cmd = $bin_tq." ".$shellpath." ".$tempfile." public.jpeg-2000 ".$wopt_thumb_size." ".$wopt_thumb_size;
			shell_exec($cmd." 2>&1");
			}
		
		// success, move thumb into the bundle
		if (file_exists($tempfile) && @filesize($tempfile)) {
			if (!is_dir($tpath)) { mkdir($tpath); }
			rename($tempfile,$tfile);
			}
	
		}
	
	echo ProgressBar::finish();

	}

// Hashes
//////////////////////////////////////////

if ($wopt_hash) {

	$dbh = new PDO("sqlite:".$bpath."/md5.sqlite");
	$dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
	$dbh->exec("CREATE TABLE IF NOT EXISTS hash (fid TEXT, hash TEXT)");
	
	if ($wopt_hash_limit) {
		$message = "Generating hashes for files under ".$wopt_hash_limit."GB";
		} else {
		$message = "Generating hashes for all files";
		}
	
	echo ProgressBar::start(count($fx),$message);

	foreach ($fx as $array) {
		$fid = $array[0];
		$pathname = $array[1];
		$size = filesize($pathname);
		$limit = $wopt_hash_limit*1000000000;
		$check = $dbh->query("SELECT EXISTS(SELECT 1 FROM hash WHERE fid='".$fid."')")->fetch()[0];
		if ($check) {
			echo ProgressBar::next(1, "Hash already exists: ".shortlabel($pathname,$max_label));
			} elseif ($wopt_hash_limit && $size > $limit) {
			echo ProgressBar::next(1, "Too big to hash: ".shortlabel($pathname,$max_label)." (".human_filesize($size).")");
			} else {
			echo ProgressBar::next(1);
			$stmt = $dbh->prepare("INSERT INTO hash VALUES (:fid, :hash)");
			$stmt->BindValue(":fid",$fid);
			$stmt->BindValue(":hash",md5_file($pathname));
			$stmt->execute();
			}
		}

	echo ProgressBar::finish();
	
	}

// Files
//////////////////////////////////////////



// Cleanup
//////////////////////////////////////////

echo "\n";

$seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]);
$dbo->exec("UPDATE _walkwalk SET status='completed_in_".$seconds."'");
echo "Finished in ".$seconds." seconds\n\n";

unset($dbo, $dbh, $files, $family, $fx);

?>