diff --git a/appinfo/info.xml b/appinfo/info.xml index 8f539794..085e414c 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -16,7 +16,7 @@ - **🚀 Build your own thing:** FaceRecognition app is just a basic building block. Through FaceRecognition API, you can build your advanced scenarios - automatically add tags to images, connect contacts and persons, share images from specific person… We want to hear your ideas! ]]> - 0.5.15 + 0.5.16 agpl Matias De lellis Branko Kokanovic diff --git a/appinfo/routes.php b/appinfo/routes.php index 9c02d49b..cd3efd3c 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -37,12 +37,30 @@ 'url' => '/file', 'verb' => 'GET' ], + // Get proposed relations on an person. + [ + 'name' => 'relation#findByPerson', + 'url' => '/relation/{personId}', + 'verb' => 'GET' + ], + // Change an relation of an person. + [ + 'name' => 'relation#updateByPersons', + 'url' => '/relation/{personId}', + 'verb' => 'PUT' + ], // Get folder preferences [ 'name' => 'file#getFolderOptions', 'url' => '/folder', 'verb' => 'GET' ], + // Change relations in batch + [ + 'name' => 'relation#updateByPersonsBatch', + 'url' => '/relations', + 'verb' => 'PUT' + ], // Set folder preferences [ 'name' => 'file#setFolderOptions', diff --git a/css/facerecognition.css b/css/facerecognition.css index cb5ecddb..b5550dc8 100644 --- a/css/facerecognition.css +++ b/css/facerecognition.css @@ -39,7 +39,7 @@ * Rename dialog */ -#fr-dialog-content-input { +#fr-rename-dialog-content-input { width: 80%; margin: 6px; } @@ -47,10 +47,16 @@ .face-preview-dialog { background-color: rgba(210, 210, 210, .75); border-radius: 25px; + margin: 2px; height: 50px; width: 50px; } + +.oc-dialog-buttonrow, .threebuttons { + justify-content: space-between; +} + /* * Admin page */ diff --git a/css/files-tabview.css b/css/files-tabview.css index 3d09bcf6..c85c5b8a 100644 --- a/css/files-tabview.css +++ b/css/files-tabview.css @@ -29,7 +29,7 @@ /* * Rename dialog */ -#fr-dialog-content-input { +#fr-rename-dialog-content-input { width: 80%; margin: 6px; } diff --git a/js/admin.js b/js/admin.js index ff0f7912..fd007f9b 100644 --- a/js/admin.js +++ b/js/admin.js @@ -106,7 +106,6 @@ $(document).ready(function() { }); }); - /* * Sensitivity */ @@ -161,6 +160,60 @@ $(document).ready(function() { }); }); + /* + * Deviation + */ + function getDeviation() { + $.ajax({ + type: 'GET', + url: OC.generateUrl('apps/facerecognition/getappvalue'), + data: { + 'type': 'deviation', + }, + success: function (data) { + if (data.status === state.OK) { + var deviation = parseFloat(data.value); + $('#deviation-range').val(deviation); + $('#deviation-value').html(deviation); + } + } + }); + } + + $('#deviation-range').on('input', function() { + $('#deviation-value').html(this.value); + $('#restore-deviation').show(); + $('#save-deviation').show(); + }); + + $('#restore-deviation').on('click', function(event) { + event.preventDefault(); + getDeviation(); + + $('#restore-deviation').hide(); + $('#save-deviation').hide(); + }); + + $('#save-deviation').on('click', function(event) { + event.preventDefault(); + var deviation = $('#deviation-range').val().toString(); + $.ajax({ + type: 'POST', + url: OC.generateUrl('apps/facerecognition/setappvalue'), + data: { + 'type': 'deviation', + 'value': deviation + }, + success: function (data) { + if (data.status === state.SUCCESS) { + OC.Notification.showTemporary(t('facerecognition', 'The changes were saved.')); + $('#restore-deviation').hide(); + $('#save-deviation').hide(); + } + } + }); + }); + /* * Confidence */ @@ -263,6 +316,7 @@ $(document).ready(function() { */ getImageArea(); getSensitivity(); + getDeviation(); getMinConfidence(); getNotGrouped(); diff --git a/js/fr-dialogs.js b/js/fr-dialogs.js index d40196b7..bd1ed5c6 100644 --- a/js/fr-dialogs.js +++ b/js/fr-dialogs.js @@ -26,7 +26,7 @@ const FrDialogs = { rename: function (name, thumbUrl, callback) { return $.when(this._getMessageTemplate()).then(function ($tmpl) { - var dialogName = 'fr-dialog-content'; + var dialogName = 'fr-rename-dialog-content'; var dialogId = '#' + dialogName; var $dlg = $tmpl.octemplate({ dialog_name: dialogName, @@ -87,6 +87,73 @@ const FrDialogs = { input.select(); }); }, + suggestPersonName: function (name, faces, callback) { + return $.when(this._getMessageTemplate()).then(function ($tmpl) { + var dialogName = 'fr-suggest-dialog-content'; + var dialogId = '#' + dialogName; + var $dlg = $tmpl.octemplate({ + dialog_name: dialogName, + title: t('facerecognition', 'Suggestions'), + message: t('facerecognition', 'Is it {personName}? Or a different person?', {personName: name}), + type: 'none' + }); + + var div = $('
').attr('style', 'display:flex; align-items: center'); + for (var face of faces) { + var thumb = $(''); + div.append(thumb); + } + $dlg.append(div); + + $('body').append($dlg); + + // wrap callback in _.once(): + // only call callback once and not twice (button handler and close + // event) but call it for the close event, if ESC or the x is hit + if (callback !== undefined) { + callback = _.once(callback); + } + + var buttonlist = [{ + text: t('facerecognition', 'No'), + click: function () { + if (callback !== undefined) { + $(dialogId).ocdialog('close'); + } + callback(true, Relation.REJECTED); + }, + }, { + text: t('facerecognition', 'I am not sure'), + click: function () { + if (callback !== undefined) { + $(dialogId).ocdialog('close'); + } + callback(true, Relation.PROPOSED); + } + }, { + text: t('facerecognition', 'Yes'), + click: function () { + if (callback !== undefined) { + $(dialogId).ocdialog('close'); + } + callback(true, Relation.ACCEPTED); + }, + defaultButton: true + }]; + + $(dialogId).ocdialog({ + closeOnEscape: true, + modal: true, + buttons: buttonlist, + close: function () { + // callback is already fired if Yes/No is clicked directly + if (callback !== undefined) { + callback(false, Relation.PROPOSED); + } + } + }); + }); + }, _getMessageTemplate: function () { var defer = $.Deferred(); if (!this.$messageTemplate) { diff --git a/js/personal.js b/js/personal.js index 971756a4..e6fb45d5 100644 --- a/js/personal.js +++ b/js/personal.js @@ -30,6 +30,7 @@ Persons.prototype = { self._enabled = response.enabled; self._clusters = response.clusters; self._loaded = true; + console.debug('The user has %i clusters',self._clusters.length); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -43,6 +44,7 @@ Persons.prototype = { var self = this; $.get(this._baseUrl+'/cluster/'+id).done(function (cluster) { self._cluster = cluster; + console.debug('Cluster id %i has %i faces', id, cluster.faces.length); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -54,6 +56,11 @@ Persons.prototype = { var self = this; $.get(this._baseUrl+'/person/'+personName).done(function (clusters) { self._clustersByName = clusters.clusters; + var total = 0; + self._clustersByName.forEach(function (cluster) { + total += cluster.faces.length; + }); + console.debug('There are %i clusters called %s with a total of %i faces', self._clustersByName.length, personName, total); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -99,7 +106,7 @@ Persons.prototype = { getAll: function () { return this._clusters; }, - renameCluster: function (clusterId, personName) { + updateCluster: function (clusterId, personName) { var self = this; var deferred = $.Deferred(); var opt = { name: personName }; @@ -108,24 +115,28 @@ Persons.prototype = { contentType: 'application/json', data: JSON.stringify(opt) }).done(function (data) { - self._clusters.forEach(function (cluster) { - if (cluster.id === clusterId) { - cluster.name = personName; - } - }); + self.renameCluster(clusterId, personName); deferred.resolve(); }).fail(function () { deferred.reject(); }); return deferred.promise(); + }, + renameCluster: function (clusterId, personName) { + this._clusters.forEach(function (cluster) { + if (cluster.id === clusterId) { + cluster.name = personName; + } + }); } }; /* * View. */ -var View = function (persons) { +var View = function (persons, similar) { this._persons = persons; + this._similar = similar; }; View.prototype = { @@ -156,6 +167,78 @@ View.prototype = { } }); }, + renamePerson: function (personId, personName, faceUrl) { + var self = this; + FrDialogs.rename( + personName, + faceUrl, + function(result, name) { + if (result === true && name) { + self._persons.updateCluster(personId, name).done(function() { + self._persons.unsetActive(); + self.renderContent(); + self._similar.findProposal(personId, name).done(function() { + if (self._similar.isEnabled()) { + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), name); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + } + } + }); + }).fail(function () { + OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); + }); + } + } + ); + }, + suggestPerson: function (proposal, personName) { + var self = this; + FrDialogs.suggestPersonName( + personName, + this._persons.getById(proposal.id).faces, + function(valid, state) { + if (valid === true) { + // It is valid must be update the proposals. + self._similar.answerProposal(proposal, state); + if (state === Relation.ACCEPTED) { + // Look for new suggestions based on accepted proposal + self._similar.findProposal(proposal.id, personName).done(function() { + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions. Applying your suggestions.')); + self._similar.applyProposals().done(function() { + self._similar.getAcceptedProposal().forEach(function (accepted) { + self._persons.renameCluster(accepted.id, personName); + }); + self._persons.unsetActive(); + self.renderContent(); + }); + } + }); + } else { + // Suggest cached proposals + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions. Applying your suggestions.')); + self._similar.applyProposals().done(function() { + self._similar.getAcceptedProposal().forEach(function (accepted) { + self._persons.renameCluster(accepted.id, personName); + }); + self._persons.unsetActive(); + self.renderContent(); + }); + } + } + } else { + OC.Notification.showTemporary(t('facerecognition', 'Canceled')); + } + } + ); + }, renderContent: function () { this._persons.sortBySize(); var context = { @@ -224,20 +307,7 @@ View.prototype = { $('#facerecognition .icon-rename').click(function () { var id = $(this).parent().data('id'); var person = self._persons.getById(id); - FrDialogs.rename( - person.name, - person.faces[0]['thumb-url'], - function(result, value) { - if (result === true && value) { - self._persons.renameCluster (id, value).done(function () { - self._persons.unsetActive(); - self.renderContent(); - }).fail(function () { - OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); - }); - } - } - ); + self.renamePerson(id, person.name, person.faces[0]['thumb-url']); }); $('#facerecognition #show-more-clusters').click(function () { @@ -260,8 +330,9 @@ View.prototype = { * Main app. */ var persons = new Persons(OC.generateUrl('/apps/facerecognition')); +var similar = new Similar(OC.generateUrl('/apps/facerecognition')); -var view = new View(persons); +var view = new View(persons, similar); view.renderContent(); diff --git a/js/similar.js b/js/similar.js new file mode 100644 index 00000000..ab48911e --- /dev/null +++ b/js/similar.js @@ -0,0 +1,147 @@ +'use strict'; + +const Relation = { + PROPOSED: 0, + ACCEPTED: 1, + REJECTED: 2 +} + +var Similar = function (baseUrl) { + this._baseUrl = baseUrl; + + this._enabled = false; + this._similarProposed = []; + this._similarAccepted = []; + this._similarRejected = []; + this._similarIgnored = []; + this._similarApplied = []; + this._similarName = undefined; +}; + +Similar.prototype = { + isEnabled: function () { + return this._enabled; + }, + findProposal: function (clusterId, clusterName) { + if (this._similarName !== clusterName) { + this.resetProposals(clusterId, clusterName); + } + var self = this; + var deferred = $.Deferred(); + $.get(this._baseUrl+'/relation/'+clusterId).done(function (response) { + self._enabled = response.enabled; + if (!self._enabled) { + self.resetSuggestions(clusterId, clusterName); + } else { + self.concatNewProposals(response.proposed); + self._similarName = clusterName; + } + deferred.resolve(); + }).fail(function () { + deferred.reject(); + }); + return deferred.promise(); + }, + hasProposal: function () { + return (this._similarProposed.length > 0); + }, + getProposal: function () { + return this._similarProposed.shift(); + }, + getAcceptedProposal: function () { + return this._similarAccepted; + }, + answerProposal: function (proposal, state) { + var self = this; + // First of all, remove from queue all relations with that ID + var newProposed = []; + self._similarProposed.forEach(function (oldProposal) { + if (proposal.id !== oldProposal.id) { + newProposed.push(oldProposal); + } else { + if (state === Relation.ACCEPTED) { + oldProposal.state = Relation.ACCEPTED; + self._similarAccepted.push(oldProposal); + self._similarApplied.push(oldProposal); + } else if (state === Relation.REJECTED) { + oldProposal.state = Relation.REJECTED; + self._similarRejected.push(oldProposal); + self._similarApplied.push(oldProposal); + } else { + oldProposal.state = Relation.PROPOSED; + self._similarIgnored.push(oldProposal); + } + } + }); + self._similarProposed = newProposed; + + // Add the old proposal to its actual state. + proposal.state = state; + if (state === Relation.ACCEPTED) { + this._similarAccepted.push(proposal); + this._similarApplied.push(proposal); + } else if (state === Relation.REJECTED) { + this._similarRejected.push(proposal); + this._similarApplied.push(proposal); + } else { + this._similarIgnored.push(proposal); + } + }, + applyProposals: function () { + var self = this; + var deferred = $.Deferred(); + var data = { + personsRelations: self._similarApplied, + personName: self._similarName + }; + $.ajax({ + url: this._baseUrl + '/relations', + method: 'PUT', + contentType: 'application/json', + data: JSON.stringify(data) + }).done(function (data) { + deferred.resolve(); + }).fail(function () { + deferred.reject(); + }); + return deferred.promise(); + }, + concatNewProposals: function (proposals) { + var self = this; + proposals.forEach(function (proposed) { + // An person ca be propoced several times, since they are related to direfents persons. + if (self._similarAccepted.find(function (oldProposed) { return proposed.id === oldProposed.id;}) !== undefined) { + proposed.state = Relation.ACCEPTED; + self._similarAccepted.push(proposed); + self._similarApplied.push(proposed); + } else if (self._similarRejected.find(function (oldProposed) { return proposed.id === oldProposed.id;}) !== undefined) { + proposed.state = Relation.REJECTED; + self._similarRejected.push(proposed); + self._similarApplied.push(proposed); + } else if (self._similarIgnored.find(function (oldProposed) { return proposed.id === oldProposed.id;}) !== undefined) { + proposed.state = Relation.REJECTED; + self._similarIgnored.push(proposed); + } else { + proposed.state = Relation.PROPOSED; + self._similarProposed.push(proposed); + } + }); + }, + resetProposals: function (clusterId, clusterName) { + this._similarAccepted = []; + this._similarProposed = []; + this._similarRejected = []; + this._similarIgnored = []; + this._similarApplied = []; + this._similarName = undefined; + + // Add a fake proposal to self-accept when referring to the initial person + var fakeProposal = { + origId: clusterId, + id: clusterId, + name: clusterName, + state: Relation.ACCEPTED + }; + this._similarAccepted.push(fakeProposal); + } +}; \ No newline at end of file diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index 9f12faac..c190f27e 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -32,6 +32,9 @@ use OCA\FaceRecognition\Db\ImageMapper; use OCA\FaceRecognition\Db\PersonMapper; +use OCA\FaceRecognition\Db\Relation; +use OCA\FaceRecognition\Db\RelationMapper; + use OCA\FaceRecognition\Helper\Euclidean; use OCA\FaceRecognition\Service\SettingsService; @@ -48,6 +51,9 @@ class CreateClustersTask extends FaceRecognitionBackgroundTask { /** @var FaceMapper Face mapper*/ private $faceMapper; + /** @var RelationMapper Relation mapper*/ + private $relationMapper; + /** @var SettingsService Settings service*/ private $settingsService; @@ -60,6 +66,7 @@ class CreateClustersTask extends FaceRecognitionBackgroundTask { public function __construct(PersonMapper $personMapper, ImageMapper $imageMapper, FaceMapper $faceMapper, + RelationMapper $relationMapper, SettingsService $settingsService) { parent::__construct(); @@ -67,6 +74,7 @@ public function __construct(PersonMapper $personMapper, $this->personMapper = $personMapper; $this->imageMapper = $imageMapper; $this->faceMapper = $faceMapper; + $this->relationMapper = $relationMapper; $this->settingsService = $settingsService; } @@ -188,12 +196,14 @@ private function createClusterIfNeeded(string $userId) { $faces = $this->faceMapper->getFaces($userId, $modelId); $this->logInfo(count($faces) . ' faces found for clustering'); + $relations = $this->relationMapper->findByUserAsMatrix($userId, $modelId); + // Cluster is associative array where key is person ID. // Value is array of face IDs. For old clusters, person IDs are some existing person IDs, // and for new clusters is whatever chinese whispers decides to identify them. // $currentClusters = $this->getCurrentClusters($faces); - $newClusters = $this->getNewClusters($faces); + $newClusters = $this->getNewClusters($faces, $relations); $this->logInfo(count($newClusters) . ' persons found after clustering'); // New merge @@ -207,8 +217,11 @@ private function createClusterIfNeeded(string $userId) { $this->logInfo('Deleted ' . $orphansDeleted . ' persons without faces'); } - // Prevents not create/recreate the clusters unnecessarily. + // Fill relation table with new clusters. + $relations = $this->fillFaceRelationsFromPersons($userId, $modelId); + $this->logInfo($relations . ' relations added as suggestions'); + // Prevents not create/recreate the clusters unnecessarily. $this->settingsService->setNeedRecreateClusters(false, $userId); $this->settingsService->setForceCreateClusters(false, $userId); } @@ -226,9 +239,8 @@ private function getCurrentClusters(array $faces): array { return $chineseClusters; } - private function getNewClusters(array $faces): array { + private function getNewClusters(array $faces, array $relations): array { // Create edges for chinese whispers - $euclidean = new Euclidean(); $sensitivity = $this->settingsService->getSensitivity(); $min_confidence = $this->settingsService->getMinimumConfidence(); $edges = array(); @@ -242,14 +254,26 @@ private function getNewClusters(array $faces): array { } for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { $face2 = $faces[$j]; + if ($this->relationMapper->existsOnMatrix($face1->id, $face2->id, $relations)) { + $state = $this->relationMapper->getStateOnMatrix($face1->id, $face2->id, $relations); + if ($state === Relation::ACCEPTED) { + $edges[] = array($i, $j); + continue; + } else if ($state === Relation::REJECTED) { + continue; + } + } + if ($face2->confidence < $min_confidence) { + continue; + } $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); - if ($distance < $sensitivity) { $edges[] = array($i, $j); } } } } else { + $euclidean = new Euclidean(); for ($i = 0, $face_count1 = count($faces); $i < $face_count1; $i++) { $face1 = $faces[$i]; if ($face1->confidence < $min_confidence) { @@ -258,6 +282,9 @@ private function getNewClusters(array $faces): array { } for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { $face2 = $faces[$j]; + if ($face2->confidence < $min_confidence) { + continue; + } // todo: can't this distance be a method in $face1->distance($face2)? $distance = $euclidean->distance($face1->descriptor, $face2->descriptor); @@ -343,4 +370,50 @@ public function mergeClusters(array $oldCluster, array $newCluster): array { } return $result; } + + private function fillFaceRelationsFromPersons(string $userId, int $modelId): int { + $deviation = $this->settingsService->getDeviation(); + if (!version_compare(phpversion('pdlib'), '1.0.2', '>=') || ($deviation === 0.0)) + return 0; + + $sensitivity = $this->settingsService->getSensitivity(); + $sensitivity += $deviation; + + $min_confidence = $this->settingsService->getMinimumConfidence(); + + // Get the representative faces of each person + $mainFaces = array(); + $persons = $this->personMapper->findAll($userId, $modelId); + foreach ($persons as $person) { + $mainFaces[] = $this->faceMapper->findRepresentativeFromPerson($userId, $modelId, $person->id, $sensitivity); + } + + // Get similar faces taking into account the deviation + $relations = array(); + $faces_count = count($mainFaces); + for ($i = 0 ; $i < $faces_count; $i++) { + $face1 = $mainFaces[$i]; + if ($face1->confidence < $min_confidence) { + continue; + } + for ($j = $i+1; $j < $faces_count; $j++) { + $face2 = $mainFaces[$j]; + if ($face2->confidence < $min_confidence) { + continue; + } + $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); + if ($distance < $sensitivity) { + $relation = new Relation(); + $relation->setFace1($face1->id); + $relation->setFace2($face2->id); + $relation->setState(Relation::PROPOSED); + $relations[] = $relation; + } + } + } + + // Merge new suggested relations + return $this->relationMapper->merge($userId, $modelId, $relations); + } + } diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index f26664a4..3747e6f8 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -46,7 +46,6 @@ use OCA\FaceRecognition\Service\SettingsService; - class PersonController extends Controller { /** @var IRootFolder */ @@ -222,7 +221,7 @@ public function findByName(string $personName) { * @param string $name */ public function updateName($id, $name) { - $person = $this->personMapper->find ($this->userId, $id); + $person = $this->personMapper->find($this->userId, $id); $person->setName($name); $this->personMapper->update($person); diff --git a/lib/Controller/RelationController.php b/lib/Controller/RelationController.php new file mode 100644 index 00000000..c2e59573 --- /dev/null +++ b/lib/Controller/RelationController.php @@ -0,0 +1,171 @@ + + * + * @author Matias De lellis + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\FaceRecognition\Controller; + +use OCP\IRequest; + +use OCP\AppFramework\Http; +use OCP\AppFramework\Http\DataResponse; +use OCP\AppFramework\Http\JSONResponse; +use OCP\AppFramework\Http\DataDisplayResponse; +use OCP\AppFramework\Controller; + +use OCA\FaceRecognition\Db\Person; +use OCA\FaceRecognition\Db\PersonMapper; + +use OCA\FaceRecognition\Db\Relation; +use OCA\FaceRecognition\Db\RelationMapper; + +use OCA\FaceRecognition\Service\SettingsService; + +class RelationController extends Controller { + + /** @var PersonMapper */ + private $personMapper; + + /** @var RelationMapper */ + private $relationMapper; + + /** @var SettingsService */ + private $settingsService; + + /** @var string */ + private $userId; + + public function __construct($AppName, + IRequest $request, + PersonMapper $personMapper, + RelationMapper $relationMapper, + SettingsService $settingsService, + $UserId) + { + parent::__construct($AppName, $request); + + $this->personMapper = $personMapper; + $this->relationMapper = $relationMapper; + $this->settingsService = $settingsService; + $this->userId = $UserId; + } + + /** + * @NoAdminRequired + * @param int $personId + */ + public function findByPerson(int $personId) { + $deviation = $this->settingsService->getDeviation(); + + $enabled = (version_compare(phpversion('pdlib'), '1.0.2', '>=') && ($deviation > 0.0)); + + $resp = array(); + $resp['enabled'] = $enabled; + $resp['proposed'] = array(); + + if (!$enabled) + return new DataResponse($resp); + + $mainPerson = $this->personMapper->find($this->userId, $personId); + + $proposed = array(); + $relations = $this->relationMapper->findFromPerson($this->userId, $personId, RELATION::PROPOSED); + foreach ($relations as $relation) { + $person1 = $this->personMapper->findFromFace($this->userId, $relation->face1); + if ($person1->getId() !== $personId) { + $proffer = array(); + $proffer['origId'] = $mainPerson->getId(); + $proffer['id'] = $person1->getId(); + $proffer['name'] = $person1->getName(); + $proposed[] = $proffer; + } + $person2 = $this->personMapper->findFromFace($this->userId, $relation->face2); + if ($person2->getId() !== $personId) { + $proffer = array(); + $proffer['origId'] = $mainPerson->getId(); + $proffer['id'] = $person2->getId(); + $proffer['name'] = $person2->getName(); + $proposed[] = $proffer; + } + } + $resp['proposed'] = $proposed; + + return new DataResponse($resp); + } + + /** + * @NoAdminRequired + * @param int $personId + * @param int $toPersonId + * @param int $state + */ + public function updateByPersons(int $personId, int $toPersonId, int $state) { + $relations = $this->relationMapper->findFromPersons($personId, $toPersonId); + + foreach ($relations as $relation) { + $relation->setState($state); + $this->relationMapper->update($relation); + } + + if ($state === RELATION::ACCEPTED) { + $person = $this->personMapper->find($this->userId, $personId); + $name = $person->getName(); + + $toPerson = $this->personMapper->find($this->userId, $toPersonId); + $toPerson->setName($name); + $this->personMapper->update($toPerson); + } + + $relations = $this->relationMapper->findFromPersons($personId, $toPersonId); + return new DataResponse($relations); + } + + /** + * @NoAdminRequired + * @param array $personsRelations + * @param string $personName + */ + public function updateByPersonsBatch(array $personsRelations, string $personName) { + foreach ($personsRelations as $personRelation) { + $origId = $personRelation['origId']; + $id = $personRelation['id']; + $state = $personRelation['state']; + + $faceRelations = $this->relationMapper->findFromPersons($origId, $id); + foreach ($faceRelations as $faceRelation) { + $faceRelation->setState($state); + $this->relationMapper->update($faceRelation); + } + + if ($state === RELATION::ACCEPTED) { + $toPerson = $this->personMapper->find($this->userId, $id); + $toPerson->setName($personName); + $this->personMapper->update($toPerson); + } + } + + $modelId = $this->settingsService->getCurrentFaceModel(); + $persons = $this->personMapper->findByName($this->userId, $modelId, $personName); + + return new DataResponse($persons); + } + +} diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php index 5a635cc7..abd35389 100644 --- a/lib/Controller/SettingsController.php +++ b/lib/Controller/SettingsController.php @@ -188,6 +188,9 @@ public function setAppValue($type, $value) { $this->settingsService->setNeedRecreateClusters(true, $user->getUID()); }); break; + case SettingsService::DEVIATION_KEY: + $this->settingsService->setDeviation($value); + break; case SettingsService::MINIMUM_CONFIDENCE_KEY: $this->settingsService->setMinimumConfidence($value); $this->userManager->callForSeenUsers(function(IUser $user) { @@ -227,6 +230,9 @@ public function getAppValue($type) { case SettingsService::SENSITIVITY_KEY: $value = $this->settingsService->getSensitivity(); break; + case SettingsService::DEVIATION_KEY: + $value = $this->settingsService->getDeviation(); + break; case SettingsService::MINIMUM_CONFIDENCE_KEY: $value = $this->settingsService->getMinimumConfidence(); break; diff --git a/lib/Db/FaceMapper.php b/lib/Db/FaceMapper.php index 36b83c8e..d11d5a47 100644 --- a/lib/Db/FaceMapper.php +++ b/lib/Db/FaceMapper.php @@ -40,7 +40,7 @@ public function find (int $faceId) { $qb = $this->db->getQueryBuilder(); $qb->select('id', 'image', 'person', 'left', 'right', 'top', 'bottom', 'landmarks', 'descriptor', 'confidence') ->from($this->getTableName(), 'f') - ->andWhere($qb->expr()->eq('id', $qb->createNamedParameter($faceId))); + ->where($qb->expr()->eq('id', $qb->createNamedParameter($faceId))); return $this->findEntity($qb); } @@ -103,7 +103,7 @@ public function getOldestCreatedFaceWithoutPerson(string $userId, int $model) { return $face; } - public function getFaces(string $userId, $model): array { + public function getFaces(string $userId, int $modelId): array { $qb = $this->db->getQueryBuilder(); $qb->select('f.id', 'f.person', 'f.confidence', 'f.descriptor') ->from($this->getTableName(), 'f') @@ -111,7 +111,7 @@ public function getFaces(string $userId, $model): array { ->where($qb->expr()->eq('user', $qb->createParameter('user'))) ->andWhere($qb->expr()->eq('model', $qb->createParameter('model'))) ->setParameter('user', $userId) - ->setParameter('model', $model); + ->setParameter('model', $modelId); return $this->findEntities($qb); } @@ -131,6 +131,50 @@ public function findFacesFromPerson(string $userId, int $personId, int $model, $ return $faces; } + public function findRepresentativeFromPerson(string $userId, int $modelId, int $personId, float $sensitivity) { + $qb = $this->db->getQueryBuilder(); + $qb->select('f.id', 'f.confidence', 'f.descriptor') + ->from($this->getTableName(), 'f') + ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) + ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('person', $qb->createNamedParameter($personId))) + ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($modelId))); + $faces = $this->findEntities($qb); + + $face_count = count($faces); + $edgesFacesCount = array(); + for ($i = 0; $i < $face_count; $i++) { + $face1 = $faces[$i]; + for ($j = $i; $j < $face_count; $j++) { + $face2 = $faces[$j]; + $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); + if ($distance < $sensitivity) { + if (isset($edgesFacesCount[$i])) { + $edgesFacesCount[$i]++; + } else { + $edgesFacesCount[$i] = 1; + } + if (isset($edgesFacesCount[$j])) { + $edgesFacesCount[$j]++; + } else { + $edgesFacesCount[$j] = 1; + } + } + } + } + + $bestFaceIndex = -1; + $bestFaceCount = -1; + foreach ($edgesFacesCount as $faceIndex => $faceCount) { + if ($faceCount > $bestFaceCount) { + $bestFaceIndex = $faceIndex; + $bestFaceCount = $faceCount; + } + } + + return $faces[$bestFaceIndex]; + } + public function getPersonOnFile(string $userId, int $personId, int $fileId, int $model): array { $qb = $this->db->getQueryBuilder(); $qb->select('f.id', 'left', 'right', 'top', 'bottom') diff --git a/lib/Db/PersonMapper.php b/lib/Db/PersonMapper.php index 3cfb4ee5..3850bcf6 100644 --- a/lib/Db/PersonMapper.php +++ b/lib/Db/PersonMapper.php @@ -102,6 +102,24 @@ public function findAll(string $userId, int $modelId): array { return $this->findEntities($qb); } + /** + * Find a person that contains a face. + * + * @param string $userId ID of the user + * @param int $faceId ID of the face that belongs to the wanted person + * @return Person + */ + public function findFromFace(string $userId, int $faceId) { + $qb = $this->db->getQueryBuilder(); + $qb->select('p.id', 'p.name', 'p.is_valid') + ->from($this->getTableName(), 'p') + ->innerJoin('p', 'facerecog_faces' ,'f', $qb->expr()->eq('p.id', 'f.person')) + ->where($qb->expr()->eq('p.user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('f.id', $qb->createNamedParameter($faceId))); + + return $this->findEntity($qb); + } + /** * Returns count of persons (clusters) found for a given user. * diff --git a/lib/Db/Relation.php b/lib/Db/Relation.php new file mode 100644 index 00000000..c13e208a --- /dev/null +++ b/lib/Db/Relation.php @@ -0,0 +1,68 @@ + + * + * @author Matias De lellis + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\FaceRecognition\Db; + +use OCP\AppFramework\Db\Entity; + +/** + * Relation represents one relation beetwen two faces + * + * @method int getFace1() + * @method int getFace2() + * @method int getState() + * @method void setFace1(int $face1) + * @method void setFace2(int $face2) + * @method void setState(int $state) + */ +class Relation extends Entity { + + /** + * Possible values of the state of a face relation + */ + public const PROPOSED = 0; + public const ACCEPTED = 1; + public const REJECTED = 2; + + /** + * Face id of a face of a person related with $face2 + * + * @var int + * */ + public $face1; + + /** + * Face id of a face of a person related with $face1 + * + * @var int + * */ + public $face2; + + /** + * State of two face relation. These are proposed, and can be accepted + * as as the same person, or rejected. + * + * @var int + * */ + public $state; + +} \ No newline at end of file diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php new file mode 100644 index 00000000..f99c36b0 --- /dev/null +++ b/lib/Db/RelationMapper.php @@ -0,0 +1,193 @@ + + * + * @author Matias De lellis + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\FaceRecognition\Db; + +use OC\DB\QueryBuilder\Literal; + +use OCP\IDBConnection; +use OCP\AppFramework\Db\QBMapper; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\DB\QueryBuilder\IQueryBuilder; + +class RelationMapper extends QBMapper { + + public function __construct(IDBConnection $db) { + parent::__construct($db, 'facerecog_relations', '\OCA\FaceRecognition\Db\Relation'); + } + + /** + * Find all relation from that user. + * + * @param string $userId User user to search + * @param int $modelId + * @return array + */ + public function findByUser(string $userId, int $modelId): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') + ->from($this->getTableName(), 'r') + ->innerJoin('r', 'facerecog_faces', 'f', $qb->expr()->eq('r.face1', 'f.id')) + ->innerJoin('f', 'facerecog_images', 'i', $qb->expr()->eq('f.image', 'i.id')) + ->where($qb->expr()->eq('i.user', $qb->createParameter('user_id'))) + ->andWhere($qb->expr()->eq('i.model', $qb->createParameter('model_id'))) + ->setParameter('user_id', $userId) + ->setParameter('model_id', $modelId); + + return $this->findEntities($qb); + } + + public function findFromPerson(string $userId, int $personId, int $state): array { + $sub = $this->db->getQueryBuilder(); + $sub->select('f.id') + ->from('facerecog_faces', 'f') + ->innerJoin('f', 'facerecog_persons' ,'p', $sub->expr()->eq('f.person', 'p.id')) + ->where($sub->expr()->eq('p.user', $sub->createParameter('user_id'))) + ->andWhere($sub->expr()->eq('f.person', $sub->createParameter('person_id'))); + + $qb = $this->db->getQueryBuilder(); + $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') + ->from($this->getTableName(), 'r') + ->where($qb->expr()->eq('r.state', $qb->createParameter('state'))) + ->andWhere('(r.face1 IN (' . $sub->getSQL() . '))') + ->orWhere('(r.face2 IN (' . $sub->getSQL() . '))') + ->setParameter('user_id', $userId) + ->setParameter('person_id', $personId) + ->setParameter('state', $state); + + return $this->findEntities($qb); + } + + public function findFromPersons(int $personId1, int $personId2) { + $sub1 = $this->db->getQueryBuilder(); + $sub1->select('f.id') + ->from('facerecog_faces', 'f') + ->where($sub1->expr()->eq('f.person', $sub1->createParameter('person1'))); + + $sub2 = $this->db->getQueryBuilder(); + $sub2->select('f.id') + ->from('facerecog_faces', 'f') + ->where($sub2->expr()->eq('f.person', $sub2->createParameter('person2'))); + + $qb = $this->db->getQueryBuilder(); + $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') + ->from($this->getTableName(), 'r') + ->where('((r.face1 IN (' . $sub1->getSQL() . ')) AND (r.face2 IN (' . $sub2->getSQL() . ')))') + ->orWhere('((r.face2 IN (' . $sub1->getSQL() . ')) AND (r.face1 IN (' . $sub2->getSQL() . ')))') + ->setParameter('person1', $personId1) + ->setParameter('person2', $personId2); + + return $this->findEntities($qb); + } + + /** + * Deletes all relations from that user. + * + * @param string $userId User to drop persons from a table. + */ + public function deleteUser(string $userId) { + $sub = $this->db->getQueryBuilder(); + $sub->select(new Literal('1')) + ->from('facerecog_faces', 'f') + ->innerJoin('f', 'facerecog_images', 'i', $sub->expr()->eq('f.image', 'i.id')) + ->andWhere($sub->expr()->eq('i.user', $sub->createParameter('user_id'))); + + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where('EXISTS (' . $sub->getSQL() . ')') + ->setParameter('user_id', $userId) + ->execute(); + } + + /** + * Find all the relations of a user as an matrix array, which is faster to access. + * @param string $userId + * @param int $modelId + * return array + */ + public function findByUserAsMatrix(string $userId, int $modelId): array { + $matrix = array(); + $relations = $this->findByUser($userId, $modelId); + foreach ($relations as $relation) { + $row = array(); + if (isset($matrix[$relation->face1])) { + $row = $matrix[$relation->face1]; + } + $row[$relation->face2] = $relation->state; + $matrix[$relation->face1] = $row; + } + return $matrix; + } + + public function getStateOnMatrix(int $face1, int $face2, array $matrix): int { + if (isset($matrix[$face1])) { + $row = $matrix[$face1]; + if (isset($row[$face2])) { + return $matrix[$face1][$face2]; + } + } + if (isset($matrix[$face2])) { + $row = $matrix[$face2]; + if (isset($row[$face1])) { + return $matrix[$face2][$face1]; + } + } + return Relation::PROPOSED; + } + + public function existsOnMatrix(int $face1, int $face2, array $matrix): bool { + if (isset($matrix[$face1])) { + $row = $matrix[$face1]; + if (isset($row[$face2])) { + return true; + } + } + if (isset($matrix[$face2])) { + $row = $matrix[$face2]; + if (isset($row[$face1])) { + return true; + } + } + return false; + } + + public function merge(string $userId, int $modelId, array $relations): int { + $added = 0; + $this->db->beginTransaction(); + try { + $oldMatrix = $this->findByUserAsMatrix($userId, $modelId); + foreach ($relations as $relation) { + if ($this->existsOnMatrix($relation->face1, $relation->face2, $oldMatrix)) + continue; + $this->insert($relation); + $added++; + } + $this->db->commit(); + } catch (\Exception $e) { + $this->db->rollBack(); + throw $e; + } + return $added; + } + +} \ No newline at end of file diff --git a/lib/Migration/Version000516Date20200420003814.php b/lib/Migration/Version000516Date20200420003814.php new file mode 100644 index 00000000..1b2de535 --- /dev/null +++ b/lib/Migration/Version000516Date20200420003814.php @@ -0,0 +1,66 @@ +hasTable('facerecog_relations')) { + $table = $schema->createTable('facerecog_relations'); + $table->addColumn('id', 'integer', [ + 'autoincrement' => true, + 'notnull' => true, + 'unsigned' => true, + ]); + $table->addColumn('face1', 'integer', [ + 'notnull' => true, + 'length' => 4, + ]); + $table->addColumn('face2', 'integer', [ + 'notnull' => true, + 'length' => 4, + ]); + $table->addColumn('state', 'integer', [ + 'notnull' => true, + 'length' => 4, + ]); + + $table->setPrimaryKey(['id']); + $table->addIndex(['face1'], 'relation_faces_1_idx'); + $table->addIndex(['face2'], 'relation_faces_2_idx'); + } + return $schema; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) { + } +} diff --git a/lib/Service/FaceManagementService.php b/lib/Service/FaceManagementService.php index 1caee95f..8c25f29a 100644 --- a/lib/Service/FaceManagementService.php +++ b/lib/Service/FaceManagementService.php @@ -30,6 +30,7 @@ use OCA\FaceRecognition\Db\FaceMapper; use OCA\FaceRecognition\Db\ImageMapper; use OCA\FaceRecognition\Db\PersonMapper; +use OCA\FaceRecognition\Db\RelationMapper; use OCA\FaceRecognition\Service\SettingsService; @@ -56,6 +57,9 @@ class FaceManagementService { /** @var PersonMapper */ private $personMapper; + /** @var RelationMapper */ + private $relationMapper; + /** @var SettingsService */ private $settingsService; @@ -63,12 +67,14 @@ public function __construct(IUserManager $userManager, FaceMapper $faceMapper, ImageMapper $imageMapper, PersonMapper $personMapper, + RelationMapper $relationMapper, SettingsService $settingsService) { $this->userManager = $userManager; $this->faceMapper = $faceMapper; $this->imageMapper = $imageMapper; $this->personMapper = $personMapper; + $this->relationMapper = $relationMapper; $this->settingsService = $settingsService; } @@ -134,6 +140,7 @@ public function resetClustersForUser(string $userId) { $this->faceMapper->unsetPersonsRelationForUser($userId, $model); $this->personMapper->deleteUserPersons($userId); + $this->relationMapper->deleteUser($userId); } /** diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php index 6878fd26..8c9a9938 100644 --- a/lib/Service/SettingsService.php +++ b/lib/Service/SettingsService.php @@ -57,6 +57,12 @@ class SettingsService { const DEFAULT_SENSITIVITY = '0.4'; const MAXIMUM_SENSITIVITY = '0.6'; + /** Deviation used to suggestions */ + const DEVIATION_KEY = 'deviation'; + const MINIMUM_DEVIATION = '0.0'; + const DEFAULT_DEVIATION = '0.0'; + const MAXIMUM_DEVIATION = '0.2'; + /** Minimum confidence used to try to clustring faces */ const MINIMUM_CONFIDENCE_KEY = 'min_confidence'; const MINIMUM_MINIMUM_CONFIDENCE = '0.0'; @@ -220,6 +226,14 @@ public function setSensitivity($sensitivity) { $this->config->setAppValue(Application::APP_NAME, self::SENSITIVITY_KEY, $sensitivity); } + public function getDeviation(): float { + return floatval($this->config->getAppValue(Application::APP_NAME, self::DEVIATION_KEY, self::DEFAULT_DEVIATION)); + } + + public function setDeviation($deviation) { + $this->config->setAppValue(Application::APP_NAME, self::DEVIATION_KEY, $deviation); + } + public function getMinimumConfidence(): float { return floatval($this->config->getAppValue(Application::APP_NAME, self::MINIMUM_CONFIDENCE_KEY, self::DEFAULT_MINIMUM_CONFIDENCE)); } diff --git a/templates/settings/admin.php b/templates/settings/admin.php index e8a9e680..beffdb24 100644 --- a/templates/settings/admin.php +++ b/templates/settings/admin.php @@ -40,6 +40,21 @@


+

+ t('Deviation for suggestions'));?> +

+

t('The deviation is numerically added to the sensitivity to compare the clusters obtained by that sensitivity and suggest renaming similar clusters.'));?> + +

+

+ + + + ... + + +

+

t('Minimum confidence'));?>

diff --git a/templates/settings/personal.php b/templates/settings/personal.php index f4119ef4..39a55117 100644 --- a/templates/settings/personal.php +++ b/templates/settings/personal.php @@ -4,6 +4,7 @@ vendor_script('facerecognition', 'js/egg'); script('facerecognition', 'templates'); script('facerecognition', 'fr-dialogs'); +script('facerecognition', 'similar'); script('facerecognition', 'personal'); style('facerecognition', 'facerecognition'); ?>