Commit 04996833 authored by Dainis Abols's avatar Dainis Abols
Browse files

Remade death field. Added source output

1.0.6
parent 3f8943b1
......@@ -69,6 +69,7 @@ class ImportCommand extends Command
$familyStatusId = 0;
$sources = [];
$personId = 0;
$fullDeathText = '';
foreach ($row as $cell => $data) {
// Save photo id
......@@ -83,11 +84,21 @@ class ImportCommand extends Command
$date = [0, 0, 0];
$temp = [];
if (strlen($row[$cell]) != 0) {
$temp = explode(" ", $row[$cell]);
if(ctype_digit(str_replace(" ","",$row[$cell])) || $row[$cell] === '' || empty($row[$cell])){
if (strlen($row[$cell]) != 0) {
$temp = explode(" ", preg_replace('!\s+!', ' ', trim($row[$cell])));
}
} else {
file_put_contents('error.txt', $personId.' - invalid date - '.$row[$cell].PHP_EOL, FILE_APPEND);
}
// Save full death date text
if ($cell == 'J') {
$fullDeathText = $data ?? '';
}
unset($row[$cell]);
array_splice($date, 0, count($temp), $temp);
if ($cell == 'H') {
......@@ -104,22 +115,23 @@ class ImportCommand extends Command
// Create and link locations
if ($cell == 'I' || $cell == 'K' || $cell == 'L' || $cell == 'M' || $cell == 'N') {
$data = trim($data, ', ');
$locations = explode(', ', $data);
$locationIds = [];
foreach ($locations as $key => $location) {
$findLocation = $db->prepare(
"SELECT uid FROM tx_lunames_domain_model_location WHERE title = '".$location."' AND deleted = 0");
$findLocation->execute();
"SELECT uid FROM tx_lunames_domain_model_location WHERE title = :locationTitle AND deleted = 0");
$findLocation->execute(['locationTitle' => $location]);
// Add new location if it doesn't exist
if ($findLocation->rowCount() == 0 && !empty($location)) {
$locationInsert = '('.$storagePid.', '.time().', '.time().', '.'0, '.'0, '."'".$location."'".')';
$locationInsert = '('.$storagePid.', '.time().', '.time().', '.'0, '.'0, ?)';
$query = "INSERT INTO tx_lunames_domain_model_location (pid, tstamp, crdate, cruser_id, deleted, title) VALUES ".$locationInsert;
$addLocation = $db->prepare(
$query
);
$addLocation->execute();
$addLocation->execute([$location]);
// Get new record uid
$findLocation->execute();
......@@ -128,8 +140,6 @@ class ImportCommand extends Command
if (!empty($location)) {
$result = $findLocation->fetch();
$locationIds[] = $result['uid'];
} else {
$locationIds[] = '';
}
}
......@@ -139,17 +149,17 @@ class ImportCommand extends Command
// Create and link occupation
if ($cell == 'O') {
$findOccupation = $db->prepare(
"SELECT uid FROM tx_lunames_domain_model_occupation WHERE title = '".$data."' AND deleted = 0");
$findOccupation->execute();
"SELECT uid FROM tx_lunames_domain_model_occupation WHERE title = :occupationTitle AND deleted = 0");
$findOccupation->execute(['occupationTitle' => $data]);
// Add new occupation if it doesn't exist
if ($findOccupation->rowCount() == 0 && !empty($data)) {
$occupationInsert = '('.$storagePid.', '.time().', '.time().', '.'0, '.'0, '."'".$data."'".')';
$occupationInsert = '('.$storagePid.', '.time().', '.time().', '.'0, '.'0, ?)';
$query = "INSERT INTO tx_lunames_domain_model_occupation (pid, tstamp, crdate, cruser_id, deleted, title) VALUES ".$occupationInsert;
$addOccupation = $db->prepare(
$query
);
$addOccupation->execute();
$addOccupation->execute([$data]);
// Get new record uid
$findOccupation->execute();
......@@ -164,17 +174,17 @@ class ImportCommand extends Command
// Create and link family status
if ($cell == 'Q') {
$findFamilyStatus = $db->prepare(
"SELECT uid FROM tx_lunames_domain_model_family_status WHERE title = '".$data."' AND deleted = 0");
$findFamilyStatus->execute();
"SELECT uid FROM tx_lunames_domain_model_family_status WHERE title = :familyStatus AND deleted = 0");
$findFamilyStatus->execute(['familyStatus' => $data]);
// Add new family status if it doesn't exist
if ($findFamilyStatus->rowCount() == 0 && !empty($data)) {
$familyStatusInsert = '('.$storagePid.', '.time().', '.time().', '.'0, '.'0, '."'".$data."'".')';
$familyStatusInsert = '('.$storagePid.', '.time().', '.time().', '.'0, '.'0, ?)';
$query = "INSERT INTO tx_lunames_domain_model_family_status (pid, tstamp, crdate, cruser_id, deleted, title) VALUES ".$familyStatusInsert;
$addFamilyStatus = $db->prepare(
$query
);
$addFamilyStatus->execute();
$addFamilyStatus->execute([$data]);
// Get new record uid
$findFamilyStatus->execute();
......@@ -196,48 +206,49 @@ class ImportCommand extends Command
$slug = $personModel->generateSlug($personId, $row['B'], $row['E']);
$sourcesCount = empty($sources) ? '' : count(explode(' ', $sources));
$insert = '('.$personId.', '.$storagePid.', '.time().', '.time().', '.'0, '.'0, ';
$insert .= "'".implode("','", $row)."',";
$insert .= $occupationId.",".$familyStatusId.","."'".$slug."'".",".$sourcesCount.")";
$insert = [$personId, $storagePid, time(), time(), 0, 0, $occupationId, $familyStatusId, $slug, $sourcesCount, $fullDeathText];
// Add remaining values converting null to string for MySQL strict mode
array_splice($insert, 6, 0, array_map('strval', $row));
$values[] = ['sources' => $sources, 'insert' => $insert, 'personId' => $personId];
}
// Construct person insert values
$insertValues = [];
foreach ($values as $value) {
$insertValues[] = $value['insert'];
}
// Add persons
$statement = $db->prepare(
'INSERT INTO tx_lunames_domain_model_person (uid, pid, tstamp, crdate, cruser_id, deleted, family_name,
"INSERT INTO tx_lunames_domain_model_person (uid, pid, tstamp, crdate, cruser_id, deleted, family_name,
maiden_name, other_name, given_name, father, mother, place_of_birth, year_of_birth, day_of_birth, month_of_birth,
year_of_death, day_of_death, month_of_death, place_of_death, region, prewar_residence, war_residence, gender, spouse, fate,
comments, occupation, family_status, slug, source_codes)
VALUES '.implode(',',$insertValues)
);
$statement->execute();
comments, occupation, family_status, slug, source_codes, death)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
$findUid = $db->prepare(
"SELECT uid FROM tx_lunames_domain_model_person WHERE uid = :uid AND deleted = 0");
foreach ($values as $key => $value) {
$findUid->execute(['uid' => $value['personId']]);
if ($findUid->rowCount() == 0) {
$statement->execute($value['insert']);
} else {
file_put_contents('not-imported.txt', $value['personId'].' - duplicate uid'.PHP_EOL, FILE_APPEND);
unset($values[$key]);
}
}
// Construct source insert values
$insertValues = [];
// Link sources
$statement = $db->prepare(
'INSERT INTO tx_lunames_domain_model_related_source (uid_local, uid_foreign, sorting, sorting_foreign) VALUES (?, ?, ?, ?)'
);
foreach ($values as $value) {
$sources = explode(' ', $value['sources']);
foreach ($sources as $sourceKey => $source) {
$insert = '';
$source = (int)ltrim(rtrim($source, ']'), '[');
$sorting = $sourceKey + 1;
$insert = '('.$value['personId'].', '.$source.', '.$sorting.', 0)';
$insertValues[] = $insert;
$insert = [$value['personId'], $source, $sorting, 0];
$statement->execute($insert);
}
}
// Link sources
$statement = $db->prepare(
'INSERT INTO tx_lunames_domain_model_related_source (uid_local, uid_foreign, sorting, sorting_foreign) VALUES '.implode(',',$insertValues)
);
$statement->execute();
// Link images
$resourceFactory = ResourceFactory::getInstance();
$storage = $resourceFactory->getDefaultStorage();
......
......@@ -4,6 +4,8 @@ namespace Lu\LuNames\Controller;
use Lu\LuNames\Domain\Model\Person;
use Lu\LuNames\Domain\Repository\PersonRepository;
use Lu\LuNames\Domain\Repository\SourceRepository;
use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use TYPO3\CMS\Core\Resource\FileRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Mvc\Controller\ActionController;
......@@ -28,6 +30,19 @@ class FrontController extends ActionController
$this->personRepository = $personRepository;
}
/**
* @var \Lu\LuNames\Domain\Repository\SourceRepository
*/
private SourceRepository $sourceRepository;
/**
* @param \Lu\LuNames\Domain\Repository\SourceRepository $sourceRepository
*/
public function injectSourceRepository(SourceRepository $sourceRepository)
{
$this->sourceRepository = $sourceRepository;
}
/**
* Detail action
*/
......@@ -46,6 +61,21 @@ class FrontController extends ActionController
$this->view->assignMultiple($assign);
}
public function listSourcesAction()
{
$config = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('lu_names');
$storagePageId = (int)$config['storage_page_id'];
// Set storage page id
$querySettings = $this->sourceRepository->createQuery()->getQuerySettings();
$querySettings->setStoragePageIds([$storagePageId]);
$this->sourceRepository->setDefaultQuerySettings($querySettings);
$sources = $this->sourceRepository->findAll();
$this->view->assign('sources', $sources);
}
/**
* Get photo from UID
*
......
......@@ -2,10 +2,11 @@
namespace Lu\LuNames\Controller;
use Lu\LuApi\DataSources\SolrLibrary;
use Lu\LuApi\DataSources\SolrNames;
use Lu\LuApi\Helpers\DataHelper;
use Lu\LuNames\Domain\Model\Person;
use Lu\LuNames\Domain\Repository\PersonRepository;
use Lu\LuNames\Helpers\CacheHelper;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Page\PageRenderer;
use TYPO3\CMS\Core\Resource\FileRepository;
......@@ -81,6 +82,11 @@ class SearchController extends ActionController
*/
private $parentIDs = [];
/**
* @var PersonRepository
*/
private PersonRepository $personRepository;
/**
* MusicController constructor.
*
......@@ -102,6 +108,11 @@ class SearchController extends ActionController
$this->uri_builder = GeneralUtility::makeInstance(UriBuilder::class)->setCreateAbsoluteUri(true)->reset();
}
public function injectPersonRepository(PersonRepository $personRepository)
{
$this->personRepository = $personRepository;
}
/**
* List Action
* Build search field and displays results.
......@@ -114,9 +125,17 @@ class SearchController extends ActionController
{
// Pre-define variables
$request = $this->request->getArguments();
$cacheHelper = new CacheHelper();
// Generate letter list
$letters = $this->getAllALetters();
$letters = $cacheHelper->get('letters');
if(!$letters) {
$letters = $this->personRepository->getAllALetters();
$cacheHelper->set('letters', $letters);
}
$this->addLetterLinks($letters);
// Get current page @FIXME There has to be a better way!
$this->currentPage = !empty($request['@widget_0']['currentPage']) ? (int)$request['@widget_0']['currentPage'] : 1;
......@@ -355,43 +374,18 @@ class SearchController extends ActionController
}
/**
* Returns array of unique letters from SOLR
* Adds URLs to letter menu
*
* @return array
* @param $letters
*
* @return void
*/
private function getAllALetters()
private function addLetterLinks(&$letters)
{
// Pre-set models and values
$letters = [];
$dataHelper = new DataHelper();
$this->uri_builder = $this->controllerContext->getUriBuilder()->setCreateAbsoluteUri(true)->reset();
// Search in SOLR
$solr = new SolrNames();
$solr->setLimit(SolrLibrary::MAXINT);
// Fetch data
$jsonData = $solr->fetch();
$result = $dataHelper->APIDecode($jsonData);
// Gather letter info
$tmp = [];
foreach ($result as $item) {
@$tmp[strtolower(substr($item['family_name'], 0, 1))]++;
}
// Build letter array
foreach ($tmp as $key => $item) {
$letters[$key] = [
'key' => strtoupper($key),
'count' => $item,
'url' => $this->uri_builder->reset()->uriFor('list', ['letter' => $key]),
];
foreach ($letters as $key => $item) {
$letters[$key]['url'] = $this->uri_builder->reset()->uriFor('list', ['letter' => $key]);
}
// Resort and return
ksort($letters);
return $letters;
}
}
<?php
namespace Lu\LuNames\Domain\Model;
use TYPO3\CMS\Extbase\DomainObject\AbstractEntity;
/**
* Class Occupation
*
* @package Lu\LuNames\Domain\Model
*/
class Occupation extends AbstractEntity
{
/**
* The name of the person
*
* @var string
**/
protected string $title = '';
/**
* @return string
*/
public function getTitle(): string
{
return $this->title;
}
}
......@@ -87,6 +87,11 @@ class Person extends AbstractEntity
*/
protected int $dayOfDeath = 0;
/**
* @var string
*/
protected string $death = '';
/**
* @var \TYPO3\CMS\Extbase\Persistence\ObjectStorage<\Lu\LuNames\Domain\Model\Location>
*/
......@@ -108,9 +113,9 @@ class Person extends AbstractEntity
protected ?ObjectStorage $warResidence = null;
/**
* @var string
* @var \TYPO3\CMS\Extbase\Persistence\ObjectStorage<\Lu\LuNames\Domain\Model\Occupation>
*/
protected string $occupation = '';
protected ?ObjectStorage $occupation = null;
/**
* @var int
......@@ -138,9 +143,9 @@ class Person extends AbstractEntity
protected string $comments = '';
/**
* @var string
* @var \TYPO3\CMS\Extbase\Persistence\ObjectStorage<\Lu\LuNames\Domain\Model\Source>
*/
protected string $sourceCodes = '';
protected ?ObjectStorage $sourceCodes = null;
public function __construct()
{
......@@ -149,6 +154,8 @@ class Person extends AbstractEntity
$this->region = new ObjectStorage();
$this->warResidence = new ObjectStorage();
$this->prewarResidence = new ObjectStorage();
$this->occupation = new ObjectStorage();
$this->sourceCodes = new ObjectStorage();
}
/**
......@@ -265,6 +272,14 @@ class Person extends AbstractEntity
return $this->dayOfDeath;
}
/**
* @return string
*/
public function getDeath(): string
{
return $this->death;
}
/**
* @return \TYPO3\CMS\Extbase\Persistence\ObjectStorage
*/
......@@ -298,9 +313,9 @@ class Person extends AbstractEntity
}
/**
* @return string
* @return \TYPO3\CMS\Extbase\Persistence\ObjectStorage
*/
public function getOccupation(): string
public function getOccupation(): ObjectStorage
{
return $this->occupation;
}
......@@ -346,9 +361,9 @@ class Person extends AbstractEntity
}
/**
* @return string
* @return \TYPO3\CMS\Extbase\Persistence\ObjectStorage
*/
public function getSourceCodes(): string
public function getSourceCodes(): ObjectStorage
{
return $this->sourceCodes;
}
......
......@@ -35,4 +35,12 @@ class Source extends AbstractEntity
{
return $this->tableName;
}
/**
* @return string
*/
public function getTitle(): string
{
return $this->title;
}
}
......@@ -2,9 +2,9 @@
namespace Lu\LuNames\Domain\Repository;
use Lu\LuNames\Domain\Model\Person;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use Lu\LuApi\DataSources\SolrLibrary;
use Lu\LuApi\DataSources\SolrNames;
use Lu\LuApi\Helpers\DataHelper;
use TYPO3\CMS\Extbase\Persistence\Repository;
/**
......@@ -14,4 +14,42 @@ use TYPO3\CMS\Extbase\Persistence\Repository;
*/
class PersonRepository extends Repository
{
/**
* Returns array of unique letters from SOLR
*
* @return array
*/
public function getAllALetters()
{
// Pre-set models and values
$letters = [];
$dataHelper = new DataHelper();
// Search in SOLR
$solr = new SolrNames();
$solr->setLimit(SolrLibrary::MAXINT);
// Fetch data
$jsonData = $solr->fetch();
$result = $dataHelper->APIDecode($jsonData);
// Gather letter info
$tmp = [];
foreach ($result as $item) {
@$tmp[strtolower(substr($item['family_name'], 0, 1))]++;
}
// Build letter array
foreach ($tmp as $key => $item) {
$letters[$key] = [
'key' => strtoupper($key),
'count' => $item,
];
}
// Resort and return
ksort($letters);
return $letters;
}
}
<?php
namespace Lu\LuNames\Domain\Repository;
use TYPO3\CMS\Extbase\Persistence\Repository;
/**
* Class SourceRepository
*
* @package Lu\LuNames\Domain\Repository
*/
class SourceRepository extends Repository
{
}
<?php
namespace Lu\LuNames\Helpers;
class CacheHelper
{
private $tableName = 'tx_lunames_cache';
private $config;
private \PDO $db;
public function __construct()
{
$this->config = require '/home/portal/typo3/typo3conf/LocalConfiguration.php';
$this->db = new \PDO(
'mysql:host='.$this->config['DB']['Connections']['Default']['host'].';dbname='.$this->config['DB']['Connections']['Default']['dbname'],
$this->config['DB']['Connections']['Default']['user'],
$this->config['DB']['Connections']['Default']['password']
);
}
public function set($identifier, $content)
{
$query = $this->db->prepare("SELECT * FROM ".$this->tableName." WHERE identifier = :identifier");
$query->bindParam('identifier', $identifier, $this->db::PARAM_STR);
$query->execute();
if ($query->rowCount() == 0) {
$query = $this->db->prepare("INSERT INTO ".$this->tableName." (crdate, identifier, content) VALUES (?, ?, ?)");
$query->execute([time(), $identifier, serialize($content)]);
} else {
$query = $this->db->prepare("UPDATE ".$this->tableName." SET crdate = :crdate, content = :content WHERE identifier = :identifier");
$content = serialize($content);
$time = time();
$query->bindParam('crdate', $time);
$query->bindParam('content', $content);
$query->bindParam('identifier', $identifier);
$query->execute();
}
}
public function get($identifier)
{
$query = $this->db->prepare("SELECT * FROM ".$this->tableName." WHERE identifier = :identifier");
$query->bindParam('identifier', $identifier, $this->db::PARAM_STR);
$query->execute();
if ($query->rowCount() == 0) {
return false;
}
return unserialize($query->fetch()['content']);
}
}
<?php
namespace Lu\LuNames\Tasks;
use Lu\LuNames\Domain\Repository\PersonRepository;
use Lu\LuNames\Helpers\CacheHelper;
use TYPO3\CMS\Core\Utility\GeneralUtility;