mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-10-29 06:34:37 +00:00
631 lines
18 KiB
PHP
631 lines
18 KiB
PHP
<?php
|
|
/**
|
|
* ImportJobRepository.php
|
|
* Copyright (c) 2017 thegrumpydictator@gmail.com
|
|
*
|
|
* This file is part of Firefly III.
|
|
*
|
|
* Firefly III is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* Firefly III is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
declare(strict_types=1);
|
|
|
|
namespace FireflyIII\Repositories\ImportJob;
|
|
|
|
use Crypt;
|
|
use FireflyIII\Exceptions\FireflyException;
|
|
use FireflyIII\Models\Attachment;
|
|
use FireflyIII\Models\ImportJob;
|
|
use FireflyIII\Models\Tag;
|
|
use FireflyIII\Models\TransactionJournalMeta;
|
|
use FireflyIII\Repositories\User\UserRepositoryInterface;
|
|
use FireflyIII\User;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\MessageBag;
|
|
use Illuminate\Support\Str;
|
|
use Log;
|
|
use SplFileObject;
|
|
use Storage;
|
|
use Symfony\Component\HttpFoundation\File\UploadedFile;
|
|
|
|
/**
|
|
* Class ImportJobRepository.
|
|
*/
|
|
class ImportJobRepository implements ImportJobRepositoryInterface
|
|
{
|
|
/** @var \Illuminate\Contracts\Filesystem\Filesystem */
|
|
protected $uploadDisk;
|
|
/** @var int */
|
|
private $maxUploadSize;
|
|
/** @var User */
|
|
private $user;
|
|
|
|
public function __construct()
|
|
{
|
|
$this->maxUploadSize = (int)config('firefly.maxUploadSize');
|
|
$this->uploadDisk = Storage::disk('upload');
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param int $index
|
|
* @param string $error
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function addError(ImportJob $job, int $index, string $error): ImportJob
|
|
{
|
|
$extended = $this->getExtendedStatus($job);
|
|
$extended['errors'][$index][] = $error;
|
|
|
|
return $this->setExtendedStatus($job, $extended);
|
|
}
|
|
|
|
/**
|
|
* Add message to job.
|
|
*
|
|
* @param ImportJob $job
|
|
* @param string $error
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function addErrorMessage(ImportJob $job, string $error): ImportJob
|
|
{
|
|
$errors = $job->errors;
|
|
$errors[] = $error;
|
|
$job->errors = $errors;
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param int $steps
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function addStepsDone(ImportJob $job, int $steps = 1): ImportJob
|
|
{
|
|
$status = $this->getExtendedStatus($job);
|
|
$status['done'] += $steps;
|
|
Log::debug(sprintf('Add %d to steps done for job "%s" making steps done %d', $steps, $job->key, $status['done']));
|
|
|
|
return $this->setExtendedStatus($job, $status);
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param int $steps
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function addTotalSteps(ImportJob $job, int $steps = 1): ImportJob
|
|
{
|
|
$extended = $this->getExtendedStatus($job);
|
|
$total = (int)($extended['steps'] ?? 0);
|
|
$total += $steps;
|
|
$extended['steps'] = $total;
|
|
|
|
return $this->setExtendedStatus($job, $extended);
|
|
|
|
}
|
|
|
|
/**
|
|
* Return number of imported rows with this hash value.
|
|
*
|
|
* @param string $hash
|
|
*
|
|
* @return int
|
|
*/
|
|
public function countByHash(string $hash): int
|
|
{
|
|
$json = json_encode($hash);
|
|
$count = TransactionJournalMeta::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
|
|
->where('data', $json)
|
|
->where('name', 'importHash')
|
|
->count();
|
|
|
|
return (int)$count;
|
|
}
|
|
|
|
/**
|
|
* @param string $importProvider
|
|
*
|
|
* @return ImportJob
|
|
*
|
|
* @throws FireflyException
|
|
*/
|
|
public function create(string $importProvider): ImportJob
|
|
{
|
|
$count = 0;
|
|
$importProvider = strtolower($importProvider);
|
|
|
|
while ($count < 30) {
|
|
$key = Str::random(12);
|
|
$existing = $this->findByKey($key);
|
|
if (null === $existing->id) {
|
|
$importJob = ImportJob::create(
|
|
[
|
|
'user_id' => $this->user->id,
|
|
'tag_id' => null,
|
|
'provider' => $importProvider,
|
|
'file_type' => '',
|
|
'key' => Str::random(12),
|
|
'status' => 'new',
|
|
'stage' => 'new',
|
|
'configuration' => [],
|
|
'extended_status' => [],
|
|
'transactions' => [],
|
|
'errors' => [],
|
|
]
|
|
);
|
|
|
|
// breaks the loop:
|
|
return $importJob;
|
|
}
|
|
++$count;
|
|
}
|
|
throw new FireflyException('Could not create an import job with a unique key after 30 tries.');
|
|
}
|
|
|
|
/**
|
|
* @param string $key
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function findByKey(string $key): ImportJob
|
|
{
|
|
/** @var ImportJob $result */
|
|
$result = $this->user->importJobs()->where('key', $key)->first(['import_jobs.*']);
|
|
if (null === $result) {
|
|
return new ImportJob;
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
/**
|
|
* Return all attachments for job.
|
|
*
|
|
* @param ImportJob $job
|
|
*
|
|
* @return Collection
|
|
*/
|
|
public function getAttachments(ImportJob $job): Collection
|
|
{
|
|
return $job->attachments()->get();
|
|
}
|
|
|
|
/**
|
|
* Return configuration of job.
|
|
*
|
|
* @param ImportJob $job
|
|
*
|
|
* @return array
|
|
*/
|
|
public function getConfiguration(ImportJob $job): array
|
|
{
|
|
$config = $job->configuration;
|
|
if (\is_array($config)) {
|
|
return $config;
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
/**
|
|
* Return extended status of job.
|
|
*
|
|
* @param ImportJob $job
|
|
*
|
|
* @return array
|
|
*/
|
|
public function getExtendedStatus(ImportJob $job): array
|
|
{
|
|
$status = $job->extended_status;
|
|
if (\is_array($status)) {
|
|
return $status;
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
*
|
|
* @return string
|
|
*/
|
|
public function getStatus(ImportJob $job): string
|
|
{
|
|
return $job->status;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param UploadedFile $file
|
|
*
|
|
* @return bool
|
|
*/
|
|
public function processConfiguration(ImportJob $job, UploadedFile $file): bool
|
|
{
|
|
/** @var UserRepositoryInterface $repository */
|
|
$repository = app(UserRepositoryInterface::class);
|
|
// demo user's configuration upload is ignored completely.
|
|
if (!$repository->hasRole($this->user, 'demo')) {
|
|
Log::debug(
|
|
'Uploaded configuration file',
|
|
['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]
|
|
);
|
|
|
|
$configFileObject = new SplFileObject($file->getRealPath());
|
|
$configRaw = $configFileObject->fread($configFileObject->getSize());
|
|
$configuration = json_decode($configRaw, true);
|
|
Log::debug(sprintf('Raw configuration is %s', $configRaw));
|
|
if (null !== $configuration && \is_array($configuration)) {
|
|
Log::debug('Found configuration', $configuration);
|
|
$this->setConfiguration($job, $configuration);
|
|
}
|
|
if (null === $configuration) {
|
|
Log::error('Uploaded configuration is NULL');
|
|
}
|
|
if (false === $configuration) {
|
|
Log::error('Uploaded configuration is FALSE');
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param null|UploadedFile $file
|
|
*
|
|
* @return bool
|
|
*
|
|
* @throws \Illuminate\Contracts\Encryption\EncryptException
|
|
* @throws \Illuminate\Contracts\Filesystem\FileNotFoundException
|
|
*/
|
|
public function processFile(ImportJob $job, ?UploadedFile $file): bool
|
|
{
|
|
if (null === $file) {
|
|
return false;
|
|
}
|
|
/** @var UserRepositoryInterface $repository */
|
|
$repository = app(UserRepositoryInterface::class);
|
|
$newName = sprintf('%s.upload', $job->key);
|
|
$uploaded = new SplFileObject($file->getRealPath());
|
|
$content = trim($uploaded->fread($uploaded->getSize()));
|
|
|
|
// verify content:
|
|
$result = mb_detect_encoding($content, 'UTF-8', true);
|
|
if ($result === false) {
|
|
Log::error(sprintf('Cannot detect encoding for uploaded import file "%s".', $file->getClientOriginalName()));
|
|
|
|
return false;
|
|
}
|
|
if ($result !== 'ASCII' && $result !== 'UTF-8') {
|
|
Log::error(sprintf('Uploaded import file is %s instead of UTF8!', var_export($result, true)));
|
|
|
|
return false;
|
|
}
|
|
|
|
$contentEncrypted = Crypt::encrypt($content);
|
|
$disk = Storage::disk('upload');
|
|
|
|
// user is demo user, replace upload with prepared file.
|
|
if ($repository->hasRole($this->user, 'demo')) {
|
|
$stubsDisk = Storage::disk('stubs');
|
|
$content = $stubsDisk->get('demo-import.csv');
|
|
$contentEncrypted = Crypt::encrypt($content);
|
|
$disk->put($newName, $contentEncrypted);
|
|
Log::debug('Replaced upload with demo file.');
|
|
|
|
// also set up prepared configuration.
|
|
$configuration = json_decode($stubsDisk->get('demo-configuration.json'), true);
|
|
$this->setConfiguration($job, $configuration);
|
|
Log::debug('Set configuration for demo user', $configuration);
|
|
}
|
|
|
|
if (!$repository->hasRole($this->user, 'demo')) {
|
|
// user is not demo, process original upload:
|
|
$disk->put($newName, $contentEncrypted);
|
|
Log::debug('Uploaded file', ['name' => $file->getClientOriginalName(), 'size' => $file->getSize(), 'mime' => $file->getClientMimeType()]);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param array $configuration
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setConfiguration(ImportJob $job, array $configuration): ImportJob
|
|
{
|
|
Log::debug('Updating configuration...');
|
|
//Log::debug(sprintf('Incoming config for job "%s" is: ', $job->key), $configuration);
|
|
$currentConfig = $job->configuration;
|
|
$newConfig = array_merge($currentConfig, $configuration);
|
|
$job->configuration = $newConfig;
|
|
$job->save();
|
|
|
|
//Log::debug(sprintf('Set config of job "%s" to: ', $job->key), $newConfig);
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param array $array
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setExtendedStatus(ImportJob $job, array $array): ImportJob
|
|
{
|
|
$currentStatus = $job->extended_status;
|
|
$newStatus = array_merge($currentStatus, $array);
|
|
$job->extended_status = $newStatus;
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param string $stage
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setStage(ImportJob $job, string $stage): ImportJob
|
|
{
|
|
$job->stage = $stage;
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param string $status
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setStatus(ImportJob $job, string $status): ImportJob
|
|
{
|
|
Log::debug(sprintf('Set status of job "%s" to "%s"', $job->key, $status));
|
|
$job->status = $status;
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param int $steps
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setStepsDone(ImportJob $job, int $steps): ImportJob
|
|
{
|
|
$status = $this->getExtendedStatus($job);
|
|
$status['done'] = $steps;
|
|
Log::debug(sprintf('Set steps done for job "%s" to %d', $job->key, $steps));
|
|
|
|
return $this->setExtendedStatus($job, $status);
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param Tag $tag
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setTag(ImportJob $job, Tag $tag): ImportJob
|
|
{
|
|
$job->tag()->associate($tag);
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param int $count
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setTotalSteps(ImportJob $job, int $count): ImportJob
|
|
{
|
|
$status = $this->getExtendedStatus($job);
|
|
$status['steps'] = $count;
|
|
Log::debug(sprintf('Set total steps for job "%s" to %d', $job->key, $count));
|
|
|
|
return $this->setExtendedStatus($job, $status);
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param array $transactions
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function setTransactions(ImportJob $job, array $transactions): ImportJob
|
|
{
|
|
$job->transactions = $transactions;
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* @param User $user
|
|
*/
|
|
public function setUser(User $user)
|
|
{
|
|
$this->user = $user;
|
|
}
|
|
|
|
/**
|
|
* Handle upload for job.
|
|
*
|
|
* @param ImportJob $job
|
|
* @param string $name
|
|
* @param UploadedFile $file
|
|
*
|
|
* @return MessageBag
|
|
* @throws FireflyException
|
|
*/
|
|
public function storeCLIUpload(ImportJob $job, string $name, string $fileName): MessageBag
|
|
{
|
|
$messages = new MessageBag;
|
|
|
|
if (!file_exists($fileName)) {
|
|
$messages->add('notfound', sprintf('File not found: %s', $fileName));
|
|
|
|
return $messages;
|
|
}
|
|
|
|
$count = $job->attachments()->get()->filter(
|
|
function (Attachment $att) use ($name) {
|
|
return $att->filename === $name;
|
|
}
|
|
)->count();
|
|
|
|
if ($count > 0) {
|
|
// don't upload, but also don't complain about it.
|
|
Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name));
|
|
|
|
return new MessageBag;
|
|
}
|
|
$content = file_get_contents($fileName);
|
|
$attachment = new Attachment; // create Attachment object.
|
|
$attachment->user()->associate($job->user);
|
|
$attachment->attachable()->associate($job);
|
|
$attachment->md5 = md5($content);
|
|
$attachment->filename = $name;
|
|
$attachment->mime = 'plain/txt';
|
|
$attachment->size = \strlen($content);
|
|
$attachment->uploaded = 0;
|
|
$attachment->save();
|
|
$encrypted = Crypt::encrypt($content);
|
|
|
|
// store it:
|
|
$this->uploadDisk->put($attachment->fileName(), $encrypted);
|
|
$attachment->uploaded = 1; // update attachment
|
|
$attachment->save();
|
|
|
|
// return it.
|
|
return new MessageBag;
|
|
}
|
|
|
|
/**
|
|
* Handle upload for job.
|
|
*
|
|
* @param ImportJob $job
|
|
* @param string $name
|
|
* @param UploadedFile $file
|
|
*
|
|
* @return MessageBag
|
|
* @throws FireflyException
|
|
*/
|
|
public function storeFileUpload(ImportJob $job, string $name, UploadedFile $file): MessageBag
|
|
{
|
|
$messages = new MessageBag;
|
|
if ($this->validSize($file)) {
|
|
$name = e($file->getClientOriginalName());
|
|
$messages->add('size', (string)trans('validation.file_too_large', ['name' => $name]));
|
|
|
|
return $messages;
|
|
}
|
|
$count = $job->attachments()->get()->filter(
|
|
function (Attachment $att) use ($name) {
|
|
return $att->filename === $name;
|
|
}
|
|
)->count();
|
|
|
|
if ($count > 0) {
|
|
// don't upload, but also don't complain about it.
|
|
Log::error(sprintf('Detected duplicate upload. Will ignore second "%s" file.', $name));
|
|
|
|
return new MessageBag;
|
|
}
|
|
|
|
$attachment = new Attachment; // create Attachment object.
|
|
$attachment->user()->associate($job->user);
|
|
$attachment->attachable()->associate($job);
|
|
$attachment->md5 = md5_file($file->getRealPath());
|
|
$attachment->filename = $name;
|
|
$attachment->mime = $file->getMimeType();
|
|
$attachment->size = $file->getSize();
|
|
$attachment->uploaded = 0;
|
|
$attachment->save();
|
|
$fileObject = $file->openFile('r');
|
|
$fileObject->rewind();
|
|
$content = $fileObject->fread($file->getSize());
|
|
$encrypted = Crypt::encrypt($content);
|
|
|
|
// store it:
|
|
$this->uploadDisk->put($attachment->fileName(), $encrypted);
|
|
$attachment->uploaded = 1; // update attachment
|
|
$attachment->save();
|
|
|
|
// return it.
|
|
return new MessageBag;
|
|
}
|
|
|
|
/**
|
|
* @param ImportJob $job
|
|
* @param string $status
|
|
*
|
|
* @return ImportJob
|
|
*/
|
|
public function updateStatus(ImportJob $job, string $status): ImportJob
|
|
{
|
|
$job->status = $status;
|
|
$job->save();
|
|
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* Return import file content.
|
|
*
|
|
* @deprecated
|
|
*
|
|
* @param ImportJob $job
|
|
*
|
|
* @return string
|
|
* @throws \Illuminate\Contracts\Filesystem\FileNotFoundException
|
|
*/
|
|
public function uploadFileContents(ImportJob $job): string
|
|
{
|
|
return $job->uploadFileContents();
|
|
}
|
|
|
|
/**
|
|
* @codeCoverageIgnore
|
|
*
|
|
* @param UploadedFile $file
|
|
*
|
|
* @return bool
|
|
*/
|
|
protected function validSize(UploadedFile $file): bool
|
|
{
|
|
$size = $file->getSize();
|
|
|
|
return $size > $this->maxUploadSize;
|
|
}
|
|
}
|