kenmirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2026-01-07 14:41:20 +00:00
Code Issues Projects Releases Wiki Activity

Collect account balances, optimized.

Browse Source
This commit is contained in:
James Cole
2025-08-06 20:15:02 +02:00
parent 0ad6beb66c
commit 70071767ab
7 changed files with 276 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
app/Support/Steam.php
View File

@@ -24,8 +24,10 @@ declare(strict_types=1);
namespace FireflyIII\Support;
use Carbon\Carbon;
use Exception;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountMeta;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionCurrency;
use FireflyIII\Support\Facades\Amount;
@@ -34,11 +36,9 @@ use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use Exception;
use ValueError;
use function Safe\preg_replace;
use function Safe\parse_url;
use function Safe\preg_replace;
/**
* Class Steam.
@@ -64,10 +64,10 @@ class Steam
// Log::debug(sprintf('Trying bcround("%s",%d)', $number, $precision));
if (str_contains($number, '.')) {
if ('-' !== $number[0]) {
return bcadd($number, '0.'.str_repeat('0', $precision).'5', $precision);
return bcadd($number, '0.' . str_repeat('0', $precision) . '5', $precision);
}
return bcsub($number, '0.'.str_repeat('0', $precision).'5', $precision);
return bcsub($number, '0.' . str_repeat('0', $precision) . '5', $precision);
}
return $number;
@@ -203,7 +203,7 @@ class Steam
Log::debug(sprintf('finalAccountBalanceInRange(#%d, %s, %s)', $account->id, $start->format('Y-m-d H:i:s'), $end->format('Y-m-d H:i:s')));
// set up cache
$cache = new CacheProperties();
$cache = new CacheProperties();
$cache->addProperty($account->id);
$cache->addProperty('final-balance-in-range');
$cache->addProperty($start);
@@ -213,21 +213,21 @@ class Steam
return $cache->get();
}
$balances = [];
$formatted = $start->format('Y-m-d');
$balances = [];
$formatted = $start->format('Y-m-d');
/*
* To make sure the start balance is correct, we need to get the balance at the exact end of the previous day.
* Since we just did "startOfDay" we can do subDay()->endOfDay() to get the correct moment.
* THAT will be the start balance.
*/
$request = clone $start;
$request = clone $start;
$request->subDay()->endOfDay();
Log::debug(sprintf('finalAccountBalanceInRange: Call finalAccountBalance with date/time "%s"', $request->toIso8601String()));
$startBalance = $this->finalAccountBalance($account, $request);
$primaryCurrency = Amount::getPrimaryCurrencyByUserGroup($account->user->userGroup);
$accountCurrency = $this->getAccountCurrency($account);
$hasCurrency = $accountCurrency instanceof TransactionCurrency;
$currency = $accountCurrency ?? $primaryCurrency;
$startBalance = $this->finalAccountBalance($account, $request);
$primaryCurrency = Amount::getPrimaryCurrencyByUserGroup($account->user->userGroup);
$accountCurrency = $this->getAccountCurrency($account);
$hasCurrency = $accountCurrency instanceof TransactionCurrency;
$currency = $accountCurrency ?? $primaryCurrency;
Log::debug(sprintf('Currency is %s', $currency->code));
@@ -240,7 +240,7 @@ class Steam
Log::debug(sprintf('Also set start balance in %s', $primaryCurrency->code));
$startBalance[$primaryCurrency->code] ??= '0';
}
$currencies = [
$currencies = [
$currency->id => $currency,
$primaryCurrency->id => $primaryCurrency,
];
@@ -250,66 +250,65 @@ class Steam
// sums up the balance changes per day.
Log::debug(sprintf('Date >= %s and <= %s', $start->format('Y-m-d H:i:s'), $end->format('Y-m-d H:i:s')));
$set = $account->transactions()
->leftJoin('transaction_journals', 'transactions.transaction_journal_id', '=', 'transaction_journals.id')
->where('transaction_journals.date', '>=', $start->format('Y-m-d H:i:s'))
->where('transaction_journals.date', '<=', $end->format('Y-m-d H:i:s'))
->groupBy('transaction_journals.date')
->groupBy('transactions.transaction_currency_id')
->orderBy('transaction_journals.date', 'ASC')
->whereNull('transaction_journals.deleted_at')
->get(
[ // @phpstan-ignore-line
'transaction_journals.date',
'transactions.transaction_currency_id',
DB::raw('SUM(transactions.amount) AS sum_of_day'),
]
)
;
$set = $account->transactions()
->leftJoin('transaction_journals', 'transactions.transaction_journal_id', '=', 'transaction_journals.id')
->where('transaction_journals.date', '>=', $start->format('Y-m-d H:i:s'))
->where('transaction_journals.date', '<=', $end->format('Y-m-d H:i:s'))
->groupBy('transaction_journals.date')
->groupBy('transactions.transaction_currency_id')
->orderBy('transaction_journals.date', 'ASC')
->whereNull('transaction_journals.deleted_at')
->get(
[ // @phpstan-ignore-line
'transaction_journals.date',
'transactions.transaction_currency_id',
DB::raw('SUM(transactions.amount) AS sum_of_day'),
]
);
$currentBalance = $startBalance;
$converter = new ExchangeRateConverter();
$currentBalance = $startBalance;
$converter = new ExchangeRateConverter();
/** @var Transaction $entry */
foreach ($set as $entry) {
// get date object
$carbon = new Carbon($entry->date, $entry->date_tz);
$carbonKey = $carbon->format('Y-m-d');
$carbon = new Carbon($entry->date, $entry->date_tz);
$carbonKey = $carbon->format('Y-m-d');
// make sure sum is a string:
$sumOfDay = (string) ($entry->sum_of_day ?? '0');
$sumOfDay = (string)($entry->sum_of_day ?? '0');
// #10426 make sure sum is not in scientific notation.
$sumOfDay = $this->floatalize($sumOfDay);
$sumOfDay = $this->floatalize($sumOfDay);
// find currency of this entry, does not have to exist.
$currencies[$entry->transaction_currency_id] ??= TransactionCurrency::find($entry->transaction_currency_id);
// make sure this $entry has its own $entryCurrency
/** @var TransactionCurrency $entryCurrency */
$entryCurrency = $currencies[$entry->transaction_currency_id];
$entryCurrency = $currencies[$entry->transaction_currency_id];
Log::debug(sprintf('Processing transaction(s) on moment %s', $carbon->format('Y-m-d H:i:s')));
// add amount to current balance in currency code.
$currentBalance[$entryCurrency->code] ??= '0';
$currentBalance[$entryCurrency->code] = bcadd($sumOfDay, (string) $currentBalance[$entryCurrency->code]);
$currentBalance[$entryCurrency->code] ??= '0';
$currentBalance[$entryCurrency->code] = bcadd($sumOfDay, (string)$currentBalance[$entryCurrency->code]);
// if not requested to convert to primary currency, add the amount to "balance", do nothing else.
if (!$convertToPrimary) {
$currentBalance['balance'] = bcadd((string) $currentBalance['balance'], $sumOfDay);
$currentBalance['balance'] = bcadd((string)$currentBalance['balance'], $sumOfDay);
}
// if convert to primary currency add the converted amount to "pc_balance".
// if there is a request to convert, convert to "pc_balance" and use "balance" for whichever amount is in the primary currency.
if ($convertToPrimary) {
$pcSumOfDay = $converter->convert($entryCurrency, $primaryCurrency, $carbon, $sumOfDay);
$currentBalance['pc_balance'] = bcadd((string) ($currentBalance['pc_balance'] ?? '0'), $pcSumOfDay);
$currentBalance['pc_balance'] = bcadd((string)($currentBalance['pc_balance'] ?? '0'), $pcSumOfDay);
// if it's the same currency as the entry, also add to balance (see other code).
if ($currency->id === $entryCurrency->id) {
$currentBalance['balance'] = bcadd((string) $currentBalance['balance'], $sumOfDay);
$currentBalance['balance'] = bcadd((string)$currentBalance['balance'], $sumOfDay);
}
}
// add to final array.
$balances[$carbonKey] = $currentBalance;
$balances[$carbonKey] = $currentBalance;
Log::debug(sprintf('Updated entry [%s]', $carbonKey), $currentBalance);
}
$cache->store($balances);
@@ -318,6 +317,70 @@ class Steam
return $balances;
}
public function finalAccountsBalanceOptimized(Collection $accounts, Carbon $date, ?TransactionCurrency $primary = null, ?bool $convertToPrimary = null): array
{
$result = [];
$convertToPrimary = $convertToPrimary ?? Amount::convertToPrimary();
$primary = $primary ?? Amount::getPrimaryCurrency();
$currencies = $this->getCurrencies($accounts);
// balance(s) in all currencies for ALL accounts.
$array = Transaction::whereIn('account_id', $accounts->pluck('id')->toArray())
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s'))
->get(['transactions.account_id', 'transaction_currencies.code', 'transactions.amount'])->toArray();
/** @var Account $account */
foreach ($accounts as $account) {
// filter array back to this account:
$filtered = array_filter($array, function ($item) use ($account) {
return (int)$item['account_id'] === $account->id;
});
$currency = $currencies[$account->id];
// this array is PER account, so we wait a bit before we change code here.
$return = [
'pc_balance' => '0',
'balance' => '0', // this key is overwritten right away, but I must remember it is always created.
];
// balance(s) in all currencies.
$others = $this->groupAndSumTransactions($filtered, 'code', 'amount');
// Log::debug('All balances are (joined)', $others);
// if there is no request to convert, take this as "balance" and "pc_balance".
$return['balance'] = $others[$currency->code] ?? '0';
if (!$convertToPrimary) {
unset($return['pc_balance']);
// Log::debug(sprintf('Set balance to %s, unset pc_balance', $return['balance']));
}
// if there is a request to convert, convert to "pc_balance" and use "balance" for whichever amount is in the primary currency.
if ($convertToPrimary) {
$return['pc_balance'] = $this->convertAllBalances($others, $primary, $date); // todo sum all and convert.
// Log::debug(sprintf('Set pc_balance to %s', $return['pc_balance']));
}
// either way, the balance is always combined with the virtual balance:
$virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance);
if ($convertToPrimary) {
// the primary currency balance is combined with a converted virtual_balance:
$converter = new ExchangeRateConverter();
$pcVirtualBalance = $converter->convert($currency, $primary, $date, $virtualBalance);
$return['pc_balance'] = bcadd($pcVirtualBalance, $return['pc_balance']);
// Log::debug(sprintf('Primary virtual balance makes the primary total %s', $return['pc_balance']));
}
if (!$convertToPrimary) {
// if not, also increase the balance + primary balance for consistency.
$return['balance'] = bcadd($return['balance'], $virtualBalance);
// Log::debug(sprintf('Virtual balance makes the (primary currency) total %s', $return['balance']));
}
$final = array_merge($return, $others);
$result[$account->id] = $final;
// Log::debug('Final balance is', $final);
}
return $result;
}
/**
* Returns smaller than or equal to, so be careful with END OF DAY.
*
@@ -334,13 +397,13 @@ class Steam
public function finalAccountBalance(Account $account, Carbon $date, ?TransactionCurrency $primary = null, ?bool $convertToPrimary = null): array
{
$cache = new CacheProperties();
$cache = new CacheProperties();
$cache->addProperty($account->id);
$cache->addProperty($date);
if ($cache->has()) {
Log::debug(sprintf('CACHED finalAccountBalance(#%d, %s)', $account->id, $date->format('Y-m-d H:i:s')));
return $cache->get();
//return $cache->get();
}
// Log::debug(sprintf('finalAccountBalance(#%d, %s)', $account->id, $date->format('Y-m-d H:i:s')));
if (null === $convertToPrimary) {
@@ -350,7 +413,7 @@ class Steam
$primary = Amount::getPrimaryCurrencyByUserGroup($account->user->userGroup);
}
// account balance thing.
$currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency'];
$currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency'];
if ($currencyPresent) {
$accountCurrency = $account->meta['currency'];
}
@@ -358,20 +421,19 @@ class Steam
$accountCurrency = $this->getAccountCurrency($account);
}
$hasCurrency = null !== $accountCurrency;
$currency = $hasCurrency ? $accountCurrency : $primary;
$return = [
'pc_balance' => '0',
'balance' => '0', // this key is overwritten right away, but I must remember it is always created.
$hasCurrency = null !== $accountCurrency;
$currency = $hasCurrency ? $accountCurrency : $primary;
$return = [
'pc_balance' => '0',
'balance' => '0', // this key is overwritten right away, but I must remember it is always created.
];
// balance(s) in all currencies.
$array = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s'))
->get(['transaction_currencies.code', 'transactions.amount'])->toArray()
;
$others = $this->groupAndSumTransactions($array, 'code', 'amount');
$array = $account->transactions()
->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'transactions.transaction_journal_id')
->leftJoin('transaction_currencies', 'transaction_currencies.id', '=', 'transactions.transaction_currency_id')
->where('transaction_journals.date', '<=', $date->format('Y-m-d H:i:s'))
->get(['transaction_currencies.code', 'transactions.amount'])->toArray();
$others = $this->groupAndSumTransactions($array, 'code', 'amount');
// Log::debug('All balances are (joined)', $others);
// if there is no request to convert, take this as "balance" and "pc_balance".
$return['balance'] = $others[$currency->code] ?? '0';
@@ -386,7 +448,7 @@ class Steam
}
// either way, the balance is always combined with the virtual balance:
$virtualBalance = (string) ('' === (string) $account->virtual_balance ? '0' : $account->virtual_balance);
$virtualBalance = (string)('' === (string)$account->virtual_balance ? '0' : $account->virtual_balance);
if ($convertToPrimary) {
// the primary currency balance is combined with a converted virtual_balance:
@@ -400,7 +462,7 @@ class Steam
$return['balance'] = bcadd($return['balance'], $virtualBalance);
// Log::debug(sprintf('Virtual balance makes the (primary currency) total %s', $return['balance']));
}
$final = array_merge($return, $others);
$final = array_merge($return, $others);
// Log::debug('Final balance is', $final);
$cache->store($final);
@@ -409,8 +471,8 @@ class Steam
public function getAccountCurrency(Account $account): ?TransactionCurrency
{
$type = $account->accountType->type;
$list = config('firefly.valid_currency_account_types');
$type = $account->accountType->type;
$list = config('firefly.valid_currency_account_types');
// return null if not in this list.
if (!in_array($type, $list, true)) {
@@ -421,7 +483,7 @@ class Steam
return null;
}
return TransactionCurrency::find((int) $result->data);
return TransactionCurrency::find((int)$result->data);
}
private function groupAndSumTransactions(array $array, string $group, string $field): array
@@ -430,7 +492,7 @@ class Steam
foreach ($array as $item) {
$groupKey = $item[$group] ?? 'unknown';
$return[$groupKey] = bcadd($return[$groupKey] ?? '0', (string) $item[$field]);
$return[$groupKey] = bcadd($return[$groupKey] ?? '0', (string)$item[$field]);
}
return $return;
@@ -445,9 +507,9 @@ class Steam
if (null === $currency) {
continue;
}
$current = $converter->convert($currency, $primary, $date, $amount);
$current = $converter->convert($currency, $primary, $date, $amount);
Log::debug(sprintf('Convert %s %s to %s %s', $currency->code, $amount, $primary->code, $current));
$total = bcadd($current, $total);
$total = bcadd($current, $total);
}
return $total;
@@ -478,28 +540,28 @@ class Steam
$hostName = $ipAddress;
}
if ('' !== (string) $hostName && $hostName !== $ipAddress) {
if ('' !== (string)$hostName && $hostName !== $ipAddress) {
$host = $hostName;
}
return (string) $host;
return (string)$host;
}
public function getLastActivities(array $accounts): array
{
$list = [];
$set = auth()->user()->transactions()
->whereIn('transactions.account_id', $accounts)
->groupBy(['transactions.account_id', 'transaction_journals.user_id'])
->get(['transactions.account_id', DB::raw('MAX(transaction_journals.date) AS max_date')]) // @phpstan-ignore-line
$set = auth()->user()->transactions()
->whereIn('transactions.account_id', $accounts)
->groupBy(['transactions.account_id', 'transaction_journals.user_id'])
->get(['transactions.account_id', DB::raw('MAX(transaction_journals.date) AS max_date')]) // @phpstan-ignore-line
;
/** @var Transaction $entry */
foreach ($set as $entry) {
$date = new Carbon($entry->max_date, config('app.timezone'));
$date = new Carbon($entry->max_date, config('app.timezone'));
$date->setTimezone(config('app.timezone'));
$list[(int) $entry->account_id] = $date;
$list[(int)$entry->account_id] = $date;
}
return $list;
@@ -517,7 +579,7 @@ class Steam
if ('equal' === $locale) {
$locale = $this->getLanguage();
}
$locale = (string) $locale;
$locale = (string)$locale;
// Check for Windows to replace the locale correctly.
if ('WIN' === strtoupper(substr(PHP_OS, 0, 3))) {
@@ -572,9 +634,9 @@ class Steam
public function getSafeUrl(string $unknownUrl, string $safeUrl): string
{
// Log::debug(sprintf('getSafeUrl(%s, %s)', $unknownUrl, $safeUrl));
$returnUrl = $safeUrl;
$unknownHost = parse_url($unknownUrl, PHP_URL_HOST);
$safeHost = parse_url($safeUrl, PHP_URL_HOST);
$returnUrl = $safeUrl;
$unknownHost = parse_url($unknownUrl, PHP_URL_HOST);
$safeHost = parse_url($safeUrl, PHP_URL_HOST);
if (null !== $unknownHost && $unknownHost === $safeHost) {
$returnUrl = $unknownUrl;
@@ -611,26 +673,26 @@ class Steam
*/
public function floatalize(string $value): string
{
$value = strtoupper($value);
$value = strtoupper($value);
if (!str_contains($value, 'E')) {
return $value;
}
Log::debug(sprintf('Floatalizing %s', $value));
$number = substr($value, 0, (int) strpos($value, 'E'));
$number = substr($value, 0, (int)strpos($value, 'E'));
if (str_contains($number, '.')) {
$post = strlen(substr($number, (int) strpos($number, '.') + 1));
$mantis = substr($value, (int) strpos($value, 'E') + 1);
$post = strlen(substr($number, (int)strpos($number, '.') + 1));
$mantis = substr($value, (int)strpos($value, 'E') + 1);
if ($mantis < 0) {
$post += abs((int) $mantis);
$post += abs((int)$mantis);
}
// TODO careless float could break financial math.
return number_format((float) $value, $post, '.', '');
return number_format((float)$value, $post, '.', '');
}
// TODO careless float could break financial math.
return number_format((float) $value, 0, '.', '');
return number_format((float)$value, 0, '.', '');
}
public function opposite(?string $amount = null): ?string
@@ -650,24 +712,24 @@ class Steam
// has a K in it, remove the K and multiply by 1024.
$bytes = bcmul(rtrim($string, 'k'), '1024');
return (int) $bytes;
return (int)$bytes;
}
if (false !== stripos($string, 'm')) {
// has a M in it, remove the M and multiply by 1048576.
$bytes = bcmul(rtrim($string, 'm'), '1048576');
return (int) $bytes;
return (int)$bytes;
}
if (false !== stripos($string, 'g')) {
// has a G in it, remove the G and multiply by (1024)^3.
$bytes = bcmul(rtrim($string, 'g'), '1073741824');
return (int) $bytes;
return (int)$bytes;
}
return (int) $string;
return (int)$string;
}
public function positive(string $amount): string
@@ -689,4 +751,42 @@ class Steam
return $amount;
}
private function getCurrencies(Collection $accounts): array
{
$currencies = [];
$accountCurrencies = [];
$accountPreferences = [];
$primary = Amount::getPrimaryCurrency();
$ids = $accounts->pluck('id')->toArray();
$result = AccountMeta::whereIn('account_id', $ids)->where('name', 'currency_id')->get();
/** @var AccountMeta $item */
foreach ($result as $item) {
$accountPreferences[(int)$item->account_id] = (int)$item->data;
}
// collect those currencies.
$set = TransactionCurrency::whereIn('id', $accountPreferences)->get();
foreach ($set as $item) {
$currencies[$item->id] = $item;
}
/** @var Account $account */
foreach ($accounts as $account) {
$accountId = $account->id;
$currencyPresent = isset($account->meta) && array_key_exists('currency', $account->meta) && null !== $account->meta['currency'];
if ($currencyPresent) {
$currencyId = $account->meta['currency']->id;
$currencies[$currencyId] ??= $account->meta['currency'];
$accountCurrencies[$accountId] = $account->meta['currency'];
}
if (!$currencyPresent && !array_key_exists($account->id, $accountPreferences)) {
$accountCurrencies[$accountId] = $primary;
}
if (!$currencyPresent && array_key_exists($account->id, $accountPreferences)) {
$accountCurrencies[$account->id] = $currencies[$accountPreferences[$account->id]];
}
}
return $accountCurrencies;
}
}