gnucash stable: Speed up during import when discarding transactions already imported
Christopher Lam
clam at code.gnucash.org
Mon Jul 24 21:15:50 EDT 2023
Updated via https://github.com/Gnucash/gnucash/commit/833ed5ae (commit)
from https://github.com/Gnucash/gnucash/commit/52b7bdc7 (commit)
commit 833ed5ae1fbc64de0881343a688f03ad6d3f29ef
Author: Christopher Lam <christopher.lck at gmail.com>
Date: Tue Jul 25 08:36:28 2023 +0800
Speed up during import when discarding transactions already imported
If a transaction online_id already exists, the code will destroy the
transaction. Defer all destruction until all transactions processed,
and efficiently destroy them within an xaccAccountBegin/CommitEdit block.
diff --git a/gnucash/import-export/aqb/gnc-ab-utils.c b/gnucash/import-export/aqb/gnc-ab-utils.c
index 0d7b6e0d8a..f88c134447 100644
--- a/gnucash/import-export/aqb/gnc-ab-utils.c
+++ b/gnucash/import-export/aqb/gnc-ab-utils.c
@@ -872,6 +872,7 @@ txn_accountinfo_cb (AB_IMEXPORTER_ACCOUNTINFO *element, gpointer user_data)
txn_transaction_cb, data,
AB_Transaction_TypeStatement, 0);
}
+ gnc_gen_trans_list_purge_existing (data->generic_importer);
return NULL;
}
diff --git a/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp b/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp
index 55823b93b1..04f9668576 100644
--- a/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp
+++ b/gnucash/import-export/csv-imp/assistant-csv-trans-import.cpp
@@ -2183,6 +2183,8 @@ CsvImpTransAssist::assist_match_page_prepare ()
draft_trans->trans = nullptr;
}
}
+
+ gnc_gen_trans_list_purge_existing(gnc_csv_importer_gui);
/* Show the matcher dialog */
gnc_gen_trans_list_show_all (gnc_csv_importer_gui);
}
diff --git a/gnucash/import-export/import-main-matcher.cpp b/gnucash/import-export/import-main-matcher.cpp
index 36d1cad898..e4d175eb7b 100644
--- a/gnucash/import-export/import-main-matcher.cpp
+++ b/gnucash/import-export/import-main-matcher.cpp
@@ -39,6 +39,8 @@
#include <stdbool.h>
#include <vector>
+#include <unordered_set>
+#include <algorithm>
#include "import-main-matcher.h"
@@ -78,6 +80,7 @@ struct _main_matcher_info
GtkWidget *show_matched_info;
GtkWidget *append_text; // Update+Clear: Append import Desc/Notes to matched Desc/Notes
GtkWidget *reconcile_after_close;
+ std::vector<Transaction*> transactions_to_delete; // transactions to delete immediately instead of importing
bool add_toggled; // flag to indicate that add has been toggled to stop selection
gint id;
GSList* temp_trans_list; // Temporary list of imported transactions
@@ -2229,8 +2232,7 @@ gnc_gen_trans_list_add_trans_internal (GNCImportMainMatcher *gui, Transaction *t
/* If it does, abort the process for this transaction, since
it is already in the system. */
DEBUG("%s", "Transaction with same online ID exists, destroying current transaction");
- xaccTransDestroy(trans);
- xaccTransCommitEdit(trans);
+ gui->transactions_to_delete.push_back (trans);
return;
}
@@ -2246,6 +2248,31 @@ gnc_gen_trans_list_add_trans_internal (GNCImportMainMatcher *gui, Transaction *t
gui->temp_trans_list = g_slist_prepend (gui->temp_trans_list, transaction_info);
}
+void
+gnc_gen_trans_list_purge_existing (GNCImportMainMatcher *gui)
+{
+ g_return_if_fail (gui);
+ std::unordered_set<Account*> accset;
+ std::for_each (gui->transactions_to_delete.begin(), gui->transactions_to_delete.end(),
+ [&accset](const Transaction* txn)
+ {
+ for (auto n = xaccTransGetSplitList (txn); n; n = n->next)
+ {
+ auto acc{xaccSplitGetAccount (static_cast<const Split*>(n->data))};
+ if (accset.insert(acc).second)
+ xaccAccountBeginEdit (acc);
+ }
+ });
+ std::for_each (gui->transactions_to_delete.begin(), gui->transactions_to_delete.end(),
+ [](Transaction* txn)
+ {
+ xaccTransBeginEdit (txn);
+ xaccTransDestroy (txn);
+ });
+ std::for_each (accset.begin(), accset.end(), xaccAccountCommitEdit);
+ gui->transactions_to_delete.clear();
+}
+
void
gnc_gen_trans_list_add_trans (GNCImportMainMatcher *gui, Transaction *trans)
{
diff --git a/gnucash/import-export/import-main-matcher.h b/gnucash/import-export/import-main-matcher.h
index cf0120333f..1d9425533b 100644
--- a/gnucash/import-export/import-main-matcher.h
+++ b/gnucash/import-export/import-main-matcher.h
@@ -200,6 +200,19 @@ void gnc_gen_trans_list_add_trans_with_ref_id (GNCImportMainMatcher *gui,
guint32 ref_id);
+/** Performs housekeeping for the previous related functions
+ * gnc_gen_trans_list_add_trans etc -- these functions will add (or
+ * mark for destroy) transactions for import. This function will
+ * actually efficiently destroy the transactions already imported.
+ *
+ * @param gui The Transaction Importer to use.
+ *
+ *
+ * @param ref_id Reference id which links an external object to the transaction.
+ */
+
+void gnc_gen_trans_list_purge_existing (GNCImportMainMatcher *gui);
+
/** Run this dialog and return only after the user pressed Ok, Cancel,
or closed the window. This means that all actual importing will
have been finished upon returning.
diff --git a/gnucash/import-export/ofx/gnc-ofx-import.cpp b/gnucash/import-export/ofx/gnc-ofx-import.cpp
index 21d1f6b558..d8c6a2ebb3 100644
--- a/gnucash/import-export/ofx/gnc-ofx-import.cpp
+++ b/gnucash/import-export/ofx/gnc-ofx-import.cpp
@@ -1339,6 +1339,7 @@ runMatcher (ofx_info* info, char * selected_filename, gboolean go_to_next_file)
info->num_trans_processed ++;
}
}
+ gnc_gen_trans_list_purge_existing (info->gnc_ofx_importer_gui);
g_list_free (info->trans_list);
g_hash_table_destroy (trans_hash);
info->trans_list = g_list_reverse (trans_list_remain);
Summary of changes:
gnucash/import-export/aqb/gnc-ab-utils.c | 1 +
.../csv-imp/assistant-csv-trans-import.cpp | 2 ++
gnucash/import-export/import-main-matcher.cpp | 31 ++++++++++++++++++++--
gnucash/import-export/import-main-matcher.h | 13 +++++++++
gnucash/import-export/ofx/gnc-ofx-import.cpp | 1 +
5 files changed, 46 insertions(+), 2 deletions(-)
More information about the gnucash-changes
mailing list