gnucash maint: Multiple changes pushed

John Ralls jralls at code.gnucash.org
Sat Dec 16 13:48:32 EST 2017


Updated	 via  https://github.com/Gnucash/gnucash/commit/51fab91f (commit)
	 via  https://github.com/Gnucash/gnucash/commit/4ce47b89 (commit)
	 via  https://github.com/Gnucash/gnucash/commit/87e833c6 (commit)
	from  https://github.com/Gnucash/gnucash/commit/0d8112bf (commit)



commit 51fab91ffef8aafea8a80de11f96a8697bc332cc
Author: John Ralls <jralls at ceridwen.us>
Date:   Sat Dec 16 10:36:27 2017 -0800

    Add test-flat-bayes to autotools build.

diff --git a/src/import-export/test/Makefile.am b/src/import-export/test/Makefile.am
index 1d32295..7b1eed1 100644
--- a/src/import-export/test/Makefile.am
+++ b/src/import-export/test/Makefile.am
@@ -6,6 +6,7 @@ AM_CPPFLAGS = \
   -I${top_srcdir}/src/test-core \
   -I${top_srcdir}/src/engine \
   -I${top_srcdir}/src/app-utils \
+  -I${top_srcdir}/src/core-utils \
   -I${top_srcdir}/src/import-export \
   -I${top_srcdir}/src/libqof/qof \
   -I${top_srcdir}/src/engine/test-core \
@@ -56,7 +57,9 @@ TESTS_ENVIRONMENT = \
 
 check_PROGRAMS = \
   test-link \
-  test-import-parse
+  test-import-parse \
+  test-import-pending-matches \
+  test-flat-bayes
 
 TEST_PROGS += test-import-pending-matches
 
@@ -74,6 +77,18 @@ test_import_pending_matches_LDADD = \
 
 test_import_pending_matches_CFLAGS = $(AM_CPPFLAGS)
 
+test_flat_bayes_SOURCES = test-flat-bayes.c
+
+test_flat_bayes_LDADD = \
+  ${top_builddir}/src/libqof/qof/libgnc-qof.la \
+  ${top_builddir}/src/engine/libgncmod-engine.la \
+  ../libgncmod-generic-import.la \
+  ${top_builddir}/src/engine/test-core/libgncmod-test-engine.la \
+  ${top_builddir}/src/test-core/libtest-core.la \
+  ${GLIB_LIBS}
+
+test_flat_bayes_CFLAGS = $(AM_CPPFLAGS)
+
 clean-local:
 	rm -f translog.*
 

commit 4ce47b898d45339b95756bb3157a0b7afb9e1830
Author: John Ralls <jralls at ceridwen.us>
Date:   Sat Dec 16 10:35:07 2017 -0800

    Add minimum version to feature and fix copy-paste error in test-flat-bayes.
    
    As recommended by Geert Janssens.

diff --git a/src/core-utils/gnc-features.c b/src/core-utils/gnc-features.c
index 2f1e37f..1cc495a 100644
--- a/src/core-utils/gnc-features.c
+++ b/src/core-utils/gnc-features.c
@@ -45,7 +45,7 @@ static gncFeature known_features[] =
     { GNC_FEATURE_NUM_FIELD_SOURCE, "User specifies source of 'num' field'; either transaction number or split action (requires at least GnuCash 2.5.0)" },
     { GNC_FEATURE_KVP_EXTRA_DATA, "Extra data for addresses, jobs or invoice entries (requires at least GnuCash 2.6.4)" },
     { GNC_FEATURE_GUID_BAYESIAN, "Use account GUID as key for Bayesian data (requires at least GnuCash 2.6.12)" },
-    { GNC_FEATURE_GUID_FLAT_BAYESIAN, "Use account GUID as key for bayesian data and store KVP flat" },
+    { GNC_FEATURE_GUID_FLAT_BAYESIAN, "Use account GUID as key for bayesian data and store KVP flat (requires at least GnuCash 2.6.19)" },
     { NULL },
 };
 
diff --git a/src/import-export/test/test-flat-bayes.c b/src/import-export/test/test-flat-bayes.c
index 7011685..22872ca 100644
--- a/src/import-export/test/test-flat-bayes.c
+++ b/src/import-export/test/test-flat-bayes.c
@@ -279,7 +279,7 @@ main (int argc, char *argv[])
     GNC_TEST_ADD (suitename, "guid_bayes_entry", Fixture, NULL, setup, &write_guid_bayes_entry, teardown);
     GNC_TEST_ADD (suitename, "normal_bayes_find", Fixture, NULL, setup, &find_normal_bayes_account, teardown);
     GNC_TEST_ADD (suitename, "guid_bayes_find", Fixture, NULL, setup, &find_guid_bayes_account, teardown);
-    GNC_TEST_ADD (suitename, "guid_bayes_find", Fixture, NULL, setup, &find_normal_bayes_when_non_exists, teardown);
+    GNC_TEST_ADD (suitename, "normal_bayes_find", Fixture, NULL, setup, &find_normal_bayes_when_non_exists, teardown);
     GNC_TEST_ADD (suitename, "Mixed case", Fixture, NULL, setup, &group_account_name_and_guid_in_mixed_case, teardown);
     GNC_TEST_ADD (suitename, "Read flat bayes", Fixture, NULL, setup, &read_flat_bayes, teardown);
     GNC_TEST_ADD (suitename, "Write flat bayes", Fixture, NULL, setup, &write_flat_bayes, teardown);

commit 87e833c693e0b0fa887d6f1fd8fd6622ec459b95
Author: lmat <dartme18 at gmail.com>
Date:   Wed Nov 22 14:02:45 2017 -0500

    GUID/Flat bayes handling in 2.6
    
    With 2.8 soon able to write flat guid bayes kvps, 2.6 will also need
    to be able to read them. This change enables 2.6 to be able to write
    all known types of import maps: not-flat guid and account name, and
    flat guid import maps, but it is not able to convert between these
    types. 2.8 is able to read earlier, not-flat bayes kvps,
    and will convert them to a flat guid model.

diff --git a/src/core-utils/gnc-features.c b/src/core-utils/gnc-features.c
index e1f5ae9..2f1e37f 100644
--- a/src/core-utils/gnc-features.c
+++ b/src/core-utils/gnc-features.c
@@ -45,6 +45,7 @@ static gncFeature known_features[] =
     { GNC_FEATURE_NUM_FIELD_SOURCE, "User specifies source of 'num' field'; either transaction number or split action (requires at least GnuCash 2.5.0)" },
     { GNC_FEATURE_KVP_EXTRA_DATA, "Extra data for addresses, jobs or invoice entries (requires at least GnuCash 2.6.4)" },
     { GNC_FEATURE_GUID_BAYESIAN, "Use account GUID as key for Bayesian data (requires at least GnuCash 2.6.12)" },
+    { GNC_FEATURE_GUID_FLAT_BAYESIAN, "Use account GUID as key for bayesian data and store KVP flat" },
     { NULL },
 };
 
@@ -160,3 +161,34 @@ void gnc_features_set_used (QofBook *book, const gchar *feature)
 
 
 }
+
+struct check_feature
+{
+    gchar const * checked_feature;
+    gboolean found;
+};
+
+static void
+gnc_feature_check_feature_cb (gchar const * key, KvpValue * value, gpointer data)
+{
+    struct check_feature * check_data = data;
+    if (!g_strcmp0 (key, check_data->checked_feature))
+        check_data->found = TRUE;
+}
+
+gboolean gnc_features_check_used (QofBook *book, const gchar * feature)
+{
+    KvpFrame *frame = qof_book_get_slots (book);
+    KvpValue *value;
+    struct check_feature check_data = {feature, FALSE};
+    /* Setup the known_features hash table */
+    gnc_features_init();
+    g_assert(frame);
+    value = kvp_frame_get_value(frame, "features");
+    if (!value) return FALSE;
+    frame = kvp_value_get_frame(value);
+    g_assert(frame);
+    /* Iterate over the members of this frame for unknown features */
+    kvp_frame_for_each_slot (frame, &gnc_feature_check_feature_cb, &check_data);
+    return check_data.found;
+}
diff --git a/src/core-utils/gnc-features.h b/src/core-utils/gnc-features.h
index ba691c7..a7ed421 100644
--- a/src/core-utils/gnc-features.h
+++ b/src/core-utils/gnc-features.h
@@ -45,6 +45,7 @@
 #define GNC_FEATURE_NUM_FIELD_SOURCE "Number Field Source"
 #define GNC_FEATURE_KVP_EXTRA_DATA "Extra data in addresses, jobs or invoice entries"
 #define GNC_FEATURE_GUID_BAYESIAN "Account GUID based Bayesian data"
+#define GNC_FEATURE_GUID_FLAT_BAYESIAN "Account GUID based bayesian with flat KVP"
 
 /** @} */
 
@@ -63,6 +64,8 @@ gchar *gnc_features_test_unknown (QofBook *book);
  */
 void gnc_features_set_used (QofBook *book, const gchar *feature);
 
+gboolean gnc_features_check_used (QofBook *book, const gchar * feature);
+
 #endif /* GNC_FEATURES_H */
 /** @} */
 /** @} */
diff --git a/src/import-export/import-match-map.c b/src/import-export/import-match-map.c
index 4a66e96..51a8e4d 100644
--- a/src/import-export/import-match-map.c
+++ b/src/import-export/import-match-map.c
@@ -33,6 +33,7 @@
 #include "import-match-map.h"
 #include "gnc-ui-util.h"
 #include "gnc-engine.h"
+#include "gnc-features.h"
 
 /********************************************************************\
  *   Constants   *
@@ -173,8 +174,8 @@ void gnc_imap_add_account (GncImportMatchMap *imap, const char *category,
 
 struct account_token_count
 {
-    char* account_name;
-    gint64 token_count; /**< occurances of a given token for this account_name */
+    char * account_guid;
+    gint64 token_count; /**< occurances of a given token for this account_guid */
 };
 
 /** total_count and the token_count for a given account let us calculate the
@@ -184,32 +185,60 @@ struct token_accounts_info
 {
     GList *accounts; /**< array of struct account_token_count */
     gint64 total_count;
+    QofBook * book;
 };
 
-/** gpointer is a pointer to a struct token_accounts_info
- * \note Can always assume that keys are unique, reduces code in this function
+static struct account_token_count * findAccountTokenCount (GList * haystack, struct account_token_count * needle)
+{
+    GList * iter = haystack;
+    for ( ; iter; iter = iter->next)
+    {
+        struct account_token_count * element = iter->data;
+        if (!g_strcmp0 (needle->account_guid, element->account_guid))
+            return element;
+    }
+    return NULL;
+}
+
+/**
+ * \note Cannot assume keys are unique. Account names and GUIDs can both
+ * come through here referring to the same account.
  */
 static void buildTokenInfo(const char *key, kvp_value *value, gpointer data)
 {
-    struct token_accounts_info *tokenInfo = (struct token_accounts_info*)data;
-    struct account_token_count* this_account;
-
-    //  PINFO("buildTokenInfo: account '%s', token_count: '%ld'\n", (char*)key,
-    //			(long)kvp_value_get_gint64(value));
-
-    /* add the count to the total_count */
+    struct token_accounts_info * tokenInfo = (struct token_accounts_info*)data;
+    struct account_token_count * otherTokenInfo = NULL;
+    struct account_token_count * this_account;
+    GncGUID temp_guid;
     tokenInfo->total_count += kvp_value_get_gint64(value);
-
-    /* allocate a new structure for this account and it's token count */
-    this_account = (struct account_token_count*)
-                   g_new0(struct account_token_count, 1);
-
-    /* fill in the account name and number of tokens found for this account name */
-    this_account->account_name = (char*)key;
+    this_account = (struct account_token_count*) g_new0(struct account_token_count, 1);
+    if (string_to_guid (key, &temp_guid))
+    { /*the key is a guid*/
+        this_account->account_guid = g_strdup (key);
+    }
+    else
+    {
+        Account * account = gnc_account_lookup_by_full_name(
+                gnc_book_get_root_account(tokenInfo->book), key);
+        gchar tempbuff [GUID_ENCODING_LENGTH + 1];
+        if (!account)
+            /* dud record. */
+            return;
+        guid_to_string_buff (xaccAccountGetGUID (account), tempbuff);
+        this_account->account_guid = g_strdup (tempbuff);
+    }
     this_account->token_count = kvp_value_get_gint64(value);
-
-    /* append onto the glist a pointer to the new account_token_count structure */
-    tokenInfo->accounts = g_list_prepend(tokenInfo->accounts, this_account);
+    if ((otherTokenInfo = findAccountTokenCount (tokenInfo->accounts, this_account)))
+    {
+        /* This is a duplicate. Aggregate it. */
+        otherTokenInfo->token_count += this_account->token_count;
+        g_free (this_account->account_guid);
+        g_free (this_account);
+    }
+    else
+    {
+        tokenInfo->accounts = g_list_prepend(tokenInfo->accounts, this_account);
+    }
 }
 
 /** intermediate values used to calculate the bayes probability of a given account
@@ -249,6 +278,7 @@ static void buildProbabilities(gpointer key, gpointer value, gpointer data)
 /** Frees an array of the same time that buildProperties built */
 static void freeProbabilities(gpointer key, gpointer value, gpointer data)
 {
+    g_free(key);
     /* free up the struct account_probability that was allocated
      * in gnc_imap_find_account_bayes()
      */
@@ -260,7 +290,7 @@ static void freeProbabilities(gpointer key, gpointer value, gpointer data)
  */
 struct account_info
 {
-    char* account_name;
+    char* account_guid;
     gint32 probability;
 };
 
@@ -279,18 +309,97 @@ static void highestProbability(gpointer key, gpointer value, gpointer data)
     {
         /* Save the new highest probability and the assoaciated account name */
         account_i->probability = GPOINTER_TO_INT(value);
-        account_i->account_name = key;
+        g_free (account_i->account_guid);
+        account_i->account_guid = g_strdup (key);
+    }
+}
+
+static struct token_accounts_info *
+get_flat_account_tokens (char const * token, KvpFrame * imap_frame, QofBook * book)
+{
+    gchar * path_prefix = g_strdup_printf ("%s/%s", IMAP_FRAME_BAYES, token);
+    GList * matching_keys = kvp_frame_get_keys_matching_prefix (imap_frame, path_prefix);
+    GList const * iter = matching_keys;
+    struct token_accounts_info * ret = (struct token_accounts_info *)
+        g_new0 (struct token_accounts_info, 1);
+    for ( ; iter; iter = iter->next)
+    {
+        /* This needs to be const because the string actually
+         * belongs to the string cache in KVP. */
+        gchar const * temp_key = iter->data;
+        struct account_token_count * temp_token_count = (struct account_token_count *)
+            g_new0 (struct account_token_count, 1);
+        /*The guid part of the key is just after the last '/'.*/
+        gchar const * guid_str = g_strrstr (temp_key, "/") + 1;
+        gchar * temp_path = g_strdup_printf ("%s/%s", path_prefix, guid_str);
+        KvpValue const * temp_count_value = kvp_frame_get_slot_path (imap_frame, temp_path, NULL);
+        GncGUID guid;
+        Account const * temp_account;
+        string_to_guid(guid_str, &guid);
+        temp_account = xaccAccountLookup (&guid, book);
+        temp_token_count->account_guid = g_strdup (guid_str);
+        temp_token_count->token_count = kvp_value_get_gint64 (temp_count_value);
+        ret->total_count += temp_token_count->token_count;
+        ret->accounts = g_list_prepend (ret->accounts, temp_token_count);
+        g_free (temp_path);
     }
+    g_list_free (matching_keys);
+    g_free (path_prefix);
+    return ret;
 }
 
+static struct token_accounts_info *
+get_not_flat_account_tokens (char const * token, KvpFrame * imap_frame, QofBook * book)
+{
+    KvpValue * value;
+    KvpFrame * token_frame;
+    struct token_accounts_info * tokenInfo = (struct token_accounts_info *)
+        g_new0 (struct token_accounts_info, 1); /**< holds the accounts and total
+             * token count for a single token */
+    tokenInfo->book = book;
+    /* zero out the token_accounts_info structure */
+    PINFO("token: '%s'", token);
+    /* find the slot for the given token off of the source account
+     * for these tokens, search off of the IMAP_FRAME_BAYES path so
+     * we aren't looking from the parent of the entire kvp tree
+     */
+    value = kvp_frame_get_slot_path(imap_frame, IMAP_FRAME_BAYES, token, NULL);
+    /* if value is null we should skip over this token */
+    if (!value)
+        return NULL;
+    /* convert the slot(value) into a the frame that contains the
+     * list of accounts
+     */
+    token_frame = kvp_value_get_frame(value);
+    /* token_frame should NEVER be null */
+    if (!token_frame)
+    {
+        PERR("token '%s' has no accounts", token);
+        return NULL;
+    }
+    /* process the accounts for this token, adding the account if it
+     * doesn't already exist or adding to the existing accounts token
+     * count if it does
+     */
+    kvp_frame_for_each_slot(token_frame, buildTokenInfo, tokenInfo);
+    return tokenInfo;
+}
+
+static struct token_accounts_info *
+get_account_tokens (char const * token, KvpFrame * imap_frame, QofBook * book,
+        gboolean flat_bayes)
+{
+    struct token_accounts_info * ret;
+    if (flat_bayes)
+        return get_flat_account_tokens (token, imap_frame, book);
+    return get_not_flat_account_tokens (token, imap_frame, book);
+}
 
 #define threshold (.90 * PROBABILITY_FACTOR) /* 90% */
 
 /** Look up an Account in the map */
 Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
 {
-    struct token_accounts_info tokenInfo; /**< holds the accounts and total
-					 * token count for a single token */
     GList *current_token;		        /**< pointer to the current token from the
 				         * input GList *tokens */
     GList *current_account_token;		/**< pointer to the struct
@@ -306,6 +415,8 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
     struct account_info account_i;
     kvp_value* value;
     kvp_frame* token_frame;
+    /* if guid flat bayes, all records should be in flat guid storage*/
+    gboolean flat_bayes = gnc_features_check_used (imap->book, GNC_FEATURE_GUID_FLAT_BAYESIAN);
 
     ENTER(" ");
 
@@ -322,56 +433,27 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
      */
     for (current_token = tokens; current_token; current_token = current_token->next)
     {
-        /* zero out the token_accounts_info structure */
-        memset(&tokenInfo, 0, sizeof(struct token_accounts_info));
-
-        PINFO("token: '%s'", (char*)current_token->data);
-
-        /* find the slot for the given token off of the source account
-         * for these tokens, search off of the IMAP_FRAME_BAYES path so
-         * we aren't looking from the parent of the entire kvp tree
-         */
-        value = kvp_frame_get_slot_path(imap->frame, IMAP_FRAME_BAYES,
-                                        (char*)current_token->data, NULL);
+        struct token_accounts_info * account_tokens = get_account_tokens ((char const *) current_token->data,
+                imap->frame, imap->book, flat_bayes);
 
-        /* if value is null we should skip over this token */
-        if (!value)
+        if (!account_tokens)
             continue;
-
-        /* convert the slot(value) into a the frame that contains the
-         * list of accounts
-         */
-        token_frame = kvp_value_get_frame(value);
-
-        /* token_frame should NEVER be null */
-        if (!token_frame)
-        {
-            PERR("token '%s' has no accounts", (char*)current_token->data);
-            continue; /* skip over this token */
-        }
-
-        /* process the accounts for this token, adding the account if it
-         * doesn't already exist or adding to the existing accounts token
-         * count if it does
-         */
-        kvp_frame_for_each_slot(token_frame, buildTokenInfo, &tokenInfo);
-
         /* for each account we have just found, see if the account already exists
          * in the list of account probabilities, if not add it
          */
-        for (current_account_token = tokenInfo.accounts; current_account_token;
+        for (current_account_token = account_tokens->accounts; current_account_token;
                 current_account_token = current_account_token->next)
         {
             /* get the account name and corresponding token count */
             account_c = (struct account_token_count*)current_account_token->data;
 
-            PINFO("account_c->account_name('%s'), "
+            PINFO("account_c->account_guid('%s'), "
                   "account_c->token_count('%ld')/total_count('%ld')",
-                  account_c->account_name, (long)account_c->token_count,
-                  (long)tokenInfo.total_count);
+                  account_c->account_guid, (long)account_c->token_count,
+                  (long)account_tokens->total_count);
 
             account_p = g_hash_table_lookup(running_probabilities,
-                                            account_c->account_name);
+                                            account_c->account_guid);
 
             /* if the account exists in the list then continue
              * the running probablities
@@ -379,14 +461,15 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
             if (account_p)
             {
                 account_p->product =
-                    ((double)account_c->token_count / (double)tokenInfo.total_count)
+                    ((double)account_c->token_count / (double)account_tokens->total_count)
                     * account_p->product;
                 account_p->product_difference =
                     ((double)1 - ((double)account_c->token_count /
-                                  (double)tokenInfo.total_count))
+                                  (double)account_tokens->total_count))
                     * account_p->product_difference;
                 PINFO("product == %f, product_difference == %f",
                       account_p->product, account_p->product_difference);
+                g_free (account_c->account_guid);
             }
             else
             {
@@ -397,10 +480,10 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
 
                 /* set the product and product difference values */
                 account_p->product = ((double)account_c->token_count /
-                                      (double)tokenInfo.total_count);
+                                      (double)account_tokens->total_count);
                 account_p->product_difference =
                     (double)1 - ((double)account_c->token_count /
-                                 (double)tokenInfo.total_count);
+                                 (double)account_tokens->total_count);
 
                 PINFO("product == %f, product_difference == %f",
                       account_p->product, account_p->product_difference);
@@ -408,19 +491,19 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
                 /* add the account name and (struct account_probability*)
                  * to the hash table */
                 g_hash_table_insert(running_probabilities,
-                                    account_c->account_name, account_p);
+                                    (char *)account_c->account_guid, account_p);
             }
         } /* for all accounts in tokenInfo */
 
         /* free the data in tokenInfo */
-        for (current_account_token = tokenInfo.accounts; current_account_token;
+        for (current_account_token = account_tokens->accounts; current_account_token;
                 current_account_token = current_account_token->next)
         {
-            /* free up each struct account_token_count we allocated */
             g_free((struct account_token_count*)current_account_token->data);
         }
 
-        g_list_free(tokenInfo.accounts); /* free the accounts GList */
+        g_list_free(account_tokens->accounts);
+        g_free(account_tokens);
     }
 
     /* build a hash table of account names and their final probabilities
@@ -441,7 +524,7 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
     g_hash_table_destroy(final_probabilities);
 
     PINFO("highest P('%s') = '%d'",
-          account_i.account_name ? account_i.account_name : "(null)",
+          account_i.account_guid ? account_i.account_guid : "(null)",
           account_i.probability);
 
     /* has this probability met our threshold? */
@@ -451,13 +534,13 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
         PINFO("Probability has met threshold");
 
         account = gnc_account_lookup_by_full_name(gnc_book_get_root_account(imap->book),
-                                               account_i.account_name);
+                                               account_i.account_guid);
 
         if (account == NULL) // Possibly we have a Guid or account not found
         {
             GncGUID *guid = g_new (GncGUID, 1);
 
-            if (string_to_guid (account_i.account_name, guid))
+            if (string_to_guid (account_i.account_guid, guid))
                 account = xaccAccountLookup (guid, imap->book);
 
             g_free (guid);
@@ -466,7 +549,7 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
         if (account != NULL)
             LEAVE("Return account is '%s'", xaccAccountGetName (account));
         else
-            LEAVE("Return NULL, account for string '%s' can not be found", account_i.account_name);
+            LEAVE("Return NULL, account for string '%s' can not be found", account_i.account_guid);
 
         return account;
     }
@@ -476,7 +559,6 @@ Account* gnc_imap_find_account_bayes(GncImportMatchMap *imap, GList *tokens)
     return NULL; /* we didn't meet our threshold, return NULL for an account */
 }
 
-
 /** Updates the imap for a given account using a list of tokens */
 void gnc_imap_add_account_bayes(GncImportMatchMap *imap, GList *tokens, Account *acc)
 {
@@ -486,7 +568,9 @@ void gnc_imap_add_account_bayes(GncImportMatchMap *imap, GList *tokens, Account
     char* account_fullname;
     kvp_value *new_value; /* the value that will be added back into the kvp tree */
     const gchar *guid_string;
-    gboolean use_fullname = TRUE;
+    gchar *guid_path;
+    gboolean guid_bayes = gnc_features_check_used (imap->book, GNC_FEATURE_GUID_BAYESIAN);
+    gboolean flat_bayes = gnc_features_check_used (imap->book, GNC_FEATURE_GUID_FLAT_BAYESIAN);
 
     ENTER(" ");
 
@@ -509,6 +593,8 @@ void gnc_imap_add_account_bayes(GncImportMatchMap *imap, GList *tokens, Account
     for (current_token = g_list_first(tokens); current_token;
             current_token = current_token->next)
     {
+        gchar const * name_or_guid;
+        gchar * flat_path = NULL;
         /* Jump to next iteration if the pointer is not valid or if the
         	 string is empty. In HBCI import we almost always get an empty
         	 string, which doesn't work in the kvp loopkup later. So we
@@ -521,57 +607,40 @@ void gnc_imap_add_account_bayes(GncImportMatchMap *imap, GList *tokens, Account
 
         PINFO("adding token '%s'\n", (char*)current_token->data);
 
-        /* is this token/account_name already in the kvp tree under fullname? */
-        value = kvp_frame_get_slot_path(imap->frame, IMAP_FRAME_BAYES,
-                                        (char*)current_token->data, account_fullname,
-                                        NULL);
-
-        if (!value) // we have not found entry under the fullname, maybe guid
+        if (flat_bayes)
         {
+            flat_path = g_strdup_printf ("%s/%s/%s", IMAP_FRAME_BAYES,
+                    (char*)current_token->data, guid_string);
+            value = kvp_frame_get_slot(imap->frame, flat_path);
+        }
+        else
+        {
+            if (guid_bayes)
+                name_or_guid = guid_string;
+            else
+                name_or_guid = account_fullname;
             value = kvp_frame_get_slot_path(imap->frame, IMAP_FRAME_BAYES,
-                                            (char*)current_token->data, guid_string,
+                                            (char*)current_token->data, name_or_guid,
                                             NULL);
-            if (value)
-                use_fullname = FALSE;
         }
-
-        /* if the token/account is already in the tree, read the current
-         * value from the tree and use this for the basis of the value we
-         * are putting back
-         */
         if (value)
         {
             PINFO("found existing value of '%ld'\n",
                   (long)kvp_value_get_gint64(value));
-
-            /* convert this value back into an integer */
             token_count += kvp_value_get_gint64(value);
         }
-
-        /* increment the token count */
         token_count++;
-
-        /* create a new value */
         new_value = kvp_value_new_gint64(token_count);
-
-        /* insert the value into the kvp tree at
-         * /imap->frame/IMAP_FRAME/token_string/account_fullname or guid
-         */
-        if (use_fullname == TRUE)
+        if (flat_bayes)
         {
-            KvpFrame  *book_frame = qof_book_get_slots (imap->book);
-            const gchar *book_path = "changed-bayesian-to-guid";
-
-            kvp_frame_set_slot_path(imap->frame, new_value, IMAP_FRAME_BAYES,
-                                    (char*)current_token->data, account_fullname, NULL);
-
-            /* Reset the run once kvp flag for versions 2.7.0 and later */
-            if (kvp_frame_get_string(book_frame, book_path) != NULL)
-                kvp_frame_set_string(book_frame, book_path, "false");
+            kvp_frame_set_slot(imap->frame, flat_path, new_value);
+            g_free (flat_path);
         }
         else
+        {
             kvp_frame_set_slot_path(imap->frame, new_value, IMAP_FRAME_BAYES,
-                                    (char*)current_token->data, guid_string, NULL);
+                                        (char*)current_token->data, name_or_guid, NULL);
+        }
         /* kvp_frame_set_slot_path() copied the value so we
          * need to delete this one ;-) */
         kvp_value_delete(new_value);
diff --git a/src/import-export/test/CMakeLists.txt b/src/import-export/test/CMakeLists.txt
index a81cdf3..46a4e07 100644
--- a/src/import-export/test/CMakeLists.txt
+++ b/src/import-export/test/CMakeLists.txt
@@ -19,8 +19,12 @@ GNC_ADD_TEST_WITH_GUILE(test-import-parse test-import-parse.c
 GNC_ADD_TEST(test-link-generic-import test-link.c
   GENERIC_IMPORT_TEST_INCLUDE_DIRS GENERIC_IMPORT_TEST_LIBS
 )
+GNC_ADD_TEST(test-flat-bayes test-flat-bayes.c
+    GENERIC_IMPORT_TEST_INCLUDE_DIRS GENERIC_IMPORT_TEST_LIBS
+)
 GNC_ADD_TEST(test-import-pending-matches test-import-pending-matches.c
   GENERIC_IMPORT_TEST_INCLUDE_DIRS GENERIC_IMPORT_TEST_LIBS
 )
 SET_DIST_LIST(test_generic_import_DIST CMakeLists.txt Makefile.am
-        test-link.c test-import-parse.c test-import-pending-matches.c)
+        test-link.c test-import-parse.c test-import-pending-matches.c
+        test-flat-bayes.c)
diff --git a/src/import-export/test/test-flat-bayes.c b/src/import-export/test/test-flat-bayes.c
new file mode 100644
index 0000000..7011685
--- /dev/null
+++ b/src/import-export/test/test-flat-bayes.c
@@ -0,0 +1,289 @@
+/*
+ * Created by:	Aaron Laws
+ * Copyright (c) 2017 Aaron Laws
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, contact:
+ *
+ * Free Software Foundation           Voice:  +1-617-542-5942
+ * 51 Franklin Street, Fifth Floor    Fax:    +1-617-542-2652
+ * Boston, MA  02110-1301,  USA       gnu at gnu.org
+ */
+
+#include "config.h"
+#include <glib.h>
+#include <libguile.h>
+
+#include "gnc-features.h"
+#include "gnc-module.h"
+#include "import-parse.h"
+
+#include "import-match-map.h"
+#include "test-engine-stuff.h"
+#include "test-stuff.h"
+#include <unittest-support.h>
+#include "kvp_frame.h"
+
+typedef struct
+{
+    QofBook * book;
+    Account * account1;
+    Account * account2;
+    Account * account3;
+    GncImportMatchMap * map1;
+    GList * tokens1;
+    gchar * account1_guid;
+    gchar * account2_guid;
+} Fixture;
+
+static gchar const * token1 = "one/one";
+static gchar const * token2 = "two";
+static gchar const * account1_name = "Asset";
+static gchar const * account2_name = "Liability";
+static gchar const * account3_name = "Equity";
+
+static void
+setup (Fixture *fixture, gconstpointer pData)
+{
+    Account * root;
+    fixture->book = qof_book_new ();
+    root = gnc_account_create_root (fixture->book);
+    fixture->account1 = xaccMallocAccount (fixture->book);
+    gnc_account_append_child (root, fixture->account1);
+    xaccAccountSetName (fixture->account1, account1_name);
+    fixture->account2 = xaccMallocAccount (fixture->book);
+    gnc_account_append_child (root, fixture->account2);
+    xaccAccountSetName (fixture->account2, account2_name);
+    fixture->account3 = xaccMallocAccount (fixture->book);
+    gnc_account_append_child (root, fixture->account3);
+    xaccAccountSetName (fixture->account3, account3_name);
+    fixture->map1 = gnc_imap_create_from_account (fixture->account1);
+    fixture->tokens1 = g_list_append (g_list_append (NULL, g_strdup (token1)), g_strdup (token2));
+    fixture->account1_guid = g_strdup (guid_to_string (xaccAccountGetGUID (fixture->account1)));
+    fixture->account2_guid = g_strdup (guid_to_string (xaccAccountGetGUID (fixture->account2)));
+}
+
+static void
+teardown (Fixture * fixture, gconstpointer pData)
+{
+    g_free (fixture->account1_guid);
+    g_free (fixture->account2_guid);
+    xaccAccountBeginEdit (gnc_book_get_root_account (fixture->book));
+    xaccAccountDestroy (gnc_book_get_root_account (fixture->book));
+    qof_book_destroy (fixture->book);
+    gnc_imap_destroy (fixture->map1);
+}
+
+/*
+ * Make sure that added imap entries are put in the right place and incremented properly.
+ */
+static void
+write_normal_bayes_entry (Fixture * fixture, gconstpointer pData)
+{
+    /* By default, this version of gnucash should write divided KVP bayes entries without
+     * dividing the token portion. */
+    KvpFrame * frame;
+    KvpValue * count_value;
+    gint64 count;
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_frame_get_slot_path (frame, "import-map-bayes", token1, account1_name, NULL);
+    count = kvp_value_get_gint64 (count_value);
+    g_assert_cmpint (1, ==, count);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_frame_get_slot_path (frame, "import-map-bayes", token1, account1_name, NULL);
+    count = kvp_value_get_gint64 (count_value);
+    g_assert_cmpint (2, ==, count);
+}
+
+/*
+ * Test adding entries to the import map when FEATURE_GUID_BAYESIAN is enabled.
+ */
+static void
+write_guid_bayes_entry (Fixture * fixture, gconstpointer pData)
+{
+    /* When the guid feature is present, gnucash should write and retrieve bayes
+     * information by account GUID, but still hierarchically.*/
+    KvpFrame * frame;
+    KvpValue * count_value;
+    gint64 count;
+    kvp_frame_set_string (qof_instance_get_slots (QOF_INSTANCE (fixture->book)),
+            "features/" GNC_FEATURE_GUID_BAYESIAN, "value");
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    /*We'll look up the added import map information.*/
+    count_value = kvp_frame_get_slot_path (frame, "import-map-bayes", token1, fixture->account1_guid, NULL);
+    count = kvp_value_get_gint64 (count_value);
+    g_assert_cmpint (1, ==, count);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_frame_get_slot_path (frame, "import-map-bayes", token1, fixture->account1_guid, NULL);
+    count = kvp_value_get_gint64 (count_value);
+    g_assert_cmpint (2, ==, count);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_frame_get_slot_path (frame, "import-map-bayes", token1, fixture->account1_guid, NULL);
+    count = kvp_value_get_gint64 (count_value);
+    g_assert_cmpint (3, ==, count);
+}
+
+/*
+ * File created in 2.7.1 (or 2.7.2) with GUID_BAYESIAN, then edited
+ * in 2.6.17. The former will create the import map with GUIDs, and the latter
+ * may add account-name entries. This version of gnucash should be able
+ * to cope with this.
+ */
+static void
+group_account_name_and_guid_in_mixed_case (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    KvpValue * count_value;
+    Account * chosen;
+    gint64 count;
+    kvp_frame_set_string (qof_instance_get_slots (QOF_INSTANCE (fixture->book)),
+            "features/" GNC_FEATURE_GUID_BAYESIAN, "value");
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_value_new_gint64 (9);
+    /* Account 1 should have 18 "points" between the name and guid */
+    kvp_frame_set_slot_path (frame, count_value, "import-map-bayes", token1, account1_name, NULL);
+    kvp_frame_set_slot_path (frame, count_value, "import-map-bayes", token1, fixture->account1_guid, NULL);
+    kvp_value_delete (count_value);
+    count_value = kvp_value_new_gint64 (1);
+    /* Account 2 and Account 3 should have 1 point each. */
+    kvp_frame_set_slot_path (frame, count_value, "import-map-bayes", token1, fixture->account2_guid, NULL);
+    kvp_frame_set_slot_path (frame, count_value, "import-map-bayes", token1, account3_name, NULL);
+    kvp_value_delete (count_value);
+    /* One might think that it's sufficient to give 4 total points (2 each) to account 1 name and guid
+     * and 3 points to account 2 and account 3. This way, acconut 1 would win.
+     * One would be mistaken in this case as it is not a simple majority win. The assurance of
+     * victory must be rather great: 90%.
+     * If the grouping by account name and guid is *not* working properly, this test will fail.
+     */
+    chosen = gnc_imap_find_account_bayes (fixture->map1, fixture->tokens1);
+    g_assert_true (chosen == fixture->account1);
+}
+
+/*
+ * File created in 2.7.3 with GUID_FLAT_BAYESIAN should be readable.
+ */
+static void
+read_flat_bayes (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    KvpValue * count_value;
+    Account * chosen;
+    gchar * path = g_strdup_printf ("%s/%s/%s", "import-map-bayes", token1, fixture->account1_guid);
+    kvp_frame_set_string (qof_instance_get_slots (QOF_INSTANCE (fixture->book)),
+            "features/" GNC_FEATURE_GUID_FLAT_BAYESIAN, "value");
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_value_new_gint64 (1);
+    kvp_frame_set_slot (frame, path, count_value);
+    kvp_value_delete (count_value);
+    chosen = gnc_imap_find_account_bayes (fixture->map1, fixture->tokens1);
+    g_assert_true (chosen == fixture->account1);
+}
+
+static void
+write_flat_bayes (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    KvpValue * count_value;
+    gint64 count;
+    gchar * path = g_strdup_printf ("%s/%s/%s", "import-map-bayes", token1, fixture->account1_guid);
+    kvp_frame_set_string (qof_instance_get_slots (QOF_INSTANCE (fixture->book)),
+            "features/" GNC_FEATURE_GUID_FLAT_BAYESIAN, "value");
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    frame = qof_instance_get_slots (QOF_INSTANCE (fixture->account1));
+    count_value = kvp_frame_get_slot (frame, path);
+    count = kvp_value_get_gint64 (count_value);
+    g_assert_true (count == 2);
+}
+
+/*
+ * We should always be able to retrieve an account from the import map after
+ * entries for that account are added.
+ */
+static void
+find_normal_bayes_account (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    Account * acc;
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    acc = gnc_imap_find_account_bayes (fixture->map1, fixture->tokens1);
+    g_assert_true (acc == fixture->account1);
+}
+
+static void
+find_flat_bayes_account (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    Account * acc;
+    kvp_frame_set_string (qof_instance_get_slots (QOF_INSTANCE (fixture->book)),
+            "features/" GNC_FEATURE_GUID_FLAT_BAYESIAN, "value");
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account2);
+    acc = gnc_imap_find_account_bayes (fixture->map1, fixture->tokens1);
+    g_assert_true (acc == fixture->account1);
+}
+
+static void
+find_guid_bayes_account (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    Account * acc;
+    kvp_frame_set_string (qof_instance_get_slots (QOF_INSTANCE (fixture->book)),
+            "features/" GNC_FEATURE_GUID_BAYESIAN, "value");
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account1);
+    gnc_imap_add_account_bayes (fixture->map1, fixture->tokens1, fixture->account2);
+    acc = gnc_imap_find_account_bayes (fixture->map1, fixture->tokens1);
+    g_assert_true (acc == fixture->account1);
+}
+
+static void
+find_normal_bayes_when_non_exists (Fixture * fixture, gconstpointer pData)
+{
+    KvpFrame * frame;
+    Account * acc;
+    acc = gnc_imap_find_account_bayes (fixture->map1, fixture->tokens1);
+    g_assert_true (acc == NULL);
+}
+
+static gchar const * suitename = "/import-export/test-flat-bayes";
+
+int
+main (int argc, char *argv[])
+{
+    int result;
+    qof_init();
+    g_test_init (&argc, &argv, NULL);
+    GNC_TEST_ADD (suitename, "normal_bayes_entry", Fixture, NULL, setup, &write_normal_bayes_entry, teardown);
+    GNC_TEST_ADD (suitename, "guid_bayes_entry", Fixture, NULL, setup, &write_guid_bayes_entry, teardown);
+    GNC_TEST_ADD (suitename, "normal_bayes_find", Fixture, NULL, setup, &find_normal_bayes_account, teardown);
+    GNC_TEST_ADD (suitename, "guid_bayes_find", Fixture, NULL, setup, &find_guid_bayes_account, teardown);
+    GNC_TEST_ADD (suitename, "guid_bayes_find", Fixture, NULL, setup, &find_normal_bayes_when_non_exists, teardown);
+    GNC_TEST_ADD (suitename, "Mixed case", Fixture, NULL, setup, &group_account_name_and_guid_in_mixed_case, teardown);
+    GNC_TEST_ADD (suitename, "Read flat bayes", Fixture, NULL, setup, &read_flat_bayes, teardown);
+    GNC_TEST_ADD (suitename, "Write flat bayes", Fixture, NULL, setup, &write_flat_bayes, teardown);
+    result =  g_test_run();
+    qof_close();
+}
+
diff --git a/src/libqof/qof/kvp_frame.c b/src/libqof/qof/kvp_frame.c
index 7597ea0..e1931c7 100644
--- a/src/libqof/qof/kvp_frame.c
+++ b/src/libqof/qof/kvp_frame.c
@@ -1638,6 +1638,30 @@ kvp_frame_to_string(const KvpFrame *frame)
     return tmp1;
 }
 
+struct key_frame_keys_helper
+{
+    GList * keys;
+    gchar const * prefix;
+};
+
+static void
+kvp_frame_keys_cb (gpointer keyp, gpointer value, gpointer data)
+{
+    struct key_frame_keys_helper * helper = data;
+    gchar const * key = keyp;
+    gboolean prefixed = g_str_has_prefix (keyp, helper->prefix);
+    if (prefixed)
+        helper->keys = g_list_append (helper->keys, g_strdup (keyp));
+}
+
+GList *
+kvp_frame_get_keys_matching_prefix (KvpFrame * frame, const gchar * prefix)
+{
+    struct key_frame_keys_helper helper = {NULL, prefix};
+    g_hash_table_foreach (frame->hash, &kvp_frame_keys_cb, &helper);
+    return helper.keys;
+}
+
 GHashTable*
 kvp_frame_get_hash(const KvpFrame *frame)
 {
diff --git a/src/libqof/qof/kvp_frame.h b/src/libqof/qof/kvp_frame.h
index 3b9aae6..36e7674 100644
--- a/src/libqof/qof/kvp_frame.h
+++ b/src/libqof/qof/kvp_frame.h
@@ -191,6 +191,8 @@ void kvp_frame_set_guid(KvpFrame * frame, const gchar * path, const GncGUID *gui
 void kvp_frame_set_frame(KvpFrame *frame, const gchar *path, KvpFrame *chld);
 void kvp_frame_set_frame_nc(KvpFrame *frame, const gchar *path, KvpFrame *chld);
 
+GList * kvp_frame_get_keys_matching_prefix (KvpFrame *, const gchar *);
+
 /** The kvp_frame_set_value() routine copies the value into the frame,
  *    at the location 'path'.   If the path contains slashes '/', these
  *    are assumed to represent a sequence of keys.  The returned value



Summary of changes:
 src/core-utils/gnc-features.c            |  32 ++++
 src/core-utils/gnc-features.h            |   3 +
 src/import-export/import-match-map.c     | 291 +++++++++++++++++++------------
 src/import-export/test/CMakeLists.txt    |   6 +-
 src/import-export/test/Makefile.am       |  17 +-
 src/import-export/test/test-flat-bayes.c | 289 ++++++++++++++++++++++++++++++
 src/libqof/qof/kvp_frame.c               |  24 +++
 src/libqof/qof/kvp_frame.h               |   2 +
 8 files changed, 551 insertions(+), 113 deletions(-)
 create mode 100644 src/import-export/test/test-flat-bayes.c



More information about the gnucash-changes mailing list