Patch to write per-account files on importing into Beancount

per-account-files.patch

*** /home/username/gnu/beancount/beancount/ingest/extract.py	2022-10-22 01:50:13.590000000 -0700
--- /home/username/.local/lib/python3.8/site-packages/beancount/ingest/extract.py	2022-12-16 01:39:31.110000000 -0800
***************
*** 11,16 ****
--- 11,18 ----
  import logging
  import sys
  import textwrap
+ import os
+ import re
  
  from beancount.core import data
  from beancount.parser import printer
***************
*** 114,119 ****
--- 116,178 ----
      return mod_entries_list
  
  
+ pat_ticker = re.compile("Assets:Invest.*:[A-Z0-9]*$")
+ 
+ 
+ def filing_target(entry, output_file_set, output_dir):
+     # Determine file to write to
+     base_acct = None
+     if 'filing_account' in entry.meta:
+         base_acct = entry.meta['filing_account']
+         del entry.meta['filing_account']
+     elif hasattr(entry, 'postings') and len(entry.postings) >= 1:
+         base_acct = entry.postings[0].account
+     elif hasattr(entry, 'account'):
+         base_acct = entry.account
+     else:
+         base_acct = 'noaccount'
+ 
+     # file Assets:Investments:Brokerage:HOOLI to Assets.Investments.Brokerage.bc
+     if pat_ticker.match(base_acct):
+         base_acct = base_acct.rsplit(':', 1)[0]
+ 
+     # Determine outfile
+     if base_acct in output_file_set:
+         outfile = output_file_set[base_acct]
+     else:
+         filename = base_acct.replace(':', '.') + ".bc"
+         filename_full = os.path.abspath(output_dir + os.sep + filename)
+         outfile = open(filename_full, 'a')
+         output_file_set[base_acct] = outfile
+ 
+     return outfile
+ 
+ 
+ def output_extracted_entries_to_account_files(entries, output_dir=None):
+     """Print the entries for the given importer, to one file per account
+     """
+     if not os.path.exists(output_dir):
+         os.makedirs(output_dir)
+     output_file_set = {}
+     # Print out the entries.
+     for entry in entries:
+         outfile = filing_target(entry, output_file_set, output_dir)
+ 
+         # Check if this entry is a dup, and if so, comment it out.
+         if DUPLICATE_META in entry.meta:
+             meta = entry.meta.copy()
+             meta.pop(DUPLICATE_META)
+             entry = entry._replace(meta=meta)
+             entry_string = textwrap.indent(printer.format_entry(entry), '; ')
+         else:
+             entry_string = printer.format_entry(entry)
+ 
+         print(entry_string, file=outfile)
+ 
+     for f in output_file_set.values():
+         f.close()
+ 
+ 
  def print_extracted_entries(entries, file):
      """Print a list of entries.
  
***************
*** 122,127 ****
--- 181,187 ----
        file: A file object to write to.
      """
      # Print the filename and which modules matched.
+     # pylint: disable=invalid-name
      pr = lambda *args: print(*args, file=file)
      pr('')
  
***************
*** 147,152 ****
--- 207,214 ----
              options_map=None,
              mindate=None,
              ascending=True,
+             output_one_file_per_account=True,
+             output_dir='ingest_output',
              hooks=None):
      """Given an importer configuration, search for files that can be imported in the
      list of files or directories, run the signature checks on them, and if it
***************
*** 211,217 ****
          output.write('\n')
          if not ascending:
              new_entries.reverse()
!         print_extracted_entries(new_entries, output)
  
  
  DESCRIPTION = "Extract transactions from downloads"
--- 273,283 ----
          output.write('\n')
          if not ascending:
              new_entries.reverse()
!         # print_extracted_entries(new_entries, output)
!         if output_one_file_per_account:
!             output_extracted_entries_to_account_files(new_entries, output_dir)
!         else:
!             print_extracted_entries(new_entries, output)
  
  
  DESCRIPTION = "Extract transactions from downloads"
***************
*** 230,235 ****
--- 296,311 ----
                          default=True, const=False,
                          help='Write out the entries in descending order')
  
+     parser.add_argument('-o', '--one-file-per-account', '--one',
+                         action='store_true', dest='output_one_file_per_account',
+                         default=False,
+                         help='Output one file per account')
+ 
+     parser.add_argument('-d', '--output-dir', '--dir', metavar='OUTPUT_DIR',
+                         dest='output_dir',
+                         default='ingest_output',
+                         help='Output dir (if output one file per account is requested')
+ 
  
  def run(args, _, importers_list, files_or_directories, hooks=None):
      """Run the subcommand."""
***************
*** 245,249 ****
--- 321,327 ----
              options_map=options_map,
              mindate=None,
              ascending=args.ascending,
+             output_one_file_per_account=args.output_one_file_per_account,
+             output_dir=args.output_dir,
              hooks=hooks)
      return 0

Can you explain the steps needed to get this working. Do we have to rebuild the package before using it? Can you share the complete file, without the diffs.

This patch is provided for folks who are reasonably comfortable with python, pip, packages, and patching. I'll admit, it is difficult to apply, debug, and use without those. That said, some pointers: see man patch, and see the top of the patch above on what file to patch. You can patch either the installed file or rebuild the package.