Converting CA stock operations histories into JSON / published by https://github.com/dacr/code-examples-manager #384c837d-b996-4139-b1bf-c5b8e70c80af/f4e0ffbf3bab970fb0dc1000d3208824bfee01c4

ca-csv-to-json-stock.sc

// summary : Converting CA stock operations histories into JSON
// keywords : scala, stocks, data, CA
// publish : gist
// authors : David Crosson
// license : Apache NON-AI License Version 2.0 (https://raw.githubusercontent.com/non-ai-licenses/non-ai-licenses/main/NON-AI-APACHE2)
// id : 384c837d-b996-4139-b1bf-c5b8e70c80af
// created-on : 2020-11-29T21:56:45Z
// managed-by : https://github.com/dacr/code-examples-manager
// execution : scala ammonite script (http://ammonite.io/) - run as follow 'amm scriptname.sc'


import $ivy.`com.github.pathikrit::better-files:3.9.1`
import $ivy.`org.json4s::json4s-jackson:3.6.10`
import $ivy.`org.json4s::json4s-ext:3.6.10`

import scala.util.Properties.envOrElse
import better.files._
import org.json4s._
import org.json4s.DefaultFormats
import org.json4s.ext.{JavaTimeSerializers, JavaTypesSerializers}
import org.json4s.jackson.Serialization.writePretty

import java.time.{LocalDate, OffsetDateTime, ZoneId, ZonedDateTime}
import java.nio.charset.Charset
import java.text.{DecimalFormat, NumberFormat}
import java.time.format.DateTimeFormatter
import java.util.Locale


val csvDataDir = envOrElse("STOCKS_CA_DATADIR", "data/").toFile
val csvFiles = csvDataDir.glob("**/*.csv", includePath = false)

val csvAccountRE = """Contrat[^:]+: (\d+)""".r.unanchored
val csvCharset = Charset.forName("UTF-8")
val csvSeparator = ";"
val csvLocale = Locale.FRENCH
val csvDateFormat = DateTimeFormatter.ofPattern("dd/MM/yyyy", csvLocale)

val numberFormat = {
  NumberFormat.getInstance(csvLocale) match {
    case nf:DecimalFormat =>
      val symbols = nf.getDecimalFormatSymbols
      symbols.setGroupingSeparator(' ')
      nf.setDecimalFormatSymbols(symbols)
      nf.setGroupingUsed(true)
      nf
    case other => other
  }
}

implicit val jsonFormat = DefaultFormats.lossless ++ JavaTimeSerializers.all ++ JavaTypesSerializers.all


object OperationKind {
  val buy="buy"
  val sell="sell"
  val dividend="dividend"
  val fee="fee"
  val ignored="ignored"
}
case class Operation(
  date: OffsetDateTime,
  kind: String,
  quantity: Int,
  code: String,
  codeISIN: String,
  label: String,
  account: String,
  totalAmount: Double
)

def string2date(str: String,delta:Int): OffsetDateTime = {
  LocalDate
    .parse(str, csvDateFormat)
    .atTime(12, 0, 0, delta*1_000_000)
    .atZone(ZoneId.systemDefault())
    .toOffsetDateTime
}

def string2double(input: String): Double = numberFormat.parse(input).doubleValue()

def string2int(input: String): Int = numberFormat.parse(input).intValue()
/*
ACHAT COMPTANT
ACHAT ETRANGER
PAIEMENT DE COUPON
RETRAIT DU COMPTE DEDIE
SOUSCRIPTION
TAXE TRANSACTIONS FINANCIERES
VENTE COMPTANT
VENTE ETRANGER
VERSEMENT SUR COMPTE DEDIE
VERSEMENT SUR COMPTE PEA
*/
val buyRE = "ACHAT .*".r
val sellRE = "VENTE .*".r
val dividendeRE = ".*COUPON".r
val taxRE = "TAXE.*".r
def convertOperationKind(input: String): String = {
  input match {
    case buyRE() => OperationKind.buy
    case sellRE() => OperationKind.sell
    case dividendeRE() => OperationKind.dividend
    case taxRE() => OperationKind.fee
    case _ => OperationKind.ignored
  }
}

def processCSV(csvFile: File): List[Operation] = {
  val lines = csvFile.lineIterator(csvCharset).toList
  val headerLines = lines.takeWhile(line => !line.contains("Date"))
  val account = headerLines.collect { case csvAccountRE(account) => account }.headOption
  val operationLines = lines.dropWhile(line => !line.contains("Date"))
  val titles = operationLines.head
  operationLines
    .tail
    .reverse // to follow chronological order VERY IMPORTANT !
    .filterNot(_.trim.size==0)
    .map(_.split(";").map(_.trim))
    .zipWithIndex
    .map {
      case (Array(date, kind, opType, quantity, code, isin, label, _, amount, fees, currency, dateOp), index) =>
        Operation(
          date = string2date(dateOp, index), // compute some deltaTime to guaranty the right operations ordering
          kind = convertOperationKind(kind),
          quantity = string2int(quantity),
          code = code,
          codeISIN = isin,
          label = label,
          account = account.get,
          totalAmount = string2double(amount)
        )
      case (entry, _) =>
        throw new RuntimeException(s"NOT UNDERSTOOD LINE ${entry.mkString(";")}")
    }
}

val operations = csvFiles.flatMap(processCSV).toList.sortBy(_.date)
val outputFile = file"stock-operations.json"
outputFile.write(writePretty(operations))