Last active
April 2, 2023 10:10
-
-
Save dacr/e5255e505411839a394e4f99bb8a46dc to your computer and use it in GitHub Desktop.
Converting CA stock operations histories into JSON / published by https://github.com/dacr/code-examples-manager #384c837d-b996-4139-b1bf-c5b8e70c80af/f4e0ffbf3bab970fb0dc1000d3208824bfee01c4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// summary : Converting CA stock operations histories into JSON | |
// keywords : scala, stocks, data, CA | |
// publish : gist | |
// authors : David Crosson | |
// license : Apache NON-AI License Version 2.0 (https://raw.githubusercontent.com/non-ai-licenses/non-ai-licenses/main/NON-AI-APACHE2) | |
// id : 384c837d-b996-4139-b1bf-c5b8e70c80af | |
// created-on : 2020-11-29T21:56:45Z | |
// managed-by : https://github.com/dacr/code-examples-manager | |
// execution : scala ammonite script (http://ammonite.io/) - run as follow 'amm scriptname.sc' | |
import $ivy.`com.github.pathikrit::better-files:3.9.1` | |
import $ivy.`org.json4s::json4s-jackson:3.6.10` | |
import $ivy.`org.json4s::json4s-ext:3.6.10` | |
import scala.util.Properties.envOrElse | |
import better.files._ | |
import org.json4s._ | |
import org.json4s.DefaultFormats | |
import org.json4s.ext.{JavaTimeSerializers, JavaTypesSerializers} | |
import org.json4s.jackson.Serialization.writePretty | |
import java.time.{LocalDate, OffsetDateTime, ZoneId, ZonedDateTime} | |
import java.nio.charset.Charset | |
import java.text.{DecimalFormat, NumberFormat} | |
import java.time.format.DateTimeFormatter | |
import java.util.Locale | |
val csvDataDir = envOrElse("STOCKS_CA_DATADIR", "data/").toFile | |
val csvFiles = csvDataDir.glob("**/*.csv", includePath = false) | |
val csvAccountRE = """Contrat[^:]+: (\d+)""".r.unanchored | |
val csvCharset = Charset.forName("UTF-8") | |
val csvSeparator = ";" | |
val csvLocale = Locale.FRENCH | |
val csvDateFormat = DateTimeFormatter.ofPattern("dd/MM/yyyy", csvLocale) | |
val numberFormat = { | |
NumberFormat.getInstance(csvLocale) match { | |
case nf:DecimalFormat => | |
val symbols = nf.getDecimalFormatSymbols | |
symbols.setGroupingSeparator(' ') | |
nf.setDecimalFormatSymbols(symbols) | |
nf.setGroupingUsed(true) | |
nf | |
case other => other | |
} | |
} | |
implicit val jsonFormat = DefaultFormats.lossless ++ JavaTimeSerializers.all ++ JavaTypesSerializers.all | |
object OperationKind { | |
val buy="buy" | |
val sell="sell" | |
val dividend="dividend" | |
val fee="fee" | |
val ignored="ignored" | |
} | |
case class Operation( | |
date: OffsetDateTime, | |
kind: String, | |
quantity: Int, | |
code: String, | |
codeISIN: String, | |
label: String, | |
account: String, | |
totalAmount: Double | |
) | |
def string2date(str: String,delta:Int): OffsetDateTime = { | |
LocalDate | |
.parse(str, csvDateFormat) | |
.atTime(12, 0, 0, delta*1_000_000) | |
.atZone(ZoneId.systemDefault()) | |
.toOffsetDateTime | |
} | |
def string2double(input: String): Double = numberFormat.parse(input).doubleValue() | |
def string2int(input: String): Int = numberFormat.parse(input).intValue() | |
/* | |
ACHAT COMPTANT | |
ACHAT ETRANGER | |
PAIEMENT DE COUPON | |
RETRAIT DU COMPTE DEDIE | |
SOUSCRIPTION | |
TAXE TRANSACTIONS FINANCIERES | |
VENTE COMPTANT | |
VENTE ETRANGER | |
VERSEMENT SUR COMPTE DEDIE | |
VERSEMENT SUR COMPTE PEA | |
*/ | |
val buyRE = "ACHAT .*".r | |
val sellRE = "VENTE .*".r | |
val dividendeRE = ".*COUPON".r | |
val taxRE = "TAXE.*".r | |
def convertOperationKind(input: String): String = { | |
input match { | |
case buyRE() => OperationKind.buy | |
case sellRE() => OperationKind.sell | |
case dividendeRE() => OperationKind.dividend | |
case taxRE() => OperationKind.fee | |
case _ => OperationKind.ignored | |
} | |
} | |
def processCSV(csvFile: File): List[Operation] = { | |
val lines = csvFile.lineIterator(csvCharset).toList | |
val headerLines = lines.takeWhile(line => !line.contains("Date")) | |
val account = headerLines.collect { case csvAccountRE(account) => account }.headOption | |
val operationLines = lines.dropWhile(line => !line.contains("Date")) | |
val titles = operationLines.head | |
operationLines | |
.tail | |
.reverse // to follow chronological order VERY IMPORTANT ! | |
.filterNot(_.trim.size==0) | |
.map(_.split(";").map(_.trim)) | |
.zipWithIndex | |
.map { | |
case (Array(date, kind, opType, quantity, code, isin, label, _, amount, fees, currency, dateOp), index) => | |
Operation( | |
date = string2date(dateOp, index), // compute some deltaTime to guaranty the right operations ordering | |
kind = convertOperationKind(kind), | |
quantity = string2int(quantity), | |
code = code, | |
codeISIN = isin, | |
label = label, | |
account = account.get, | |
totalAmount = string2double(amount) | |
) | |
case (entry, _) => | |
throw new RuntimeException(s"NOT UNDERSTOOD LINE ${entry.mkString(";")}") | |
} | |
} | |
val operations = csvFiles.flatMap(processCSV).toList.sortBy(_.date) | |
val outputFile = file"stock-operations.json" | |
outputFile.write(writePretty(operations)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment