From 11107baa6adf865f13b5ae82cfb211bb33a572b5 Mon Sep 17 00:00:00 2001 From: niten Date: Sat, 23 Mar 2024 11:23:14 -0700 Subject: [PATCH] Updates... --- deps.edn | 5 +- deps.nix | 269 ++++++++++++++++++++-------- flake.nix | 15 +- src/worther/core.clj | 102 ++++++++++- src/worther/injest/bittrex.clj | 50 +++--- src/worther/injest/coinbase.clj | 195 ++++++++++++-------- src/worther/injest/coinbase_pro.clj | 66 +++---- src/worther/injest/common.clj | 72 ++------ src/worther/originator.clj | 158 ++++++++++++++++ src/worther/txn.clj | 119 ++++++++++++ 10 files changed, 778 insertions(+), 273 deletions(-) create mode 100644 src/worther/originator.clj create mode 100644 src/worther/txn.clj diff --git a/deps.edn b/deps.edn index 9ebaba6..bab549d 100644 --- a/deps.edn +++ b/deps.edn @@ -1,6 +1,9 @@ {:paths ["src"] :deps - {org.clojure/data.csv {:mvn/version "1.0.0"} + {org.clojure/clojure {:mvn/version "1.11.0"} + org.clojure/data.csv {:mvn/version "1.0.1"} + org.clojure/core.async {:mvn/version "1.5.648"} org.clojure/core.match {:mvn/version "1.0.0"} + org.clojure/tools.cli {:mvn/version "1.0.206"} arachne-framework/valuehash {:git/url "https://git.fudo.org/fudo-public/valuehash.git" :sha "9d2dbafdb5db886a57f44c5b7fe32c824713e6c7"}}} diff --git a/deps.nix b/deps.nix index b0c3c82..70f4853 100644 --- a/deps.nix +++ b/deps.nix @@ -1,82 +1,54 @@ -# generated by clj2nix-1.0.7 -{ pkgs ? import {} }: +# generated by clj2nix-1.1.0-rc +{ fetchMavenArtifact, fetchgit, lib }: let repos = [ "https://repo1.maven.org/maven2/" "https://repo.clojars.org/" ]; in rec { - fetchmaven = pkgs.callPackage (pkgs.fetchurl { - url = "https://raw.githubusercontent.com/NixOS/nixpkgs/ba5e2222458a52357a3ba5873d88779d5c223269/pkgs/build-support/fetchmavenartifact/default.nix"; - sha512 = "05m7i8hbhyfz7p2f106mfbsasjf04svd9xkgc26pl3shljrk0dfacz39wiwzm6xqw7czgrsx745vciram7al621v7634nfdq3m1x88a"; - }) {}; - makePaths = {extraClasspaths ? null}: - (pkgs.lib.concatMap - (dep: - builtins.map - (path: - if builtins.isString path then - path - else if builtins.hasAttr "jar" path then - path.jar - else if builtins.hasAttr "outPath" path then - path.outPath - else - path - ) - dep.paths) - packages) - ++ (if extraClasspaths != null then [ extraClasspaths ] else []); - makeClasspaths = {extraClasspaths ? null}: builtins.concatStringsSep ":" (makePaths {inherit extraClasspaths;}); + makePaths = {extraClasspaths ? []}: + if (builtins.typeOf extraClasspaths != "list") + then builtins.throw "extraClasspaths must be of type 'list'!" + else (lib.concatMap (dep: + builtins.map (path: + if builtins.isString path then + path + else if builtins.hasAttr "jar" path then + path.jar + else if builtins.hasAttr "outPath" path then + path.outPath + else + path + ) + dep.paths) + packages) ++ extraClasspaths; + makeClasspaths = {extraClasspaths ? []}: + if (builtins.typeOf extraClasspaths != "list") + then builtins.throw "extraClasspaths must be of type 'list'!" + else builtins.concatStringsSep ":" (makePaths {inherit extraClasspaths;}); packageSources = builtins.map (dep: dep.src) packages; packages = [ - rec { - name = "data.csv/org.clojure"; - src = fetchmaven { - inherit repos; - artifactId = "data.csv"; - groupId = "org.clojure"; - sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b"; - version = "1.0.0"; - - }; - paths = [ src ]; - } - - rec { - name = "core.match/org.clojure"; - src = fetchmaven { - inherit repos; - artifactId = "core.match"; - groupId = "org.clojure"; - sha512 = "52ada3bbe73ed1b429be811d3990df0cdb3e9d50f2a6c92b70d490a8ea922d4794da93c3b7487653f801954fc599704599b318b4d7926a9594583df37c55e926"; - version = "1.0.0"; - - }; - paths = [ src ]; - } - - (rec { - name = "arachne-framework/valuehash"; - src = pkgs.fetchgit { - name = "valuehash"; - url = "https://git.fudo.org/fudo-public/valuehash.git"; - rev = "9d2dbafdb5db886a57f44c5b7fe32c824713e6c7"; - sha256 = "1civ393c4yy9p2xbmrrvpbyqczx55k3fkvimkf850fl62ns5zl9r"; - }; - paths = map (path: src + path) [ - "/src" - ]; - }) - rec { name = "clojure/org.clojure"; - src = fetchmaven { + src = fetchMavenArtifact { inherit repos; artifactId = "clojure"; groupId = "org.clojure"; - sha512 = "d9e2c0676cdc349a3455d92b3ce3c3f01a2410de448c9416edfe72bc7eaf356cfadbb6d746740a821940c3b4cab100ca941e23bab482e98b404ed9ef79c562df"; - version = "1.10.0-alpha4"; + sha512 = "efb87dfd347d2be6cb251d550e312d77797e35500b75ebe8e3fca824d16223803305ce89d4ae0349e5dff22a99c24b8719bf791f24685a12404bd56a44693010"; + version = "1.11.0"; + + }; + paths = [ src ]; + } + + rec { + name = "tools.analyzer/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "tools.analyzer"; + groupId = "org.clojure"; + sha512 = "c51752a714848247b05c6f98b54276b4fe8fd44b3d970070b0f30cd755ac6656030fd8943a1ffd08279af8eeff160365be47791e48f05ac9cc2488b6e2dfe504"; + version = "1.1.0"; }; paths = [ src ]; @@ -84,7 +56,7 @@ let repos = [ rec { name = "tools.logging/org.clojure"; - src = fetchmaven { + src = fetchMavenArtifact { inherit repos; artifactId = "tools.logging"; groupId = "org.clojure"; @@ -96,26 +68,169 @@ let repos = [ } rec { - name = "spec.alpha/org.clojure"; - src = fetchmaven { + name = "core.specs.alpha/org.clojure"; + src = fetchMavenArtifact { inherit repos; - artifactId = "spec.alpha"; + artifactId = "core.specs.alpha"; groupId = "org.clojure"; - sha512 = "b8fc40ed9bc52b545e699ed188dd61bfd144ee67f0c70364b8f2715e9f1fea608d3721db7f618f6ef4bc3056e3c2984c626080486ca710f3595dda8ba23730ac"; - version = "0.1.143"; + sha512 = "f521f95b362a47bb35f7c85528c34537f905fb3dd24f2284201e445635a0df701b35d8419d53c6507cc78d3717c1f83cda35ea4c82abd8943cd2ab3de3fcad70"; + version = "0.2.62"; }; paths = [ src ]; } rec { - name = "core.specs.alpha/org.clojure"; - src = fetchmaven { + name = "spec.alpha/org.clojure"; + src = fetchMavenArtifact { inherit repos; - artifactId = "core.specs.alpha"; + artifactId = "spec.alpha"; groupId = "org.clojure"; - sha512 = "b4f5eee01da39914e6024dd529d1f72952d5a9dae65e1e41bf386b1e86a004a0d197b5be95aa70e7e8d6438c92b7fa8fc0c5039f2013e97c0b91c22d86fb7968"; - version = "0.1.24"; + sha512 = "ddfe4fa84622abd8ac56e2aa565a56e6bdc0bf330f377ff3e269ddc241bb9dbcac332c13502dfd4c09c2c08fe24d8d2e8cf3d04a1bc819ca5657b4e41feaa7c2"; + version = "0.3.218"; + + }; + paths = [ src ]; + } + + rec { + name = "tools.cli/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "tools.cli"; + groupId = "org.clojure"; + sha512 = "1d88aa03eb6a664bf2c0ce22c45e7296d54d716e29b11904115be80ea1661623cf3e81fc222d164047058239010eb678af92ffedc7c3006475cceb59f3b21265"; + version = "1.0.206"; + + }; + paths = [ src ]; + } + + rec { + name = "tools.analyzer.jvm/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "tools.analyzer.jvm"; + groupId = "org.clojure"; + sha512 = "36ad50a7a79c47dea16032fc4b927bd7b56b8bedcbd20cc9c1b9c85edede3a455369b8806509b56a48457dcd32e1f708f74228bce2b4492bd6ff6fc4f1219d56"; + version = "1.2.2"; + + }; + paths = [ src ]; + } + + rec { + name = "asm/org.ow2.asm"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "asm"; + groupId = "org.ow2.asm"; + sha512 = "876eac7406e60ab8b9bd6cd3c221960eaa53febea176a88ae02f4fa92dbcfe80a3c764ba390d96b909c87269a30a69b1ee037a4c642c2f535df4ea2e0dd499f2"; + version = "9.2"; + + }; + paths = [ src ]; + } + + rec { + name = "data.csv/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "data.csv"; + groupId = "org.clojure"; + sha512 = "6b667a56cbb6632a90564f217f9c28a6670c13c729fb205ced091d9ee006382143dd6b615dd5a4f900660946199cac449fe9fabc90820bb34b92a9e6c8550473"; + version = "1.0.1"; + + }; + paths = [ src ]; + } + + rec { + name = "core.match/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "core.match"; + groupId = "org.clojure"; + sha512 = "52ada3bbe73ed1b429be811d3990df0cdb3e9d50f2a6c92b70d490a8ea922d4794da93c3b7487653f801954fc599704599b318b4d7926a9594583df37c55e926"; + version = "1.0.0"; + + }; + paths = [ src ]; + } + + rec { + name = "tools.reader/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "tools.reader"; + groupId = "org.clojure"; + sha512 = "3481259c7a1eac719db2921e60173686726a0c2b65879d51a64d516a37f6120db8ffbb74b8bd273404285d7b25143ab5c7ced37e7c0eaf4ab1e44586ccd3c651"; + version = "1.3.6"; + + }; + paths = [ src ]; + } + + rec { + name = "core.memoize/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "core.memoize"; + groupId = "org.clojure"; + sha512 = "67196537084b7cc34a01454d2a3b72de3fddce081b72d7a6dc1592d269a6c2728b79630bd2d52c1bf2d2f903c12add6f23df954c02ef8237f240d7394ccc3dde"; + version = "1.0.253"; + + }; + paths = [ src ]; + } + + rec { + name = "data.priority-map/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "data.priority-map"; + groupId = "org.clojure"; + sha512 = "bb8bc5dbfd3738c36b99a51880ac3f1381d6564e67601549ef5e7ae2b900e53cdcdfb8d0fa4bf32fb8ebc4de89d954bfa3ab7e8a1122bc34ee5073c7c707ac13"; + version = "1.1.0"; + + }; + paths = [ src ]; + } + + (rec { + name = "arachne-framework/valuehash"; + src = fetchgit { + name = "valuehash"; + url = "https://git.fudo.org/fudo-public/valuehash.git"; + rev = "9d2dbafdb5db886a57f44c5b7fe32c824713e6c7"; + sha256 = "1civ393c4yy9p2xbmrrvpbyqczx55k3fkvimkf850fl62ns5zl9r"; + }; + paths = map (path: src + path) [ + "/src" + ]; + }) + + rec { + name = "core.cache/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "core.cache"; + groupId = "org.clojure"; + sha512 = "0a07ceffc2fa3a536b23773eefc7ef5e1108913b93c3a5416116a6566de76dd5c218f3fb0cc19415cbaa8843838de310b76282f20bf1fc3467006c9ec373667e"; + version = "1.0.225"; + + }; + paths = [ src ]; + } + + rec { + name = "core.async/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "core.async"; + groupId = "org.clojure"; + sha512 = "160a77da25382d7c257eee56cfe83538620576a331e025a2d672fc26d9f04e606666032395f3c2e26247c782544816a5862348f3a921b1ffffcd309c62ac64f5"; + version = "1.5.648"; }; paths = [ src ]; diff --git a/flake.nix b/flake.nix index 5a15498..96ca944 100644 --- a/flake.nix +++ b/flake.nix @@ -16,14 +16,15 @@ # defaultPackage = packages.${system}.worther; # }; - outputs = { self, nixpkgs, flake-utils, ... }: with nixpkgs.lib; + outputs = { self, nixpkgs, flake-utils, ... }: + with nixpkgs.lib; flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { inherit system; }; - worther = pkgs.callPackage ./worther.nix { pkgs = pkgs; }; + let pkgs = import nixpkgs { inherit system; }; - in rec { - packages.worther = worther; - defaultPackage = self.packages.${system}.worther; + in { + packages = rec { + default = worther; + worther = pkgs.callPackage ./worther.nix { pkgs = pkgs; }; + }; }); } diff --git a/src/worther/core.clj b/src/worther/core.clj index d02c8aa..2987dc2 100644 --- a/src/worther/core.clj +++ b/src/worther/core.clj @@ -1,5 +1,101 @@ -(ns worther.core) +(ns worther.core + (:require [worther.injest.bittrex :as bittrex] + [worther.injest.coinbase :as cb] + [worther.injest.coinbase-pro :as cb-pro] + [worther.txn :as txn] + + [clojure.pprint :refer [pprint]] + [clojure.string :as str] + [clojure.tools.cli :refer [parse-opts]]) + (:gen-class)) + +(defn- ensure-directory [dir] + (if (not (.isDirectory dir)) + (throw (java.io.FileNotFoundException. + (str "Invalid directory: " dir))) + dir)) + +(defn- directory-contents [dir-str] + (-> dir-str clojure.java.io/file ensure-directory file-seq)) + +(defn- has-ext? [ext] + (fn [file] + (re-matches (re-pattern (str ".+" "\\." ext "$")) + (str file)))) + +(defn- pthru [obj] + (pprint obj) + obj) + +(defn- load-dir-csvs [loader dir] + (mapcat loader + (filter (has-ext? "csv") + (directory-contents dir)))) + +(def bittrex-load (partial load-dir-csvs bittrex/load-csv)) +(def coinbase-load (partial load-dir-csvs cb/load-csv)) +(def coinbase-pro-load (partial load-dir-csvs cb-pro/load-csv)) + +(defn- directory? [dir] + (-> dir clojure.java.io/file .isDirectory)) + +(defn- valid-year? [year] + (< 2010 year 2100)) + +(def cli-options + [["-h" "--help"] + + ["-c" "--coinbase DIR" "Coinbase CSV directory" + :default nil + :parse-fn clojure.java.io/file + :validate [directory? "Must be a valid directory."]] + + ["-C" "--coinbase-pro DIR" "Coinbase Pro CSV directory" + :default nil + :parse-fn clojure.java.io/file + :validate [directory? "Must be a valid directory."]] + + ["-b" "--bittrex DIR" "Bittrex CSV directory" + :default nil + :parse-fn clojure.java.io/file + :validate [directory? "Must be a valid directory."]] + + ["-y" "--year YEAR" "Year for which to generate tax data." + :parse-fn #(Integer/parseInt %) + :validate [valid-year? "Must be a year between 2010 and 2100."]]]) + +(defn- print-usage [opts] + (->> ["usage: worther [options]" "" "options:" opts] + (str/join \newline))) + +(defn- error? [status] + (not (= 0 status))) + +(defn- error-quit [status msg] + (if (error? status) + (println (str "error: " msg)) + (println msg)) + (System/exit status)) + +(def join-lines (partial str/join \newline)) + +(defmacro launch-task [name body] + (let [res (gensym)] + `(do (println (str "beginning " ~name "...")) + (let [~res ~body] + (println (str "completed " ~name ".")) + ~res)))) + +(defn- load-data [opts] + (let [before (fn [a b] (.before a b))] + (sort-by ::txn/timestamp + before + (concat (or (some-> opts :coinbase coinbase-load) []) + (or (some-> opts :coinbase-pro coinbase-pro-load) []) + (or (some-> opts :bittrex bittrex-load) []))))) (defn -main [& args] - (println "Not implemented yet!") - (println (str "called with: " args))) + (let [opts (parse-opts args cli-options)] + (cond (-> opts :options :help) (->> opts :summary print-usage (error-quit 0)) + (-> opts :errors empty? not) (->> opts :errors join-lines (error-quit 1)) + :else (println (str "count: " (count (load-data (:options opts)))))))) diff --git a/src/worther/injest/bittrex.clj b/src/worther/injest/bittrex.clj index 493a875..c0a7426 100644 --- a/src/worther/injest/bittrex.clj +++ b/src/worther/injest/bittrex.clj @@ -1,30 +1,32 @@ (ns worther.injest.bittrex (:require [worther.injest.common :as common] + [worther.txn :as txn] + [clojure.core.match :refer [match]] [clojure.spec.alpha :as s] [clojure.string :as str])) (defn split-buy [txn] [(-> txn - (assoc ::common/txn-type ::common/buy - ::common/quantity-transacted (::quantity txn) - ::common/currency (::market-currency txn))) + (assoc ::txn/txn-type ::txn/buy + ::txn/quantity-transacted (::quantity txn) + ::txn/currency (::market-currency txn))) (-> txn - (assoc ::common/txn-type ::common/sell - ::common/quantity-transacted (- (* (::price-per txn) + (assoc ::txn/txn-type ::txn/sell + ::txn/quantity-transacted (- (* (::price-per txn) (::quantity txn))) - ::common/currency (::base-currency txn)))]) + ::txn/currency (::base-currency txn)))]) (defn split-sell [txn] [(-> txn - (assoc ::common/txn-type ::common/sell - ::common/quantity-transacted (- (::quantity txn)) - ::common/currency (::market-currency txn))) + (assoc ::txn/txn-type ::txn/sell + ::txn/quantity-transacted (- (::quantity txn)) + ::txn/currency (::market-currency txn))) (-> txn - (assoc ::common/txn-type ::common/buy - ::common/quantity-transacted (* (::price-per txn) + (assoc ::txn/txn-type ::txn/buy + ::txn/quantity-transacted (* (::price-per txn) (::quantity txn)) - ::common/currency (::base-currency txn)))]) + ::txn/currency (::base-currency txn)))]) (defmulti type-specific-translation ::txn-type) @@ -41,10 +43,10 @@ (split-buy txn)) (defmethod type-specific-translation :fee [txn] - [(assoc txn ::common/txn-type ::common/fee)]) + [(assoc txn ::txn/txn-type ::txn/fee)]) (defmethod type-specific-translation nil [txn] - (throw (RuntimeException. (str "::txn-type missing from transaction: " txn)))) + (throw (RuntimeException. (str "::txn-type missing from transaction: " (common/pprint-str txn))))) (defn to-uuid [str] (java.util.UUID/fromString str)) @@ -52,11 +54,11 @@ [txn { ::txn-type :fee - ::common/timestamp (::common/timestamp txn) - ::common/account (::common/account txn) - ::common/usd-value :deferred - ::common/quantity-transacted (::quantity txn) - ::common/currency (::base-currency txn) + ::txn/timestamp (::txn/timestamp txn) + ::txn/account (::txn/account txn) + ::txn/usd-value :deferred + ::txn/quantity-transacted (::quantity txn) + ::txn/currency (::base-currency txn) }]) (defn split-currencies [txn] @@ -74,7 +76,7 @@ split-currencies (common/alter-field :uuid to-uuid) (common/rename-field :timestamp - ::common/timestamp + ::txn/timestamp (common/date-parser "M/d/yyyy HH:mm:ss aa")) (common/rename-field :ordertype ::txn-type @@ -97,15 +99,17 @@ (common/rename-field :closed ::closed (common/date-parser "M/d/yyyy HH:mm:ss aa")) - (common/add-field ::common/account + (common/add-field ::txn/account (fn [_] :bittrex)))) (defn load-csv [filename] (let [txn-translator (common/flatcomp - (common/add-txid ::common/txid) + (common/add-txid ::txn/txid) type-specific-translation translate-bittrex-txn)] (->> (common/load-csv filename) (common/pivot-csv) (common/flatmap txn-translator)))) - +(s/fdef load-csv + :args (s/cat :filename string?) + :ret (s/coll-of ::txn/txn)) diff --git a/src/worther/injest/coinbase.clj b/src/worther/injest/coinbase.clj index 87d58fc..f9239ac 100644 --- a/src/worther/injest/coinbase.clj +++ b/src/worther/injest/coinbase.clj @@ -1,6 +1,9 @@ (ns worther.injest.coinbase (:require [worther.injest.common :as common] - [clojure.spec.alpha :as s])) + [worther.txn :as txn] + [clojure.spec.alpha :as s] + [clojure.string :as str] + [clojure.pprint :refer [pprint]])) (defn print-through [o] (clojure.pprint/pprint o) @@ -13,90 +16,115 @@ (def txn-type-map { - :fee ::common/fee - :buy ::common/buy - :sell ::common/sell - :send ::common/send - :receive ::common/receive - :paid-for-an-order ::common/sell + :fee ::txn/fee + :buy ::txn/buy + :sell ::txn/sell + :send ::txn/send + :receive ::txn/receive + :paid-for-an-order ::txn/sell + :cardspend ::txn/sell + :cardbuyback ::txn/buy + :rewards-income ::txn/income + :convert ::txn/sell }) (s/def ::txn-type (set (keys txn-type-map))) (defn update-txn-type [txn] - (assoc txn ::common/txn-type + (assoc txn ::txn/txn-type (get txn-type-map (::txn-type txn)))) (defmulti type-specific-translation ::txn-type) +(s/fdef type-specific-translation + :args (s/cat :txn ::coinbase-txn) + :ret (s/+ ::txn/txn)) (defn split-fee [txn] - (if (and (::usd-fees txn) - (> (::usd-fees txn) 0)) + (if (> (::usd-fees txn) 0) [(update-txn-type txn) { - ::txn-type :fee - ::common/quantity-transacted (::usd-fees txn) - ::common/currency :usd - ::common/account :coinbase - ::common/timestamp (::common/timestamp txn) - ::common/notes (str "Fee for: " (::common/notes txn)) - ::common/usd-value (::usd-fees txn) + ::txn-type :fee + ::txn/txn-type ::txn/fee + ::txn/quantity-transacted (::usd-fees txn) + ::txn/currency :usd + ::txn/account :coinbase + ::txn/timestamp (::txn/timestamp txn) + ::txn/notes (str "Fee for: " (::txn/notes txn)) + ::txn/usd-value (::usd-fees txn) }] [(update-txn-type txn)])) -(defmethod type-specific-translation :buy [txn] - [(update-txn-type txn) - { - ::common/txn-type ::common/deposit - ::common/quantity-transacted (::common/usd-value txn) - ::common/currency :usd - ::common/account :coinbase - ::common/timestamp (::common/timestamp txn) - ::common/notes (str "Deposit for: " (::common/notes txn)) - ::common/usd-value (::common/usd-value txn) - }]) -(defmethod type-specific-translation :sell [txn] - [(update (update-txn-type txn) ::common/quantity-transacted -) - { - ::common/txn-type ::common/withdrawal - ::common/quantity-transacted (::common/usd-value txn) - ::common/currency :usd - ::common/account :coinbase - ::common/timestamp (::common/timestamp txn) - ::common/notes (str "Withdrawal for: " (::common/notes txn)) - ::common/usd-value (::common/usd-value txn) - }]) +(defn- extract-conversion-data [conv-str] + (let [read-amount (fn [amt] + (-> amt + (str/replace #"," "") + (bigdec))) + number-rx "[0-9]+(,[0-9]+)*\\.[0-9]+" + currency-rx "[A-Z][A-Z0-9]{2,5}" + conversion-rx (re-pattern (str "^Converted " + number-rx + " " + currency-rx + " to (?" + number-rx + ") (?" + currency-rx + ")$")) + matches (re-matches conversion-rx conv-str) + amt (-> matches (nth 2) read-amount) + currency (-> matches (nth 4) common/to-keyword)] + [amt currency])) -(defmethod type-specific-translation :paid-for-an-order [txn] - (let [usd-value (* (::common/usd-current-price txn) - (::common/quantity-transacted txn))] - [(-> (update-txn-type txn) - (assoc ::common/usd-value (- usd-value))) +(defmethod type-specific-translation :convert [txn] + (let [[qty currency] (-> txn ::txn/notes extract-conversion-data)] + [(-> txn + update-txn-type + (update ::txn/quantity-transacted (comp - abs))) { - ::common/txn-type ::common/send - ::common/quantity-transacted usd-value - ::common/currency :usd - ::common/account :coinbase - ::common/timestamp (::common/timestamp txn) - ::common/notes (str "Send to merchant for: " (::common/notes txn)) - ::common/usd-value usd-value - ::common/recipient :merchant + ::txn-type :buy + ::txn/txn-type ::txn/buy + ::txn/quantity-transacted qty + ::txn/currency currency + ::txn/account :coinbase + ::txn/timestamp (::txn/timestamp txn) + ::txn/notes (str "Converted from " (::txn/currency txn) " to " currency) + ::txn/usd-value :deferred }])) +(defmethod type-specific-translation :buy [txn] + [(-> txn update-txn-type)]) + +(defmethod type-specific-translation :cardbuyback [txn] + [(-> txn update-txn-type)]) + +(defmethod type-specific-translation :sell [txn] + [(-> txn update-txn-type + (update ::txn/quantity-transacted (comp - abs)))]) + +(defmethod type-specific-translation :paid-for-an-order [txn] + (let [usd-value (* (::txn/usd-current-price txn) + (::txn/quantity-transacted txn))] + [(-> txn update-txn-type (assoc ::txn/usd-value (- usd-value)))])) + +(defmethod type-specific-translation :cardspend [txn] + (let [usd-value (* (::txn/usd-current-price txn) + (::txn/quantity-transacted txn))] + [(-> txn update-txn-type (assoc ::txn/usd-value (- usd-value)))])) + (defmethod type-specific-translation :send [txn] - (let [usd-value (* (::common/usd-current-price txn) - (::common/quantity-transacted txn))] + (let [usd-value (* (::txn/usd-current-price txn) + (::txn/quantity-transacted txn))] [(-> (update-txn-type txn) - (assoc ::common/recipient :unknown) - (assoc ::common/usd-value usd-value) - (update ::common/quantity-transacted -))])) + (assoc ::txn/recipient :unknown) + (assoc ::txn/usd-value usd-value) + (update ::txn/quantity-transacted -))])) (defmethod type-specific-translation :receive [txn] - (let [usd-value (* (::common/usd-current-price txn) - (::common/quantity-transacted txn))] + (let [usd-value (* (::txn/usd-current-price txn) + (::txn/quantity-transacted txn))] [(-> (update-txn-type txn) - (assoc ::common/usd-value usd-value) - (assoc ::common/sender :unknown))])) + (assoc ::txn/usd-value usd-value) + (assoc ::txn/sender :unknown))])) (defmethod type-specific-translation :fee [txn] [(update-txn-type txn)]) @@ -104,42 +132,57 @@ (defmethod type-specific-translation nil [txn] (throw (RuntimeException. (str "Missing txn-type: " txn)))) +(defn pthru [obj] (pprint obj) obj) + +(defn- register-reward + "Coinbase Card pays rewards in XLM." + [txn] + (if (and (= (::txn/currency txn) :xlm) + (= (::txn-type txn) :receive)) + [(-> txn + (assoc ::txn/txn-type ::txn/income) + (assoc ::txn/sender :coinbase-card))] + [txn])) + (def translate-coinbase-txns (common/flatcomp - (common/add-field ::common/account + (common/add-field ::txn/account (fn [_] :coinbase)) (common/rename-field :notes - ::common/notes) + ::txn/notes) (common/rename-field :asset - ::common/currency + ::txn/currency common/to-keyword) - (common/rename-field :usd-fees + (common/rename-field :fees ::usd-fees (common/zero-or bigdec)) - (common/rename-field :usd-subtotal - ::common/usd-value + (common/rename-field :subtotal + ::txn/usd-value (common/null-or bigdec)) (common/rename-field :timestamp - ::common/timestamp + ::txn/timestamp (common/date-parser "yyyy-MM-dd'T'HH:mm:ss'Z'")) (common/rename-field :transaction-type ::txn-type common/to-keyword) - (common/rename-field :usd-spot-price-at-transaction - ::common/usd-current-price + (common/rename-field :spot-price-at-transaction + ::txn/usd-current-price (common/null-or bigdec)) (common/rename-field :quantity-transacted - ::common/quantity-transacted + ::txn/quantity-transacted bigdec))) (defn load-csv [filename] (let [header-line? (fn [cells] (= (first cells) "Timestamp")) - txn-translator (common/flatcomp - (common/add-txid ::common/txid) - type-specific-translation - split-fee - translate-coinbase-txns)] + txn-translator (common/flatcomp (common/add-txid ::txn/txid) + register-reward + type-specific-translation + split-fee + translate-coinbase-txns)] (->> (common/load-csv filename) (drop-while (comp not header-line?)) (common/pivot-csv) (common/flatmap txn-translator)))) +(s/fdef load-csv + :args (s/cat :filename string?) + :ret (s/coll-of ::txn/txn)) diff --git a/src/worther/injest/coinbase_pro.clj b/src/worther/injest/coinbase_pro.clj index 5aaf4ff..63abcda 100644 --- a/src/worther/injest/coinbase_pro.clj +++ b/src/worther/injest/coinbase_pro.clj @@ -1,5 +1,6 @@ (ns worther.injest.coinbase-pro (:require [worther.injest.common :as common] + [worther.txn :as txn] [clojure.core.match :refer [match]] [clojure.spec.alpha :as s])) @@ -30,62 +31,63 @@ (defn parse-int [i] (Integer/parseInt i)) -(defn parse-uuid [str] (java.util.UUID/fromString str)) +;; Supplied by Clojure as of 1.11 +;; (defn parse-uuid [str] (java.util.UUID/fromString str)) (defn print-through [o] (clojure.pprint/pprint o) o) (defn coinbase-pro-txn->common-txn [txn] (get { - :withdrawal ::common/withdrawal - :deposit ::common/deposit + :withdrawal ::txn/withdrawal + :deposit ::txn/deposit } (::txn-type txn))) (defn merge-match-order [spend receive common] - (match (map ::common/currency [spend receive]) + (match (map ::txn/currency [spend receive]) [:usd _] [(merge common { - ::common/txn-type ::common/buy - ::common/usd-value (::common/quantity-transacted spend) - ::common/quantity-transacted (::common/quantity-transacted receive) - ::common/currency (::common/currency receive) + ::txn/txn-type ::txn/buy + ::txn/usd-value (::txn/quantity-transacted spend) + ::txn/quantity-transacted (::txn/quantity-transacted receive) + ::txn/currency (::txn/currency receive) })] [_ :usd] [(merge common { - ::common/txn-type ::common/sell - ::common/usd-value (::common/quantity-transacted receive) - ::common/quantity-transacted (::common/quantity-transacted spend) - ::common/currency (::common/currency spend) + ::txn/txn-type ::txn/sell + ::txn/usd-value (::txn/quantity-transacted receive) + ::txn/quantity-transacted (::txn/quantity-transacted spend) + ::txn/currency (::txn/currency spend) })] :else [(merge common { - ::common/txn-type ::common/sell - ::common/usd-value :deferred - ::common/quantity-transacted (::common/quantity-transacted spend) - ::common/currency (::common/currency spend) + ::txn/txn-type ::txn/sell + ::txn/usd-value :deferred + ::txn/quantity-transacted (::txn/quantity-transacted spend) + ::txn/currency (::txn/currency spend) }) (merge common { - ::common/txn-type ::common/buy - ::common/usd-value :deferred - ::common/quantity-transacted (::common/quantity-transacted receive) - ::common/currency (::common/currency receive) + ::txn/txn-type ::txn/buy + ::txn/usd-value :deferred + ::txn/quantity-transacted (::txn/quantity-transacted receive) + ::txn/currency (::txn/currency receive) })])) (def process-transfer (common/flatcomp - (common/add-field ::common/txn-type coinbase-pro-txn->common-txn))) + (common/add-field ::txn/txn-type coinbase-pro-txn->common-txn))) (defn process-match-order [[trade-id elems]] (let [txn0 (first elems) fee? (*-> ::txn-type (= :fee)) - spend? (and-> (*-> ::common/quantity-transacted neg?) (*-> ::txn-type (= :match))) - receive? (and-> (comp not neg? ::common/quantity-transacted) (*-> ::txn-type (= :match))) + spend? (and-> (*-> ::txn/quantity-transacted neg?) (*-> ::txn-type (= :match))) + receive? (and-> (comp not neg? ::txn/quantity-transacted) (*-> ::txn-type (= :match))) common-fields (select-keys txn0 [::order-id ::trade-id - ::common/account - ::common/timestamp + ::txn/account + ::txn/timestamp ::transfer-id ::balance ::portfolio])] @@ -93,9 +95,9 @@ (find-first receive? elems) common-fields) (map (fn [txn] - (-> (select-keys txn [::common/quantity-transacted ::common/currency]) + (-> (select-keys txn [::txn/quantity-transacted ::txn/currency]) (merge common-fields) - (assoc ::common/txn-type ::common/fee))) + (assoc ::txn/txn-type ::txn/fee))) (filter fee? elems))))) (defn group-transactions [txns] @@ -107,13 +109,13 @@ (def preclean-txns (common/flatcomp (common/rename-field :amount - ::common/quantity-transacted + ::txn/quantity-transacted bigdec) (common/rename-field :amount/balance-unit - ::common/currency + ::txn/currency common/to-keyword) (common/rename-field :time - ::common/timestamp + ::txn/timestamp (common/date-parser "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")) (common/rename-field :trade-id ::trade-id @@ -137,7 +139,7 @@ (defn load-csv [filename] (->> (common/load-csv filename) (common/pivot-csv) - (map #(assoc % ::common/account :coinbase-pro)) + (map #(assoc % ::txn/account :coinbase-pro)) (common/flatmap preclean-txns) (group-transactions) - (common/flatmap (common/add-txid ::common/txid)))) + (common/flatmap (common/add-txid ::txn/txid)))) diff --git a/src/worther/injest/common.clj b/src/worther/injest/common.clj index dd28318..3b75448 100644 --- a/src/worther/injest/common.clj +++ b/src/worther/injest/common.clj @@ -3,7 +3,9 @@ [clojure.java.io :as io] [clojure.string :as str] [clojure.spec.alpha :as s] - [valuehash.api :as hash])) + [valuehash.api :as hash] + + [worther.txn :as txn])) (s/def ::header keyword?) @@ -54,11 +56,11 @@ (defn generate-id [entry] ;; Use a vector because I'm not sure the map will be ordered - (let [make-entry-vector (juxt ::timestamp - ::quantity-transacted - ::txn-type - ::account - ::currency)] + (let [make-entry-vector (juxt :txn/timestamp + :txn/quantity-transacted + :txn/txn-type + :txn/account + :txn/currency)] (-> entry (make-entry-vector) (hash/sha-256-str)))) @@ -70,7 +72,7 @@ ([field new-field] (rename-field field new-field identity)) ([field new-field f] (fn [entry] (when (not (contains? entry field)) - (throw (RuntimeException. (str "Field not found: " field)))) + (throw (RuntimeException. (str "Field not found: " field " in " entry)))) [(assoc (dissoc entry field) new-field (f (get entry field)))]))) @@ -95,60 +97,22 @@ (defn zero-or [f] (fn [str] (if (or (nil? str) (empty? str)) (bigdec 0) (f str)))) -(s/def ::txn-type #{::buy ::sell ::send ::receive ::deposit ::withdrawal ::fee ::income}) - -(defn valid-timestamp? [ts] (instance? java.util.Date ts)) - -(s/def ::timestamp valid-timestamp?) - -(defn bigdec? [i] (instance? java.math.BigDecimal i)) - -(s/def ::quantity-transacted bigdec?) - -(s/def ::usd-value (s/or :deferred #(= % :deferred) - :number bigdec?)) - -(s/def ::currency keyword?) - -(s/def ::account keyword?) - -(s/def ::sender keyword?) - -(s/def ::recipient keyword?) - -(defmulti txn ::txn-type) -(s/def ::txn (s/multi-spec txn ::txn-type)) - -(let [txn-common-req [::txid - ::quantity-transacted - ::currency - ::txn-type - ::account - ::timestamp]] - (defmethod txn ::buy [_] - (s/keys :req (concat txn-common-req [::usd-value]))) - (defmethod txn ::sell [_] - (s/keys :req (concat txn-common-req [::usd-value]))) - (defmethod txn ::send [_] - (s/keys :req (concat txn-common-req [::recipient]))) - (defmethod txn ::receive [_] - (s/keys :req (concat txn-common-req [::sender]))) - (defmethod txn ::fee [_] - (s/keys :req (concat txn-common-req))) - (defmethod txn ::income [_] - (s/keys :req (concat txn-common-req [::src]))) - (defmethod txn ::deposit [_] - (s/keys :req (concat txn-common-req [::src]))) - (defmethod txn ::withdrawal [_] - (s/keys :req (concat txn-common-req [::dest])))) - (defn map-select [m] (fn [om] (every? (fn [key] (= (get om key) (get m key))) (keys m)))) +(defn add-tag [tag src obj] + (with-meta obj { tag src })) + +(def add-source-tag (partial add-tag :source)) + (defn matching-files [rx dir] (filter (fn [file] (re-matches rx file)) (map str (file-seq (clojure.java.io/file dir))))) + +(defn pprint-str [obj] + (with-out-str (clojure.pprint/pprint obj)) + obj) diff --git a/src/worther/originator.clj b/src/worther/originator.clj new file mode 100644 index 0000000..206f46d --- /dev/null +++ b/src/worther/originator.clj @@ -0,0 +1,158 @@ +(ns worther.originator + (:require [worther.txn :as txn] + [clojure.spec.alpha :as s])) + +;; Think this through. +;; +;; - The input will be a list of transactions. Step one: extract the sales. +;; - Order chronologically. These need mapping to purchases. Step two: extract +;; the purchases. Order chronologically, split by currency. Step three: +;; starting with the first sale, start consuming purchases until the total of +;; the sale is covered. +;; - Return: [ filled sales, remaining sales, remaining purchases ]. +;; - In the end, return only filled sales. + +(defn- positive? [n] (>= 0)) + +(s/def ::remaining (comp positive? number?)) +(s/def ::contribution-amount (comp positive? number?)) + +(s/def ::contributions (s/coll-of (s/keys :req [::amount ::txn]))) + +(defn- sort-by-date [txns] + (sort (fn [a b] (.before a b)) txns)) + +(defn- normalize [i] (.setScale (bigdec i) 6 java.math.RoundingMode/HALF_UP)) + +(defn- txn-sourced? [txn] + (let [sum-coll (apply +) + src-amount (-> txn + ::contributions + (map ::contribution-amount) + (sum-coll))] + (.equals (normalize (::amount txn)) + (normalize src-amount)))) + +(defn- txn-consumed? [txn] + (.equals (normalize 0) (-> txn ::remaining normalize))) + +(s/def ::contributions + (s/keys :req [::src-txn ::contribution-amount])) + +(s/def ::sourced-txn + (s/and txn-sourced? + ::sink-txn + (s/keys :req [::contributions]))) + +(defn originate-txn [sink srcs] + (let [src (first srcs)] + (if (>= (::remaining src) (::remaining sink)) + ;; Sink is fully covered + (let [contribution (::remaining sink)] + [(-> sink + (update ::remaining 0) + (update ::contributions conj + { + ::contribution-amount contribution + ::src-txn src + })) + (conj (rest srcs) + (update src ::remaining - contribution))]) + ;; Sink is not yet fully covered + (let [contribution (::remaining src)] + (originate-txn (-> sink + (update ::remaining - contribution) + (update ::contributions conj + { + ::contribuution-amount contribution + ::src-txn src + })) + (rest srcs)))))) +(s/fdef originate-txn + :args (s/cat :sink ::sink-txn + :srcs (s/coll-of ::src-txn)) + :ret ::sourced-txn) + +(s/def ::txn-with-remaining + (s/and :txn/txn + (s/keys :req [::remaining]))) + +(defn- originate-txns-exec + "Given a sorted list of sink transactions and a sorted list of src transactions, + return a list of sinks mapped to originating src transactions." + [sinks srcs] + (loop [sinks sinks + srcs srcs + filled-sinks []] + (if (empty? sinks) + filled-sinks + (let [[filled-sink remaining-srcs] + (originate-txn (first sinks) srcs)] + (recur (rest sinks) + remaining-srcs + (conj filled-sinks filled-sink)))))) +(s/fdef originate-txns-exec + :args (s/cat :sinks (s/coll-of (s/and :txn/sink-txn ::txn-with-remaining)) + :srcs (s/coll-of (s/and :txn/src-txn ::txn-with-remaining))) + :ret (s/coll-of ::sourced-txn)) + +(defmulti inject-remaining + "Add a ::remaining key, containing the unallocated amount." + :txn/txn-type) +(s/fdef inject-remaining + :args (s/cat :txn :txn/txn) + :ret ::txn-with-remaining) + +(defn- remaining-from [txn key] + (assoc txn ::remaining (get txn key))) +(s/fdef remaining-from + :args (s/cat :txn :txn/txn + :key keyword?) + :ret ::txn-with-remaining) + +(defmethod inject-remaining :default [txn] + (remaining-from txn :txn/amount)) + +;; In most cases, simple: map every sink to sources for every account. But in +;; the case of a receive source, we need to look at the original account and +;; start finding sources there. They can't be shared. + +;; TODO: inject ::remaining +#_(defn originate-txn [all-txns] + (let [srcs (group-by ::txn/account (filter txn/src-txn? all-txns)) + sinks (filter txn/sink-txn? all-txns)] + { + :filled-sales [] + :remaining-sales remaining-sales + :remaining-buys remaining-buys + })) + +(defn- consume-src [sink src] + (let [contribution (apply min (map ::remaining [sink src]))] + [(-> sink + (update ::remaining - contribution) + (update ::contributions conj + { + ::contribution-amount contribution + ::src-txn src + })) + (update src ::remaining - contribution)])) +(s/fdef consume-src + :args (s/cat :sink ::txn/sink-txn + :src ::txn/src-txn) + :ret (s/cat :sink ::txn/sink-txn + :src ::txn/src-txn)) + +(s/def ::srcs-by-account (s/map-of ::txn/account (s/coll-of ::txn/src-txn))) +(defn originate-txn [sink srcs] + (let [sink-acct (::txn/account sink) + acct-srcs (get srcs sink-acct)] + (loop []))) +(s/fdef originate-txn + :args (s/cat :sink ::txn/sink-txn + :srcs ::srcs-by-account) + :ret (s/cat :sink ::sourced-txn + :srcs ::srcs-by-account)) + +(defn originate-txns [all-txns] + (group-by ::txn/account all-txns)) diff --git a/src/worther/txn.clj b/src/worther/txn.clj new file mode 100644 index 0000000..fa3218d --- /dev/null +++ b/src/worther/txn.clj @@ -0,0 +1,119 @@ +(ns worther.txn + (:require [clojure.pprint :refer [pprint]] + [clojure.spec.alpha :as s] + [clojure.set :refer [union]] + [worther.txn :as txn])) + +(def src-txn-types + #{ + ::buy + ::receive + ::deposit + ::income + }) + +(def sink-txn-types + #{ + ::sell + ::send + ::withdrawal + ::fee + }) + +(s/def ::txn-type (union src-txn-types sink-txn-types)) +(s/def ::src-txn-type src-txn-types) +(s/def ::sink-txn-type sink-txn-types) + +(defn src-txn? [txn] (->> txn ::txn-type (contains? src-txn-types))) +(defn sink-txn? [txn] (->> txn ::txn-type (contains? sink-txn-types))) + +(defn valid-timestamp? [ts] (instance? java.util.Date ts)) + +(s/def ::timestamp valid-timestamp?) + +(defn bigdec? [i] (instance? java.math.BigDecimal i)) + +(s/def ::quantity-transacted bigdec?) + +(s/def ::usd-value (s/or :deferred #(= % :deferred) + :number bigdec?)) + +(s/def ::currency keyword?) + +(s/def ::account keyword?) + +(s/def ::sender keyword?) + +(s/def ::recipient keyword?) + +(defmulti txn ::txn-type) + +(s/def ::txn (s/multi-spec txn ::txn-type)) + +(defn src-txn? [txn] + (->> txn + ::txn-type + (contains? src-txn-types))) + +(defn sink-txn? [txn] + (->> txn ::txn-type (contains? sink-txn-types))) + +(s/def ::src-txn src-txn?) +(s/def ::sink-txn sink-txn?) + +(defn- pprint-to-string [o] + (with-out-str (pprint o))) + +(defn- pthru [obj] + (clojure.pprint/pprint obj) + obj) + +(defn- negative? [key] + (fn [txn] (-> txn key (<= 0)))) + +(defn- positive? [key] + (fn [txn] (-> txn key (>= 0)))) + +(let [txn-common-req [::txid + ::quantity-transacted + ::currency + ::txn-type + ::account + ::timestamp]] + (defmethod txn ::buy [_] + (s/and (s/keys :req (concat txn-common-req [::usd-value])) + ;; deferred + ;;(negative? ::usd-value) + (positive? ::quantity-transacted))) + (defmethod txn ::sell [_] + (s/and (s/keys :req (concat txn-common-req [::usd-value])) + (negative? ::quantity-transacted) + ;; Deferred + ;;(positive? ::usd-value) + )) + (defmethod txn ::send [_] + (s/and (s/keys :req (concat txn-common-req [::recipient])) + (negative? ::quantity-transacted))) + (defmethod txn ::receive [_] + (s/and (s/keys :req (concat txn-common-req [::sender])) + (positive? ::quantity-transacted))) + (defmethod txn ::fee [_] + (s/and (s/keys :req (concat txn-common-req)) + ;; Deferred + ;;(negative? ::usd-value) + )) + (defmethod txn ::income [_] + (s/and (s/keys :req (concat txn-common-req [::src])) + (positive? ::quantity-transacted))) + (defmethod txn ::deposit [_] + (s/and (s/keys :req (concat txn-common-req [::src])) + (positive? ::quantity-transacted))) + (defmethod txn ::withdrawal [_] + (s/and (s/keys :req (concat txn-common-req [::dest])) + (negative? ::quantity-transacted))) + (defmethod txn :default [txn] + (throw (RuntimeException. + (str "No method defined for txn-type: " + (::txn-type txn) + "\n" + (pprint-to-string txn))))))