niten
/
worther
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updates...

This commit is contained in:
niten 2024-03-23 11:23:14 -07:00
parent 9c6b0acaba
commit 11107baa6a
10 changed files with 778 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
deps.edn
View File

@ -1,6 +1,9 @@
{:paths ["src"]
:deps
{org.clojure/data.csv {:mvn/version "1.0.0"}
{org.clojure/clojure {:mvn/version "1.11.0"}
org.clojure/data.csv {:mvn/version "1.0.1"}
org.clojure/core.async {:mvn/version "1.5.648"}
org.clojure/core.match {:mvn/version "1.0.0"}
org.clojure/tools.cli {:mvn/version "1.0.206"}
arachne-framework/valuehash {:git/url "https://git.fudo.org/fudo-public/valuehash.git"
:sha "9d2dbafdb5db886a57f44c5b7fe32c824713e6c7"}}}

269
deps.nix
View File

@ -1,82 +1,54 @@
# generated by clj2nix-1.0.7
{ pkgs ? import <nixpkgs> {} }:
# generated by clj2nix-1.1.0-rc
{ fetchMavenArtifact, fetchgit, lib }:
let repos = [
"https://repo1.maven.org/maven2/"
"https://repo.clojars.org/" ];
in rec {
fetchmaven = pkgs.callPackage (pkgs.fetchurl {
url = "https://raw.githubusercontent.com/NixOS/nixpkgs/ba5e2222458a52357a3ba5873d88779d5c223269/pkgs/build-support/fetchmavenartifact/default.nix";
sha512 = "05m7i8hbhyfz7p2f106mfbsasjf04svd9xkgc26pl3shljrk0dfacz39wiwzm6xqw7czgrsx745vciram7al621v7634nfdq3m1x88a";
}) {};
makePaths = {extraClasspaths ? null}:
(pkgs.lib.concatMap
(dep:
builtins.map
(path:
if builtins.isString path then
path
else if builtins.hasAttr "jar" path then
path.jar
else if builtins.hasAttr "outPath" path then
path.outPath
else
path
)
dep.paths)
packages)
++ (if extraClasspaths != null then [ extraClasspaths ] else []);
makeClasspaths = {extraClasspaths ? null}: builtins.concatStringsSep ":" (makePaths {inherit extraClasspaths;});
makePaths = {extraClasspaths ? []}:
if (builtins.typeOf extraClasspaths != "list")
then builtins.throw "extraClasspaths must be of type 'list'!"
else (lib.concatMap (dep:
builtins.map (path:
if builtins.isString path then
path
else if builtins.hasAttr "jar" path then
path.jar
else if builtins.hasAttr "outPath" path then
path.outPath
else
path
)
dep.paths)
packages) ++ extraClasspaths;
makeClasspaths = {extraClasspaths ? []}:
if (builtins.typeOf extraClasspaths != "list")
then builtins.throw "extraClasspaths must be of type 'list'!"
else builtins.concatStringsSep ":" (makePaths {inherit extraClasspaths;});
packageSources = builtins.map (dep: dep.src) packages;
packages = [
rec {
name = "data.csv/org.clojure";
src = fetchmaven {
inherit repos;
artifactId = "data.csv";
groupId = "org.clojure";
sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b";
version = "1.0.0";
};
paths = [ src ];
}
rec {
name = "core.match/org.clojure";
src = fetchmaven {
inherit repos;
artifactId = "core.match";
groupId = "org.clojure";
sha512 = "52ada3bbe73ed1b429be811d3990df0cdb3e9d50f2a6c92b70d490a8ea922d4794da93c3b7487653f801954fc599704599b318b4d7926a9594583df37c55e926";
version = "1.0.0";
};
paths = [ src ];
}
(rec {
name = "arachne-framework/valuehash";
src = pkgs.fetchgit {
name = "valuehash";
url = "https://git.fudo.org/fudo-public/valuehash.git";
rev = "9d2dbafdb5db886a57f44c5b7fe32c824713e6c7";
sha256 = "1civ393c4yy9p2xbmrrvpbyqczx55k3fkvimkf850fl62ns5zl9r";
};
paths = map (path: src + path) [
"/src"
];
})
rec {
name = "clojure/org.clojure";
src = fetchmaven {
src = fetchMavenArtifact {
inherit repos;
artifactId = "clojure";
groupId = "org.clojure";
sha512 = "d9e2c0676cdc349a3455d92b3ce3c3f01a2410de448c9416edfe72bc7eaf356cfadbb6d746740a821940c3b4cab100ca941e23bab482e98b404ed9ef79c562df";
version = "1.10.0-alpha4";
sha512 = "efb87dfd347d2be6cb251d550e312d77797e35500b75ebe8e3fca824d16223803305ce89d4ae0349e5dff22a99c24b8719bf791f24685a12404bd56a44693010";
version = "1.11.0";
};
paths = [ src ];
}
rec {
name = "tools.analyzer/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "tools.analyzer";
groupId = "org.clojure";
sha512 = "c51752a714848247b05c6f98b54276b4fe8fd44b3d970070b0f30cd755ac6656030fd8943a1ffd08279af8eeff160365be47791e48f05ac9cc2488b6e2dfe504";
version = "1.1.0";
};
paths = [ src ];
@ -84,7 +56,7 @@ let repos = [
rec {
name = "tools.logging/org.clojure";
src = fetchmaven {
src = fetchMavenArtifact {
inherit repos;
artifactId = "tools.logging";
groupId = "org.clojure";
@ -96,26 +68,169 @@ let repos = [
}
rec {
name = "spec.alpha/org.clojure";
src = fetchmaven {
name = "core.specs.alpha/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "spec.alpha";
artifactId = "core.specs.alpha";
groupId = "org.clojure";
sha512 = "b8fc40ed9bc52b545e699ed188dd61bfd144ee67f0c70364b8f2715e9f1fea608d3721db7f618f6ef4bc3056e3c2984c626080486ca710f3595dda8ba23730ac";
version = "0.1.143";
sha512 = "f521f95b362a47bb35f7c85528c34537f905fb3dd24f2284201e445635a0df701b35d8419d53c6507cc78d3717c1f83cda35ea4c82abd8943cd2ab3de3fcad70";
version = "0.2.62";
};
paths = [ src ];
}
rec {
name = "core.specs.alpha/org.clojure";
src = fetchmaven {
name = "spec.alpha/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "core.specs.alpha";
artifactId = "spec.alpha";
groupId = "org.clojure";
sha512 = "b4f5eee01da39914e6024dd529d1f72952d5a9dae65e1e41bf386b1e86a004a0d197b5be95aa70e7e8d6438c92b7fa8fc0c5039f2013e97c0b91c22d86fb7968";
version = "0.1.24";
sha512 = "ddfe4fa84622abd8ac56e2aa565a56e6bdc0bf330f377ff3e269ddc241bb9dbcac332c13502dfd4c09c2c08fe24d8d2e8cf3d04a1bc819ca5657b4e41feaa7c2";
version = "0.3.218";
};
paths = [ src ];
}
rec {
name = "tools.cli/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "tools.cli";
groupId = "org.clojure";
sha512 = "1d88aa03eb6a664bf2c0ce22c45e7296d54d716e29b11904115be80ea1661623cf3e81fc222d164047058239010eb678af92ffedc7c3006475cceb59f3b21265";
version = "1.0.206";
};
paths = [ src ];
}
rec {
name = "tools.analyzer.jvm/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "tools.analyzer.jvm";
groupId = "org.clojure";
sha512 = "36ad50a7a79c47dea16032fc4b927bd7b56b8bedcbd20cc9c1b9c85edede3a455369b8806509b56a48457dcd32e1f708f74228bce2b4492bd6ff6fc4f1219d56";
version = "1.2.2";
};
paths = [ src ];
}
rec {
name = "asm/org.ow2.asm";
src = fetchMavenArtifact {
inherit repos;
artifactId = "asm";
groupId = "org.ow2.asm";
sha512 = "876eac7406e60ab8b9bd6cd3c221960eaa53febea176a88ae02f4fa92dbcfe80a3c764ba390d96b909c87269a30a69b1ee037a4c642c2f535df4ea2e0dd499f2";
version = "9.2";
};
paths = [ src ];
}
rec {
name = "data.csv/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "data.csv";
groupId = "org.clojure";
sha512 = "6b667a56cbb6632a90564f217f9c28a6670c13c729fb205ced091d9ee006382143dd6b615dd5a4f900660946199cac449fe9fabc90820bb34b92a9e6c8550473";
version = "1.0.1";
};
paths = [ src ];
}
rec {
name = "core.match/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "core.match";
groupId = "org.clojure";
sha512 = "52ada3bbe73ed1b429be811d3990df0cdb3e9d50f2a6c92b70d490a8ea922d4794da93c3b7487653f801954fc599704599b318b4d7926a9594583df37c55e926";
version = "1.0.0";
};
paths = [ src ];
}
rec {
name = "tools.reader/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "tools.reader";
groupId = "org.clojure";
sha512 = "3481259c7a1eac719db2921e60173686726a0c2b65879d51a64d516a37f6120db8ffbb74b8bd273404285d7b25143ab5c7ced37e7c0eaf4ab1e44586ccd3c651";
version = "1.3.6";
};
paths = [ src ];
}
rec {
name = "core.memoize/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "core.memoize";
groupId = "org.clojure";
sha512 = "67196537084b7cc34a01454d2a3b72de3fddce081b72d7a6dc1592d269a6c2728b79630bd2d52c1bf2d2f903c12add6f23df954c02ef8237f240d7394ccc3dde";
version = "1.0.253";
};
paths = [ src ];
}
rec {
name = "data.priority-map/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "data.priority-map";
groupId = "org.clojure";
sha512 = "bb8bc5dbfd3738c36b99a51880ac3f1381d6564e67601549ef5e7ae2b900e53cdcdfb8d0fa4bf32fb8ebc4de89d954bfa3ab7e8a1122bc34ee5073c7c707ac13";
version = "1.1.0";
};
paths = [ src ];
}
(rec {
name = "arachne-framework/valuehash";
src = fetchgit {
name = "valuehash";
url = "https://git.fudo.org/fudo-public/valuehash.git";
rev = "9d2dbafdb5db886a57f44c5b7fe32c824713e6c7";
sha256 = "1civ393c4yy9p2xbmrrvpbyqczx55k3fkvimkf850fl62ns5zl9r";
};
paths = map (path: src + path) [
"/src"
];
})
rec {
name = "core.cache/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "core.cache";
groupId = "org.clojure";
sha512 = "0a07ceffc2fa3a536b23773eefc7ef5e1108913b93c3a5416116a6566de76dd5c218f3fb0cc19415cbaa8843838de310b76282f20bf1fc3467006c9ec373667e";
version = "1.0.225";
};
paths = [ src ];
}
rec {
name = "core.async/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "core.async";
groupId = "org.clojure";
sha512 = "160a77da25382d7c257eee56cfe83538620576a331e025a2d672fc26d9f04e606666032395f3c2e26247c782544816a5862348f3a921b1ffffcd309c62ac64f5";
version = "1.5.648";
};
paths = [ src ];

15
flake.nix
View File

@ -16,14 +16,15 @@
# defaultPackage = packages.${system}.worther;
# };
outputs = { self, nixpkgs, flake-utils, ... }: with nixpkgs.lib;
outputs = { self, nixpkgs, flake-utils, ... }:
with nixpkgs.lib;
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
worther = pkgs.callPackage ./worther.nix { pkgs = pkgs; };
let pkgs = import nixpkgs { inherit system; };
in rec {
packages.worther = worther;
defaultPackage = self.packages.${system}.worther;
in {
packages = rec {
default = worther;
worther = pkgs.callPackage ./worther.nix { pkgs = pkgs; };
};
});
}

102
src/worther/core.clj
View File

@ -1,5 +1,101 @@
(ns worther.core)
(ns worther.core
(:require [worther.injest.bittrex :as bittrex]
[worther.injest.coinbase :as cb]
[worther.injest.coinbase-pro :as cb-pro]
[worther.txn :as txn]
[clojure.pprint :refer [pprint]]
[clojure.string :as str]
[clojure.tools.cli :refer [parse-opts]])
(:gen-class))
(defn- ensure-directory [dir]
(if (not (.isDirectory dir))
(throw (java.io.FileNotFoundException.
(str "Invalid directory: " dir)))
dir))
(defn- directory-contents [dir-str]
(-> dir-str clojure.java.io/file ensure-directory file-seq))
(defn- has-ext? [ext]
(fn [file]
(re-matches (re-pattern (str ".+" "\\." ext "$"))
(str file))))
(defn- pthru [obj]
(pprint obj)
obj)
(defn- load-dir-csvs [loader dir]
(mapcat loader
(filter (has-ext? "csv")
(directory-contents dir))))
(def bittrex-load (partial load-dir-csvs bittrex/load-csv))
(def coinbase-load (partial load-dir-csvs cb/load-csv))
(def coinbase-pro-load (partial load-dir-csvs cb-pro/load-csv))
(defn- directory? [dir]
(-> dir clojure.java.io/file .isDirectory))
(defn- valid-year? [year]
(< 2010 year 2100))
(def cli-options
[["-h" "--help"]
["-c" "--coinbase DIR" "Coinbase CSV directory"
:default nil
:parse-fn clojure.java.io/file
:validate [directory? "Must be a valid directory."]]
["-C" "--coinbase-pro DIR" "Coinbase Pro CSV directory"
:default nil
:parse-fn clojure.java.io/file
:validate [directory? "Must be a valid directory."]]
["-b" "--bittrex DIR" "Bittrex CSV directory"
:default nil
:parse-fn clojure.java.io/file
:validate [directory? "Must be a valid directory."]]
["-y" "--year YEAR" "Year for which to generate tax data."
:parse-fn #(Integer/parseInt %)
:validate [valid-year? "Must be a year between 2010 and 2100."]]])
(defn- print-usage [opts]
(->> ["usage: worther [options]" "" "options:" opts]
(str/join \newline)))
(defn- error? [status]
(not (= 0 status)))
(defn- error-quit [status msg]
(if (error? status)
(println (str "error: " msg))
(println msg))
(System/exit status))
(def join-lines (partial str/join \newline))
(defmacro launch-task [name body]
(let [res (gensym)]
`(do (println (str "beginning " ~name "..."))
(let [~res ~body]
(println (str "completed " ~name "."))
~res))))
(defn- load-data [opts]
(let [before (fn [a b] (.before a b))]
(sort-by ::txn/timestamp
before
(concat (or (some-> opts :coinbase coinbase-load) [])
(or (some-> opts :coinbase-pro coinbase-pro-load) [])
(or (some-> opts :bittrex bittrex-load) [])))))
(defn -main [& args]
(println "Not implemented yet!")
(println (str "called with: " args)))
(let [opts (parse-opts args cli-options)]
(cond (-> opts :options :help) (->> opts :summary print-usage (error-quit 0))
(-> opts :errors empty? not) (->> opts :errors join-lines (error-quit 1))
:else (println (str "count: " (count (load-data (:options opts))))))))

50
src/worther/injest/bittrex.clj
View File

@ -1,30 +1,32 @@
(ns worther.injest.bittrex
(:require [worther.injest.common :as common]
[worther.txn :as txn]
[clojure.core.match :refer [match]]
[clojure.spec.alpha :as s]
[clojure.string :as str]))
(defn split-buy [txn]
[(-> txn
(assoc ::common/txn-type ::common/buy
::common/quantity-transacted (::quantity txn)
::common/currency (::market-currency txn)))
(assoc ::txn/txn-type ::txn/buy
::txn/quantity-transacted (::quantity txn)
::txn/currency (::market-currency txn)))
(-> txn
(assoc ::common/txn-type ::common/sell
::common/quantity-transacted (- (* (::price-per txn)
(assoc ::txn/txn-type ::txn/sell
::txn/quantity-transacted (- (* (::price-per txn)
(::quantity txn)))
::common/currency (::base-currency txn)))])
::txn/currency (::base-currency txn)))])
(defn split-sell [txn]
[(-> txn
(assoc ::common/txn-type ::common/sell
::common/quantity-transacted (- (::quantity txn))
::common/currency (::market-currency txn)))
(assoc ::txn/txn-type ::txn/sell
::txn/quantity-transacted (- (::quantity txn))
::txn/currency (::market-currency txn)))
(-> txn
(assoc ::common/txn-type ::common/buy
::common/quantity-transacted (* (::price-per txn)
(assoc ::txn/txn-type ::txn/buy
::txn/quantity-transacted (* (::price-per txn)
(::quantity txn))
::common/currency (::base-currency txn)))])
::txn/currency (::base-currency txn)))])
(defmulti type-specific-translation ::txn-type)
@ -41,10 +43,10 @@
(split-buy txn))
(defmethod type-specific-translation :fee [txn]
[(assoc txn ::common/txn-type ::common/fee)])
[(assoc txn ::txn/txn-type ::txn/fee)])
(defmethod type-specific-translation nil [txn]
(throw (RuntimeException. (str "::txn-type missing from transaction: " txn))))
(throw (RuntimeException. (str "::txn-type missing from transaction: " (common/pprint-str txn)))))
(defn to-uuid [str] (java.util.UUID/fromString str))
@ -52,11 +54,11 @@
[txn
{
::txn-type :fee
::common/timestamp (::common/timestamp txn)
::common/account (::common/account txn)
::common/usd-value :deferred
::common/quantity-transacted (::quantity txn)
::common/currency (::base-currency txn)
::txn/timestamp (::txn/timestamp txn)
::txn/account (::txn/account txn)
::txn/usd-value :deferred
::txn/quantity-transacted (::quantity txn)
::txn/currency (::base-currency txn)
}])
(defn split-currencies [txn]
@ -74,7 +76,7 @@
split-currencies
(common/alter-field :uuid to-uuid)
(common/rename-field :timestamp
::common/timestamp
::txn/timestamp
(common/date-parser "M/d/yyyy HH:mm:ss aa"))
(common/rename-field :ordertype
::txn-type
@ -97,15 +99,17 @@
(common/rename-field :closed
::closed
(common/date-parser "M/d/yyyy HH:mm:ss aa"))
(common/add-field ::common/account
(common/add-field ::txn/account
(fn [_] :bittrex))))
(defn load-csv [filename]
(let [txn-translator (common/flatcomp
(common/add-txid ::common/txid)
(common/add-txid ::txn/txid)
type-specific-translation
translate-bittrex-txn)]
(->> (common/load-csv filename)
(common/pivot-csv)
(common/flatmap txn-translator))))
(s/fdef load-csv
:args (s/cat :filename string?)
:ret (s/coll-of ::txn/txn))

195
src/worther/injest/coinbase.clj
View File

@ -1,6 +1,9 @@
(ns worther.injest.coinbase
(:require [worther.injest.common :as common]
[clojure.spec.alpha :as s]))
[worther.txn :as txn]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[clojure.pprint :refer [pprint]]))
(defn print-through [o]
(clojure.pprint/pprint o)
@ -13,90 +16,115 @@
(def txn-type-map
{
:fee ::common/fee
:buy ::common/buy
:sell ::common/sell
:send ::common/send
:receive ::common/receive
:paid-for-an-order ::common/sell
:fee ::txn/fee
:buy ::txn/buy
:sell ::txn/sell
:send ::txn/send
:receive ::txn/receive
:paid-for-an-order ::txn/sell
:cardspend ::txn/sell
:cardbuyback ::txn/buy
:rewards-income ::txn/income
:convert ::txn/sell
})
(s/def ::txn-type (set (keys txn-type-map)))
(defn update-txn-type [txn]
(assoc txn ::common/txn-type
(assoc txn ::txn/txn-type
(get txn-type-map (::txn-type txn))))
(defmulti type-specific-translation ::txn-type)
(s/fdef type-specific-translation
:args (s/cat :txn ::coinbase-txn)
:ret (s/+ ::txn/txn))
(defn split-fee [txn]
(if (and (::usd-fees txn)
(> (::usd-fees txn) 0))
(if (> (::usd-fees txn) 0)
[(update-txn-type txn)
{
::txn-type :fee
::common/quantity-transacted (::usd-fees txn)
::common/currency :usd
::common/account :coinbase
::common/timestamp (::common/timestamp txn)
::common/notes (str "Fee for: " (::common/notes txn))
::common/usd-value (::usd-fees txn)
::txn-type :fee
::txn/txn-type ::txn/fee
::txn/quantity-transacted (::usd-fees txn)
::txn/currency :usd
::txn/account :coinbase
::txn/timestamp (::txn/timestamp txn)
::txn/notes (str "Fee for: " (::txn/notes txn))
::txn/usd-value (::usd-fees txn)
}]
[(update-txn-type txn)]))
(defmethod type-specific-translation :buy [txn]
[(update-txn-type txn)
{
::common/txn-type ::common/deposit
::common/quantity-transacted (::common/usd-value txn)
::common/currency :usd
::common/account :coinbase
::common/timestamp (::common/timestamp txn)
::common/notes (str "Deposit for: " (::common/notes txn))
::common/usd-value (::common/usd-value txn)
}])
(defmethod type-specific-translation :sell [txn]
[(update (update-txn-type txn) ::common/quantity-transacted -)
{
::common/txn-type ::common/withdrawal
::common/quantity-transacted (::common/usd-value txn)
::common/currency :usd
::common/account :coinbase
::common/timestamp (::common/timestamp txn)
::common/notes (str "Withdrawal for: " (::common/notes txn))
::common/usd-value (::common/usd-value txn)
}])
(defn- extract-conversion-data [conv-str]
(let [read-amount (fn [amt]
(-> amt
(str/replace #"," "")
(bigdec)))
number-rx "[0-9]+(,[0-9]+)*\\.[0-9]+"
currency-rx "[A-Z][A-Z0-9]{2,5}"
conversion-rx (re-pattern (str "^Converted "
number-rx
" "
currency-rx
" to (?<amt>"
number-rx
") (?<currency>"
currency-rx
")$"))
matches (re-matches conversion-rx conv-str)
amt (-> matches (nth 2) read-amount)
currency (-> matches (nth 4) common/to-keyword)]
[amt currency]))
(defmethod type-specific-translation :paid-for-an-order [txn]
(let [usd-value (* (::common/usd-current-price txn)
(::common/quantity-transacted txn))]
[(-> (update-txn-type txn)
(assoc ::common/usd-value (- usd-value)))
(defmethod type-specific-translation :convert [txn]
(let [[qty currency] (-> txn ::txn/notes extract-conversion-data)]
[(-> txn
update-txn-type
(update ::txn/quantity-transacted (comp - abs)))
{
::common/txn-type ::common/send
::common/quantity-transacted usd-value
::common/currency :usd
::common/account :coinbase
::common/timestamp (::common/timestamp txn)
::common/notes (str "Send to merchant for: " (::common/notes txn))
::common/usd-value usd-value
::common/recipient :merchant
::txn-type :buy
::txn/txn-type ::txn/buy
::txn/quantity-transacted qty
::txn/currency currency
::txn/account :coinbase
::txn/timestamp (::txn/timestamp txn)
::txn/notes (str "Converted from " (::txn/currency txn) " to " currency)
::txn/usd-value :deferred
}]))
(defmethod type-specific-translation :buy [txn]
[(-> txn update-txn-type)])
(defmethod type-specific-translation :cardbuyback [txn]
[(-> txn update-txn-type)])
(defmethod type-specific-translation :sell [txn]
[(-> txn update-txn-type
(update ::txn/quantity-transacted (comp - abs)))])
(defmethod type-specific-translation :paid-for-an-order [txn]
(let [usd-value (* (::txn/usd-current-price txn)
(::txn/quantity-transacted txn))]
[(-> txn update-txn-type (assoc ::txn/usd-value (- usd-value)))]))
(defmethod type-specific-translation :cardspend [txn]
(let [usd-value (* (::txn/usd-current-price txn)
(::txn/quantity-transacted txn))]
[(-> txn update-txn-type (assoc ::txn/usd-value (- usd-value)))]))
(defmethod type-specific-translation :send [txn]
(let [usd-value (* (::common/usd-current-price txn)
(::common/quantity-transacted txn))]
(let [usd-value (* (::txn/usd-current-price txn)
(::txn/quantity-transacted txn))]
[(-> (update-txn-type txn)
(assoc ::common/recipient :unknown)
(assoc ::common/usd-value usd-value)
(update ::common/quantity-transacted -))]))
(assoc ::txn/recipient :unknown)
(assoc ::txn/usd-value usd-value)
(update ::txn/quantity-transacted -))]))
(defmethod type-specific-translation :receive [txn]
(let [usd-value (* (::common/usd-current-price txn)
(::common/quantity-transacted txn))]
(let [usd-value (* (::txn/usd-current-price txn)
(::txn/quantity-transacted txn))]
[(-> (update-txn-type txn)
(assoc ::common/usd-value usd-value)
(assoc ::common/sender :unknown))]))
(assoc ::txn/usd-value usd-value)
(assoc ::txn/sender :unknown))]))
(defmethod type-specific-translation :fee [txn]
[(update-txn-type txn)])
@ -104,42 +132,57 @@
(defmethod type-specific-translation nil [txn]
(throw (RuntimeException. (str "Missing txn-type: " txn))))
(defn pthru [obj] (pprint obj) obj)
(defn- register-reward
"Coinbase Card pays rewards in XLM."
[txn]
(if (and (= (::txn/currency txn) :xlm)
(= (::txn-type txn) :receive))
[(-> txn
(assoc ::txn/txn-type ::txn/income)
(assoc ::txn/sender :coinbase-card))]
[txn]))
(def translate-coinbase-txns
(common/flatcomp
(common/add-field ::common/account
(common/add-field ::txn/account
(fn [_] :coinbase))
(common/rename-field :notes
::common/notes)
::txn/notes)
(common/rename-field :asset
::common/currency
::txn/currency
common/to-keyword)
(common/rename-field :usd-fees
(common/rename-field :fees
::usd-fees
(common/zero-or bigdec))
(common/rename-field :usd-subtotal
::common/usd-value
(common/rename-field :subtotal
::txn/usd-value
(common/null-or bigdec))
(common/rename-field :timestamp
::common/timestamp
::txn/timestamp
(common/date-parser "yyyy-MM-dd'T'HH:mm:ss'Z'"))
(common/rename-field :transaction-type
::txn-type
common/to-keyword)
(common/rename-field :usd-spot-price-at-transaction
::common/usd-current-price
(common/rename-field :spot-price-at-transaction
::txn/usd-current-price
(common/null-or bigdec))
(common/rename-field :quantity-transacted
::common/quantity-transacted
::txn/quantity-transacted
bigdec)))
(defn load-csv [filename]
(let [header-line? (fn [cells] (= (first cells) "Timestamp"))
txn-translator (common/flatcomp
(common/add-txid ::common/txid)
type-specific-translation
split-fee
translate-coinbase-txns)]
txn-translator (common/flatcomp (common/add-txid ::txn/txid)
register-reward
type-specific-translation
split-fee
translate-coinbase-txns)]
(->> (common/load-csv filename)
(drop-while (comp not header-line?))
(common/pivot-csv)
(common/flatmap txn-translator))))
(s/fdef load-csv
:args (s/cat :filename string?)
:ret (s/coll-of ::txn/txn))

66
src/worther/injest/coinbase_pro.clj
View File

@ -1,5 +1,6 @@
(ns worther.injest.coinbase-pro
(:require [worther.injest.common :as common]
[worther.txn :as txn]
[clojure.core.match :refer [match]]
[clojure.spec.alpha :as s]))
@ -30,62 +31,63 @@
(defn parse-int [i] (Integer/parseInt i))
(defn parse-uuid [str] (java.util.UUID/fromString str))
;; Supplied by Clojure as of 1.11
;; (defn parse-uuid [str] (java.util.UUID/fromString str))
(defn print-through [o] (clojure.pprint/pprint o) o)
(defn coinbase-pro-txn->common-txn [txn]
(get {
:withdrawal ::common/withdrawal
:deposit ::common/deposit
:withdrawal ::txn/withdrawal
:deposit ::txn/deposit
}
(::txn-type txn)))
(defn merge-match-order [spend receive common]
(match (map ::common/currency [spend receive])
(match (map ::txn/currency [spend receive])
[:usd _] [(merge common
{
::common/txn-type ::common/buy
::common/usd-value (::common/quantity-transacted spend)
::common/quantity-transacted (::common/quantity-transacted receive)
::common/currency (::common/currency receive)
::txn/txn-type ::txn/buy
::txn/usd-value (::txn/quantity-transacted spend)
::txn/quantity-transacted (::txn/quantity-transacted receive)
::txn/currency (::txn/currency receive)
})]
[_ :usd] [(merge common
{
::common/txn-type ::common/sell
::common/usd-value (::common/quantity-transacted receive)
::common/quantity-transacted (::common/quantity-transacted spend)
::common/currency (::common/currency spend)
::txn/txn-type ::txn/sell
::txn/usd-value (::txn/quantity-transacted receive)
::txn/quantity-transacted (::txn/quantity-transacted spend)
::txn/currency (::txn/currency spend)
})]
:else [(merge common
{
::common/txn-type ::common/sell
::common/usd-value :deferred
::common/quantity-transacted (::common/quantity-transacted spend)
::common/currency (::common/currency spend)
::txn/txn-type ::txn/sell
::txn/usd-value :deferred
::txn/quantity-transacted (::txn/quantity-transacted spend)
::txn/currency (::txn/currency spend)
})
(merge common
{
::common/txn-type ::common/buy
::common/usd-value :deferred
::common/quantity-transacted (::common/quantity-transacted receive)
::common/currency (::common/currency receive)
::txn/txn-type ::txn/buy
::txn/usd-value :deferred
::txn/quantity-transacted (::txn/quantity-transacted receive)
::txn/currency (::txn/currency receive)
})]))
(def process-transfer
(common/flatcomp
(common/add-field ::common/txn-type coinbase-pro-txn->common-txn)))
(common/add-field ::txn/txn-type coinbase-pro-txn->common-txn)))
(defn process-match-order [[trade-id elems]]
(let [txn0 (first elems)
fee? (*-> ::txn-type (= :fee))
spend? (and-> (*-> ::common/quantity-transacted neg?) (*-> ::txn-type (= :match)))
receive? (and-> (comp not neg? ::common/quantity-transacted) (*-> ::txn-type (= :match)))
spend? (and-> (*-> ::txn/quantity-transacted neg?) (*-> ::txn-type (= :match)))
receive? (and-> (comp not neg? ::txn/quantity-transacted) (*-> ::txn-type (= :match)))
common-fields (select-keys txn0
[::order-id
::trade-id
::common/account
::common/timestamp
::txn/account
::txn/timestamp
::transfer-id
::balance
::portfolio])]
@ -93,9 +95,9 @@
(find-first receive? elems)
common-fields)
(map (fn [txn]
(-> (select-keys txn [::common/quantity-transacted ::common/currency])
(-> (select-keys txn [::txn/quantity-transacted ::txn/currency])
(merge common-fields)
(assoc ::common/txn-type ::common/fee)))
(assoc ::txn/txn-type ::txn/fee)))
(filter fee? elems)))))
(defn group-transactions [txns]
@ -107,13 +109,13 @@
(def preclean-txns
(common/flatcomp
(common/rename-field :amount
::common/quantity-transacted
::txn/quantity-transacted
bigdec)
(common/rename-field :amount/balance-unit
::common/currency
::txn/currency
common/to-keyword)
(common/rename-field :time
::common/timestamp
::txn/timestamp
(common/date-parser "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"))
(common/rename-field :trade-id
::trade-id
@ -137,7 +139,7 @@
(defn load-csv [filename]
(->> (common/load-csv filename)
(common/pivot-csv)
(map #(assoc % ::common/account :coinbase-pro))
(map #(assoc % ::txn/account :coinbase-pro))
(common/flatmap preclean-txns)
(group-transactions)
(common/flatmap (common/add-txid ::common/txid))))
(common/flatmap (common/add-txid ::txn/txid))))

72
src/worther/injest/common.clj
View File

@ -3,7 +3,9 @@
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.spec.alpha :as s]
[valuehash.api :as hash]))
[valuehash.api :as hash]
[worther.txn :as txn]))
(s/def ::header keyword?)
@ -54,11 +56,11 @@
(defn generate-id [entry]
;; Use a vector because I'm not sure the map will be ordered
(let [make-entry-vector (juxt ::timestamp
::quantity-transacted
::txn-type
::account
::currency)]
(let [make-entry-vector (juxt :txn/timestamp
:txn/quantity-transacted
:txn/txn-type
:txn/account
:txn/currency)]
(-> entry
(make-entry-vector)
(hash/sha-256-str))))
@ -70,7 +72,7 @@
([field new-field] (rename-field field new-field identity))
([field new-field f] (fn [entry]
(when (not (contains? entry field))
(throw (RuntimeException. (str "Field not found: " field))))
(throw (RuntimeException. (str "Field not found: " field " in " entry))))
[(assoc (dissoc entry field)
new-field
(f (get entry field)))])))
@ -95,60 +97,22 @@
(defn zero-or [f]
(fn [str] (if (or (nil? str) (empty? str)) (bigdec 0) (f str))))
(s/def ::txn-type #{::buy ::sell ::send ::receive ::deposit ::withdrawal ::fee ::income})
(defn valid-timestamp? [ts] (instance? java.util.Date ts))
(s/def ::timestamp valid-timestamp?)
(defn bigdec? [i] (instance? java.math.BigDecimal i))
(s/def ::quantity-transacted bigdec?)
(s/def ::usd-value (s/or :deferred #(= % :deferred)
:number bigdec?))
(s/def ::currency keyword?)
(s/def ::account keyword?)
(s/def ::sender keyword?)
(s/def ::recipient keyword?)
(defmulti txn ::txn-type)
(s/def ::txn (s/multi-spec txn ::txn-type))
(let [txn-common-req [::txid
::quantity-transacted
::currency
::txn-type
::account
::timestamp]]
(defmethod txn ::buy [_]
(s/keys :req (concat txn-common-req [::usd-value])))
(defmethod txn ::sell [_]
(s/keys :req (concat txn-common-req [::usd-value])))
(defmethod txn ::send [_]
(s/keys :req (concat txn-common-req [::recipient])))
(defmethod txn ::receive [_]
(s/keys :req (concat txn-common-req [::sender])))
(defmethod txn ::fee [_]
(s/keys :req (concat txn-common-req)))
(defmethod txn ::income [_]
(s/keys :req (concat txn-common-req [::src])))
(defmethod txn ::deposit [_]
(s/keys :req (concat txn-common-req [::src])))
(defmethod txn ::withdrawal [_]
(s/keys :req (concat txn-common-req [::dest]))))
(defn map-select [m]
(fn [om]
(every? (fn [key] (= (get om key)
(get m key)))
(keys m))))
(defn add-tag [tag src obj]
(with-meta obj { tag src }))
(def add-source-tag (partial add-tag :source))
(defn matching-files [rx dir]
(filter (fn [file]
(re-matches rx file))
(map str (file-seq (clojure.java.io/file dir)))))
(defn pprint-str [obj]
(with-out-str (clojure.pprint/pprint obj))
obj)

158
src/worther/originator.clj Normal file
View File

@ -0,0 +1,158 @@
(ns worther.originator
(:require [worther.txn :as txn]
[clojure.spec.alpha :as s]))
;; Think this through.
;;
;; - The input will be a list of transactions. Step one: extract the sales.
;; - Order chronologically. These need mapping to purchases. Step two: extract
;; the purchases. Order chronologically, split by currency. Step three:
;; starting with the first sale, start consuming purchases until the total of
;; the sale is covered.
;; - Return: [ filled sales, remaining sales, remaining purchases ].
;; - In the end, return only filled sales.
(defn- positive? [n] (>= 0))
(s/def ::remaining (comp positive? number?))
(s/def ::contribution-amount (comp positive? number?))
(s/def ::contributions (s/coll-of (s/keys :req [::amount ::txn])))
(defn- sort-by-date [txns]
(sort (fn [a b] (.before a b)) txns))
(defn- normalize [i] (.setScale (bigdec i) 6 java.math.RoundingMode/HALF_UP))
(defn- txn-sourced? [txn]
(let [sum-coll (apply +)
src-amount (-> txn
::contributions
(map ::contribution-amount)
(sum-coll))]
(.equals (normalize (::amount txn))
(normalize src-amount))))
(defn- txn-consumed? [txn]
(.equals (normalize 0) (-> txn ::remaining normalize)))
(s/def ::contributions
(s/keys :req [::src-txn ::contribution-amount]))
(s/def ::sourced-txn
(s/and txn-sourced?
::sink-txn
(s/keys :req [::contributions])))
(defn originate-txn [sink srcs]
(let [src (first srcs)]
(if (>= (::remaining src) (::remaining sink))
;; Sink is fully covered
(let [contribution (::remaining sink)]
[(-> sink
(update ::remaining 0)
(update ::contributions conj
{
::contribution-amount contribution
::src-txn src
}))
(conj (rest srcs)
(update src ::remaining - contribution))])
;; Sink is not yet fully covered
(let [contribution (::remaining src)]
(originate-txn (-> sink
(update ::remaining - contribution)
(update ::contributions conj
{
::contribuution-amount contribution
::src-txn src
}))
(rest srcs))))))
(s/fdef originate-txn
:args (s/cat :sink ::sink-txn
:srcs (s/coll-of ::src-txn))
:ret ::sourced-txn)
(s/def ::txn-with-remaining
(s/and :txn/txn
(s/keys :req [::remaining])))
(defn- originate-txns-exec
"Given a sorted list of sink transactions and a sorted list of src transactions,
return a list of sinks mapped to originating src transactions."
[sinks srcs]
(loop [sinks sinks
srcs srcs
filled-sinks []]
(if (empty? sinks)
filled-sinks
(let [[filled-sink remaining-srcs]
(originate-txn (first sinks) srcs)]
(recur (rest sinks)
remaining-srcs
(conj filled-sinks filled-sink))))))
(s/fdef originate-txns-exec
:args (s/cat :sinks (s/coll-of (s/and :txn/sink-txn ::txn-with-remaining))
:srcs (s/coll-of (s/and :txn/src-txn ::txn-with-remaining)))
:ret (s/coll-of ::sourced-txn))
(defmulti inject-remaining
"Add a ::remaining key, containing the unallocated amount."
:txn/txn-type)
(s/fdef inject-remaining
:args (s/cat :txn :txn/txn)
:ret ::txn-with-remaining)
(defn- remaining-from [txn key]
(assoc txn ::remaining (get txn key)))
(s/fdef remaining-from
:args (s/cat :txn :txn/txn
:key keyword?)
:ret ::txn-with-remaining)
(defmethod inject-remaining :default [txn]
(remaining-from txn :txn/amount))
;; In most cases, simple: map every sink to sources for every account. But in
;; the case of a receive source, we need to look at the original account and
;; start finding sources there. They can't be shared.
;; TODO: inject ::remaining
#_(defn originate-txn [all-txns]
(let [srcs (group-by ::txn/account (filter txn/src-txn? all-txns))
sinks (filter txn/sink-txn? all-txns)]
{
:filled-sales []
:remaining-sales remaining-sales
:remaining-buys remaining-buys
}))
(defn- consume-src [sink src]
(let [contribution (apply min (map ::remaining [sink src]))]
[(-> sink
(update ::remaining - contribution)
(update ::contributions conj
{
::contribution-amount contribution
::src-txn src
}))
(update src ::remaining - contribution)]))
(s/fdef consume-src
:args (s/cat :sink ::txn/sink-txn
:src ::txn/src-txn)
:ret (s/cat :sink ::txn/sink-txn
:src ::txn/src-txn))
(s/def ::srcs-by-account (s/map-of ::txn/account (s/coll-of ::txn/src-txn)))
(defn originate-txn [sink srcs]
(let [sink-acct (::txn/account sink)
acct-srcs (get srcs sink-acct)]
(loop [])))
(s/fdef originate-txn
:args (s/cat :sink ::txn/sink-txn
:srcs ::srcs-by-account)
:ret (s/cat :sink ::sourced-txn
:srcs ::srcs-by-account))
(defn originate-txns [all-txns]
(group-by ::txn/account all-txns))

119
src/worther/txn.clj Normal file
View File

@ -0,0 +1,119 @@
(ns worther.txn
(:require [clojure.pprint :refer [pprint]]
[clojure.spec.alpha :as s]
[clojure.set :refer [union]]
[worther.txn :as txn]))
(def src-txn-types
#{
::buy
::receive
::deposit
::income
})
(def sink-txn-types
#{
::sell
::send
::withdrawal
::fee
})
(s/def ::txn-type (union src-txn-types sink-txn-types))
(s/def ::src-txn-type src-txn-types)
(s/def ::sink-txn-type sink-txn-types)
(defn src-txn? [txn] (->> txn ::txn-type (contains? src-txn-types)))
(defn sink-txn? [txn] (->> txn ::txn-type (contains? sink-txn-types)))
(defn valid-timestamp? [ts] (instance? java.util.Date ts))
(s/def ::timestamp valid-timestamp?)
(defn bigdec? [i] (instance? java.math.BigDecimal i))
(s/def ::quantity-transacted bigdec?)
(s/def ::usd-value (s/or :deferred #(= % :deferred)
:number bigdec?))
(s/def ::currency keyword?)
(s/def ::account keyword?)
(s/def ::sender keyword?)
(s/def ::recipient keyword?)
(defmulti txn ::txn-type)
(s/def ::txn (s/multi-spec txn ::txn-type))
(defn src-txn? [txn]
(->> txn
::txn-type
(contains? src-txn-types)))
(defn sink-txn? [txn]
(->> txn ::txn-type (contains? sink-txn-types)))
(s/def ::src-txn src-txn?)
(s/def ::sink-txn sink-txn?)
(defn- pprint-to-string [o]
(with-out-str (pprint o)))
(defn- pthru [obj]
(clojure.pprint/pprint obj)
obj)
(defn- negative? [key]
(fn [txn] (-> txn key (<= 0))))
(defn- positive? [key]
(fn [txn] (-> txn key (>= 0))))
(let [txn-common-req [::txid
::quantity-transacted
::currency
::txn-type
::account
::timestamp]]
(defmethod txn ::buy [_]
(s/and (s/keys :req (concat txn-common-req [::usd-value]))
;; deferred
;;(negative? ::usd-value)
(positive? ::quantity-transacted)))
(defmethod txn ::sell [_]
(s/and (s/keys :req (concat txn-common-req [::usd-value]))
(negative? ::quantity-transacted)
;; Deferred
;;(positive? ::usd-value)
))
(defmethod txn ::send [_]
(s/and (s/keys :req (concat txn-common-req [::recipient]))
(negative? ::quantity-transacted)))
(defmethod txn ::receive [_]
(s/and (s/keys :req (concat txn-common-req [::sender]))
(positive? ::quantity-transacted)))
(defmethod txn ::fee [_]
(s/and (s/keys :req (concat txn-common-req))
;; Deferred
;;(negative? ::usd-value)
))
(defmethod txn ::income [_]
(s/and (s/keys :req (concat txn-common-req [::src]))
(positive? ::quantity-transacted)))
(defmethod txn ::deposit [_]
(s/and (s/keys :req (concat txn-common-req [::src]))
(positive? ::quantity-transacted)))
(defmethod txn ::withdrawal [_]
(s/and (s/keys :req (concat txn-common-req [::dest]))
(negative? ::quantity-transacted)))
(defmethod txn :default [txn]
(throw (RuntimeException.
(str "No method defined for txn-type: "
(::txn-type txn)
"\n"
(pprint-to-string txn))))))