Skip to content

Commit

Permalink
Addition of gitSectionA files
Browse files Browse the repository at this point in the history
  • Loading branch information
tmcfarlane795 committed Jan 11, 2024
1 parent fc7011e commit f049762
Show file tree
Hide file tree
Showing 5 changed files with 460 additions and 0 deletions.
149 changes: 149 additions & 0 deletions dbmaint.q
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/ kdb+ partitioned database maintenance
\d .os
WIN:.z.o in`w32`w64
pth:{p:$[10h=type x;x;string x];if[WIN;p[where"/"=p]:"\\"];(":"=first p)_ p}
cpy:{system$[WIN;"copy /v /z ";"cp "],pth[x]," ",pth y}
del:{system$[WIN;"del ";"rm "],pth x}
ren:{system$[WIN;"move ";"mv "],pth[x]," ",pth y}
here:{hsym`$system$[WIN;"cd";"pwd"]}
\d .

add1col:{[tabledir;colname;defaultvalue]
if[not colname in ac:allcols tabledir;
stdout"adding column ",(string colname)," (type ",(string type defaultvalue),") to `",string tabledir;
num:count get(`)sv tabledir,first ac;
.[(`)sv tabledir,colname;();:;num#defaultvalue];
@[tabledir;`.d;,;colname]]}

allcols:{[tabledir]get tabledir,`.d}

allpaths:{[dbdir;table]
files:key dbdir;
if[any files like"par.txt";:raze allpaths[;table]each hsym each`$read0(`)sv dbdir,`par.txt];
files@:where files like"[0-9]*";(`)sv'dbdir,'files,'table}

copy1col:{[tabledir;oldcol;newcol]
if[(oldcol in ac)and not newcol in ac:allcols tabledir;
stdout"copying ",(string oldcol)," to ",(string newcol)," in `",string tabledir;
.os.cpy[(`)sv tabledir,oldcol;(`)sv tabledir,newcol];@[tabledir;`.d;,;newcol]]}

delete1col:{[tabledir;col]
if[col in ac:allcols tabledir;
stdout"deleting column ",(string col)," from `",string tabledir;
.os.del[(`)sv tabledir,col];@[tabledir;`.d;:;ac except col]]}

/
enum:{[tabledir;val]
if[not 11=abs type val;:val];
.[p;();,;u@:iasc u@:where not(u:distinct enlist val)in v:$[type key p:(`)sv tabledir,`sym;get p;0#`]];`sym!(v,u)?val}
\

enum:{[tabledir;val]if[not 11=abs type val;:val];.Q.dd[tabledir;`sym]?val}


find1col:{[tabledir;col]
$[col in allcols tabledir;
stdout"column ",string[col]," (type ",(string first"i"$read1((`)sv tabledir,col;8;1)),") in `",string tabledir;
stdout"column ",string[col]," *NOT*FOUND* in `",string tabledir]}

fix1table:{[tabledir;goodpartition;goodpartitioncols]
if[count missing:goodpartitioncols except allcols tabledir;
stdout"fixing table `",string tabledir;{add1col[x;z;0#get y,z]}[tabledir;goodpartition]each missing]}

fn1col:{[tabledir;col;fn]
if[col in allcols tabledir;
oldattr:-2!oldvalue:get p:tabledir,col;
newattr:-2!newvalue:fn oldvalue;
if[$[not oldattr~newattr;1b;not oldvalue~newvalue];
stdout"resaving column ",(string col)," (type ",(string type newvalue),") in `",string tabledir;
oldvalue:0;.[(`)sv p;();:;newvalue]]]}

reordercols0:{[tabledir;neworder]
if[not((count ac)=count neworder)or all neworder in ac:allcols tabledir;'`order];
stdout"reordering columns in `",string tabledir;
@[tabledir;`.d;:;neworder]}

rename1col:{[tabledir;oldname;newname]
if[(oldname in ac)and not newname in ac:allcols tabledir;
stdout"renaming ",(string oldname)," to ",(string newname)," in `",string tabledir;
.os.ren[` sv tabledir,oldname;` sv tabledir,newname];@[tabledir;`.d;:;.[ac;where ac=oldname;:;newname]]]}

ren1table:{[old;new]stdout"renaming ",(string old)," to ",string new;.os.ren[old;new];}

add1table:{[dbdir;tablename;table]
stdout"adding ",string tablename;
@[tablename;`;:;.Q.en[dbdir]0#table];}

stdout:{-1 raze[" "sv string`date`second$.z.P]," ",x;}
validcolname:{(not x in `i,.Q.res,key`.q)and x = .Q.id x}

//////////////////////////////////////////////////////////////////////////////////////////////////////////
// * public

thisdb:`:. / if functions are to be run within the database instance then use <thisdb> (`:.) as dbdir

addcol:{[dbdir;table;colname;defaultvalue] / addcol[`:/data/taq;`trade;`noo;0h]
if[not validcolname colname;'(`)sv colname,`invalid.colname];
add1col[;colname;enum[dbdir;defaultvalue]]each allpaths[dbdir;table];}

castcol:{[dbdir;table;col;newtype] / castcol[thisdb;`trade;`size;`short]
fncol[dbdir;table;col;newtype$]}

clearattrcol:{[dbdir;table;col] / clearattr[thisdb;`trade;`sym]
setattrcol[dbdir;table;col;(`)]}

copycol:{[dbdir;table;oldcol;newcol] / copycol[`:/k4/data/taq;`trade;`size;`size2]
if[not validcolname newcol;'(`)sv newcol,`invalid.newname];
copy1col[;oldcol;newcol]each allpaths[dbdir;table];}

deletecol:{[dbdir;table;col] / deletecol[`:/k4/data/taq;`trade;`iz]
delete1col[;col]each allpaths[dbdir;table];}

findcol:{[dbdir;table;col] / findcol[`:/k4/data/taq;`trade;`iz]
find1col[;col]each allpaths[dbdir;table];}

/ adds missing columns, but DOESN'T delete extra columns - do that manually
fixtable:{[dbdir;table;goodpartition] / fixtable[`:/k4/data/taq;`trade;`:/data/taq/2005.02.19]
fix1table[;goodpartition;allcols goodpartition]each allpaths[dbdir;table]except goodpartition;}

fncol:{[dbdir;table;col;fn] / fncol[thisdb;`trade;`price;2*]
fn1col[;col;fn]each allpaths[dbdir;table];}

listcols:{[dbdir;table] / listcols[`:/k4/data/taq;`trade]
allcols first allpaths[dbdir;table]}

renamecol:{[dbdir;table;oldname;newname] / renamecol[`:/k4/data/taq;`trade;`woz;`iz]
if[not validcolname newname;'` sv newname,`invalid.newname];
rename1col[;oldname;newname]each allpaths[dbdir;table];}

reordercols:{[dbdir;table;neworder] / reordercols[`:/k4/data/taq;`trade;reverse cols trade]
reordercols0[;neworder]each allpaths[dbdir;table];}

setattrcol:{[dbdir;table;col;newattr] / setattr[thisdb;`trade;`sym;`g] / `s `p `u
fncol[dbdir;table;col;newattr#]}

addtable:{[dbdir;tablename;table] / addtable[`:.;`trade;([]price...)]
add1table[dbdir;;table]each allpaths[dbdir;tablename];}

rentable:{[dbdir;old;new] / rentable[`:.;`trade;`transactions]
ren1table'[allpaths[dbdir;old];allpaths[dbdir;new]];}

\
test with https://github.com/KxSystems/kdb/blob/master/tq.q (sample taq database)
if making changes to current database you need to reload (\l .) to make modifications visible
if the database you've been modifying is a tick database don't forget to adjust the schema (tick/???.q) to reflect your changes to the data
addcol[`:.;`trade;`num;10]
addcol[`:.;`trade;`F;`test]
delete1col[`:./2000.10.02/trade;`F]
fixtable[`:.;`trade;`:./2000.10.03/trade]
reordercols[`:.;`quote;except[2 rotate cols quote;`date]]
clearattrcol[`:.;`trade;`sym]
setattrcol[`:.;`trade;`sym;`p]
castcol[`:.;`trade;`time;`second]
renamecol[`:.;`trade;`price;`PRICE]
pxcols:{(y,())renamecol[`:.;x]'z,()]
`PRICE`size renamecol[`:.;`trade]'`p`s
71 changes: 71 additions & 0 deletions quote.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
time sym bid ask bsize asize
2019-06-18D03:05:15.716040000 GS.N 4.194429 28.92601 4 5
2019-06-18D03:05:17.715982000 GS.N 11.53192 48.36199 9 4
2019-06-18D03:05:19.715528000 MSFT.O 7.836585 7.540665 1 8
2019-06-18D03:05:20.715186000 MSFT.O 24.88246 30.54409 4 2
2019-06-18D03:05:22.714657000 IBM.N 12.80329 2.876258 8 7
2019-06-18D03:05:24.714268000 IBM.N 47.99482 5.434121 7 3
2019-06-18D03:05:27.703161000 GS.N 21.64267 31.62057 8 2
2019-06-18D03:05:32.703864000 MSFT.O 24.22364 38.8941 6 6
2019-06-18D03:05:36.702931000 GS.N 38.3743 21.11112 7 8
2019-06-18D03:05:42.717138000 IBM.N 39.67756 44.86179 5 0
2019-06-18D03:05:44.716618000 GS.N 38.85652 31.78591 4 3
2019-06-18D03:05:45.716420000 MSFT.O 45.41356 32.07488 4 8
2019-06-18D03:05:48.710737000 MSFT.O 36.2474 28.09518 4 4
2019-06-18D03:05:50.704024000 IBM.N 4.991336 40.98007 5 7
2019-06-18D03:05:51.703557000 MSFT.O 15.0702 48.37882 1 2
2019-06-18D03:05:53.703353000 GS.N 15.57182 43.24631 2 8
2019-06-18D03:05:56.711853000 BA.N 32.84367 43.62513 6 9
2019-06-18D03:05:57.711573000 GS.N 18.07572 17.7519 4 8
2019-06-18D03:05:58.711376000 MSFT.O 16.69403 26.14589 5 4
2019-06-18D03:05:59.711168000 BA.N 47.58373 17.07496 4 0
2019-06-18D03:06:01.710647000 BA.N 39.79765 9.314345 0 9
2019-06-18D03:06:03.703738000 BA.N 9.363172 15.93321 5 1
2019-06-18D03:06:04.703629000 GS.N 0.1092236 35.46211 7 3
2019-06-18D03:06:06.703179000 GS.N 19.61625 17.64781 9 0
2019-06-18D03:06:07.702876000 BA.N 22.59818 46.28021 2 7
2019-06-18D03:06:09.702319000 BA.N 5.239721 0.8342128 4 1
2019-06-18D03:06:10.702161000 GS.N 15.04851 0.2208297 6 4
2019-06-18D03:06:12.717323000 MSFT.O 46.37225 38.16163 4 8
2019-06-18D03:06:14.716694000 BA.N 1.405337 32.67986 2 1
2019-06-18D03:06:16.716342000 MSFT.O 24.18711 32.5099 6 1
2019-06-18D03:06:18.715898000 IBM.N 23.07442 31.69767 1 5
2019-06-18D03:06:19.715737000 GS.N 36.08687 28.26131 1 4
2019-06-18D03:06:20.715418000 BA.N 30.32804 14.53779 5 1
2019-06-18D03:06:23.714658000 GS.N 15.82207 13.62928 2 9
2019-06-18D03:06:24.714425000 MSFT.O 38.29525 29.69318 0 4
2019-06-18D03:06:27.713770000 BA.N 31.07077 42.14983 7 4
2019-06-18D03:06:33.185300000 MSFT.O 12.00385 7.485022 5 2
2019-06-18D03:06:33.712352000 BA.N 8.324299 10.96574 6 7
2019-06-18D03:06:36.711239000 IBM.N 18.81573 44.36463 2 3
2019-06-18D03:06:37.711231000 MSFT.O 31.61489 20.45336 1 3
2019-06-18D03:06:39.706445000 MSFT.O 4.038664 23.12236 6 9
2019-06-18D03:06:40.706252000 MSFT.O 24.2475 46.81004 5 6
2019-06-18D03:06:42.705819000 BA.N 6.383121 9.113048 0 5
2019-06-18D03:06:43.705597000 BA.N 26.9158 22.50462 3 8
2019-06-18D03:06:44.705333000 GS.N 11.85489 12.08228 1 2
2019-06-18D03:06:47.704608000 BA.N 13.23763 12.97067 5 5
2019-06-18D03:06:49.704167000 IBM.N 43.99847 31.0608 5 7
2019-06-18D03:06:52.703447000 MSFT.O 39.60351 33.07996 4 5
2019-06-18D03:06:53.703062000 MSFT.O 6.65758 46.37964 3 4
2019-06-18D03:06:55.702757000 GS.N 34.01953 36.8386 9 8
2019-06-18D03:06:57.702259000 IBM.N 23.66585 27.28721 8 6
2019-06-18D03:07:00.717191000 GS.N 4.775345 35.20805 5 6
2019-06-18D03:07:01.716814000 BA.N 32.64634 46.93086 0 6
2019-06-18D03:07:02.716682000 GS.N 15.7824 9.282377 0 9
2019-06-18D03:07:03.716454000 IBM.N 28.97735 47.92627 5 1
2019-06-18D03:07:04.716242000 MSFT.O 6.41355 38.08624 3 2
2019-06-18D03:07:07.715561000 GS.N 1.062656 12.47356 9 8
2019-06-18D03:07:11.714443000 IBM.N 3.415027 14.99222 2 6
2019-06-18D03:07:12.714438000 IBM.N 34.22852 31.04312 7 4
2019-06-18D03:07:15.709614000 MSFT.O 8.238095 28.05404 7 5
2019-06-18D03:07:16.716735000 GS.N 0.9810431 21.17852 1 5
2019-06-18D03:07:17.716585000 IBM.N 12.26745 8.909183 9 9
2019-06-18D03:07:23.715263000 GS.N 39.31552 13.63137 0 1
2019-06-18D03:07:26.714542000 IBM.N 5.039161 10.74923 1 0
2019-06-18D03:07:27.714268000 BA.N 38.01613 40.07364 7 7
2019-06-18D03:07:28.714092000 BA.N 9.187076 21.18061 0 2
2019-06-18D03:07:32.712777000 GS.N 14.56379 27.17526 6 2
2019-06-18D03:07:35.711844000 MSFT.O 27.55229 3.059558 2 0
2019-06-18D03:07:37.711543000 GS.N 12.33477 40.97651 3 6
2019-06-18D03:07:38.711250000 IBM.N 21.45296 28.87573 9 4
7 changes: 7 additions & 0 deletions sym.q
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// schema.q
// Quote Table Schema
quote:([]time:`timestamp$();sym:`symbol$();bid:`float$();ask:`float$();bsize:`long$();asize:`long$());
// trade Table Schema
trade:([]time:`timestamp$();sym:`symbol$();price:`float$();size:`long$());
// Aggregation Table Schema
aggTrade:([sym:`symbol$()];time:`timestamp$();maxTPrice:`float$();minTPice:`float$();tVolume:`long$();bestBid:`float$();bestAsk:`float$());
Loading

0 comments on commit f049762

Please sign in to comment.