diff --git a/README.md b/README.md deleted file mode 100644 index f9fbf663..00000000 --- a/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# NormalizingFlows.jl - -[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://turinglang.github.io/NormalizingFlows.jl/dev/) -[![Build Status](https://github.com/TuringLang/NormalizingFlows.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/TuringLang/NormalizingFlows.jl/actions/workflows/CI.yml?query=branch%3Amain) diff --git a/example/HamiltonianVI/data/bank_dat.csv b/example/HamiltonianVI/data/bank_dat.csv new file mode 100644 index 00000000..a8159d20 --- /dev/null +++ b/example/HamiltonianVI/data/bank_dat.csv @@ -0,0 +1,401 @@ +"","age","marital","balance","housing","duration","campaign","pdays","previous","y" +"1",0.19806976350248,-1.04629924020543,4457.41775891603,-0.959567210136544,121.830353246711,-0.723428206559773,0.353721396016046,0.452003571795249,"1" +"2",104.146940630705,-0.702269515361112,-0.260956339631889,-1.04629924020543,3678.39336905394,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"3",-0.689720178893626,-0.476696519712976,4730.53644018208,-0.702269515361112,33.7987805129382,-1.03860930317821,-2.99624765331573,-0.959567210136544,"1" +"4",118.986334056931,-0.00208617190785308,0.804669047638875,-0.484837871827967,779.493453783607,-0.702269515361112,20.6706339632953,-1.04373592786303,"1" +"5",-0.954019995878704,0.998749217771909,508.810513174902,-0.280242426288262,-0.287920489113158,-0.468555167597986,242.406856709252,-0.212029015981627,"1" +"6",39.0072109343771,-4.02497038533371,21.0058768415407,-0.998749217771909,856.653626849399,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"7",-0.102194322423417,0.452003571795249,846.923553911768,-4.02497038533371,-0.454174089108914,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"8",10.5678514945477,-0.723428206559773,20.9269181905968,-2.2068409681768,317.954055081304,-3.65774950282621,-1.04886255254784,-0.998749217771909,"1" +"9",58.9262038485426,1.03953114431459,134.347384693829,-0.723428206559773,5.23853266501029,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"10",-1.45442420778125,-1.04373592786303,13116.5734769985,1.03953114431459,220.019511042995,-0.722978766190891,1.94128505843918,0.452003571795249,"1" +"11",159.982675970118,-0.702269515361112,111.28238672202,-1.04629924020543,1003.74296386077,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"12",-0.703203389960085,-0.476696519712976,2135.50416916935,-0.702269515361112,93.4721739204061,-1.04117261552062,720.098186013546,1.03953114431459,"1" +"13",98.9953505124201,-0.280242426288262,-0.545899260851401,-0.484837871827967,1901.52584965181,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"14",-0.925823560112203,0.998749217771909,7347.72598375223,-0.280242426288262,-0.537359893842643,-0.468555167597986,907.117991702265,-0.702269515361112,"1" +"15",15.9659074635414,-4.02497038533371,-0.564396519832509,0.998749217771909,3727.35886384123,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"16",-0.175466491458331,-2.2068409681768,582.193684246846,-4.02497038533371,-0.838670940470291,2.99624765331573,780.687889380256,0.554226336852964,"1" +"17",7.78628895074358,-0.723428206559773,-4.49852446451824,0.452003571795249,261.576397652663,-3.93316516470684,-0.190152917840769,6.99124452440336,"1" +"18",116.853658479313,1.03953114431459,-0.280242426288262,-0.723877646928655,3.5369900729773,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"19",-0.53637200151251,-1.04373592786303,104.86866786605,-0.959567210136544,71.2059149494767,-0.723428206559773,2.72285486147826,0.452003571795249,"1" +"20",82.8761843109289,-0.702269515361112,27.8314411721915,-1.04373592786303,1848.6848020958,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"21",-0.701855068853439,-0.476696519712976,4863.47866718068,-0.702269515361112,19.8443869776534,-1.04373592786303,356.553470744572,1.03953114431459,"1" +"22",69.0088751956531,-0.00208617190785308,-0.334212847108,-0.476696519712976,609.327403225395,0.278211483397858,-4.20858082658746,-1.04629924020543,"1" +"23",-0.96683655759075,0.998749217771909,2809.77271914816,-0.00208617190785308,-0.233538204478442,-0.468555167597986,-4.86568550814886,-1.1925100147406,"1" +"24",14.9854264647824,-4.11677560596058,8.04320632597751,0.998749217771909,1552.33985419839,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"25",-0.183607843573321,0.452003571795249,91.462944367982,-4.11677560596058,-0.397781217575912,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"26",12.5149452752105,-0.723877646928655,-0.484837871827967,-2.2068409681768,500.323520850472,-3.84135994407996,-1.04886255254784,-0.998749217771909,"1" +"27",64.9186991551741,1.03953114431459,237.543355068961,-0.723877646928655,0.410710860820979,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"28",0.657095866636848,-1.04373592786303,186.766103723347,-0.959567210136544,67.5898836425314,-0.723428206559773,-0.492979223942957,-2.2068409681768,"1" +"29",112.123474250621,-0.702269515361112,8.00151351678674,-1.04629924020543,1145.56535278438,1.03953114431459,23.084882941666,-0.722529325822009,"1" +"30",-0.703652830328967,-0.484837871827967,-552.587660742392,-1.1925100147406,76.4882081044345,-1.04117261552062,-2.99624765331573,-0.959567210136544,"1" +"31",90.9989570946156,-0.00208617190785308,-0.236234846691733,-0.476696519712976,1388.36885343721,-0.212029015981627,28.2904672753258,-0.897627124345702,"1" +"32",-0.923260247769794,0.998749217771909,3189.60140649387,-0.00208617190785308,-0.382302966578369,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"33",11.553742969126,-4.02497038533371,3.04218394593721,-0.998749217771909,308.900677729789,-0.00208617190785308,-0.67668640819605,-0.476696519712976,"1" +"34",-0.142901082998369,0.452003571795249,0.768451982777342,-4.02497038533371,0.991534072009863,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"35",13.9057265471126,-0.723428206559773,-0.484837871827967,0.452003571795249,462.575002398252,-3.84135994407996,-1.04886255254784,-0.998749217771909,"1" +"36",68.9136960262617,1.03953114431459,327.109668979452,-0.723877646928655,1.4935106921147,8.42853719171141,-1.68275051412008,-4.11677560596058,"1" +"37",-0.628177222139383,-1.04373592786303,8799.97935778829,1.03953114431459,191.369416841813,-0.722978766190891,-0.492979223942957,-2.2068409681768,"1" +"38",128.076541490454,-0.702269515361112,3.68666814732368,-1.04373592786303,1177.52532775308,9.03592456211912,99.5778528962784,-0.723428206559773,"1" +"39",-0.709944995493314,-0.484837871827967,574.762424205759,-0.702269515361112,158.929296227367,-1.04117261552062,-2.99624765331573,-0.959567210136544,"1" +"40",110.989940639127,-0.00208617190785308,0.848264763420425,-0.484837871827967,699.728117584445,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"41",-0.946330058851477,0.998749217771909,3669.38501156214,-0.280242426288262,-0.318482434197131,-0.460413815482995,-4.86568550814886,-1.1925100147406,"1" +"42",14.4951859654029,-4.02497038533371,0.194343933520608,0.998749217771909,164.965596209307,1.38869509999419,-0.639832297947729,-0.460413815482995,"1" +"43",-0.020780801273513,0.452003571795249,-46.2946359576532,-4.02497038533371,3.09857681747021,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"44",18.0780703628187,-0.723428206559773,4.29413581967141,-2.2068409681768,222.847398201684,-3.93316516470684,-0.0414808019810366,2.99624765331573,"1" +"45",64.9186991551741,1.03953114431459,19.4688516347207,-0.723877646928655,2.24251508669382,5.76969265173935,48.8120209219668,-3.93316516470684,"1" +"46",-0.628177222139383,-1.04629924020543,138.826141270295,-0.959567210136544,72.8748524757592,-0.723428206559773,-0.492979223942957,-2.2068409681768,"1" +"47",77.5584952309848,-0.702269515361112,240.635942585285,-1.04629924020543,947.813007665542,3.03862949876572,25.5882892310897,-0.722079885453127,"1" +"48",-0.705450591804495,-0.476696519712976,1539.92299221561,-0.702269515361112,76.120987221927,-1.04373592786303,724.093182884634,1.03953114431459,"1" +"49",120.985432411383,-0.00208617190785308,4.15749419949826,-0.484837871827967,596.033180525535,-0.702269515361112,4.23749947108492,-1.03348267849339,"1" +"50",-0.943766746509068,0.998749217771909,2613.86108041195,-0.280242426288262,0.0743284482056978,-0.468555167597986,-4.86568550814886,-1.1925100147406,"1" +"51",15.4756669641619,-4.02497038533371,17.612051300191,0.998749217771909,1762.24518141576,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"52",-0.020780801273513,0.452003571795249,1139.10689154194,-4.11677560596058,-0.0773671747747651,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"53",12.5149452752105,-0.723428206559773,2.35649401630369,-2.2068409681768,227.259562696099,-3.74955472345309,-0.820727754073427,4.99374608885955,"1" +"54",110.861163172682,-0.959567210136544,192.760198113715,-0.723877646928655,0.687516832730653,0.452003571795249,48.8120209219668,-4.02497038533371,"1" +"55",1.20792719039809,-1.04373592786303,1127.58786686449,-0.959567210136544,42.2776644939142,-0.723428206559773,0.272307874866142,0.452003571795249,"1" +"56",152.006142350202,-1.1925100147406,519.264787187846,-1.04629924020543,706.11569696474,1.03953114431459,23.9193517048073,-0.723428206559773,"1" +"57",-0.69511346332021,-0.476696519712976,1008.1540842212,-1.1925100147406,39.8579250743119,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"58",67.009776841202,-0.280242426288262,0.572757817295786,-0.484837871827967,832.670344583048,-0.212029015981627,10.2048388118317,-1.0360459908358,"1" +"59",-0.889937187318475,0.998749217771909,5156.71418727379,-0.280242426288262,-0.602528747330527,-0.460413815482995,534.879756106178,2.2391734809158,"1" +"60",16.9463884623003,-4.02497038533371,7.03838788775312,0.998749217771909,806.676167988121,-0.00208617190785308,-0.586798334419658,-0.460413815482995,"1" +"61",-0.020780801273513,0.452003571795249,1393.54171071989,-4.02497038533371,1.88356676716826,8.98874295994718,-2.95866556458768,-0.280242426288262,"1" +"62",11.6804765120693,-0.723877646928655,4.23714635486647,0.452003571795249,741.031606045799,-4.02497038533371,-0.318318534961228,2.99624765331573,"1" +"63",70.9111944618055,1.03953114431459,418.623076670607,-0.723877646928655,2.16924291765891,5.76969265173935,-1.68275051412008,-4.11677560596058,"1" +"64",0.657095866636848,-1.04373592786303,2449.93183119449,-0.959567210136544,312.08923124291,-0.722079885453127,-0.492979223942957,-2.2068409681768,"1" +"65",165.300365050063,-0.702269515361112,261.751143329466,-1.04629924020543,606.240775187549,1.03953114431459,32.8203518449803,-0.722978766190891,"1" +"66",-0.697810105533501,-0.484837871827967,5174.56347835741,-1.1925100147406,16.8148146969666,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"67",92.9980554490667,-0.00208617190785308,-0.723877646928655,-0.484837871827967,1569.17028215531,-0.702269515361112,-3.19872339969185,-1.04373592786303,"1" +"68",-0.915570310742567,0.998749217771909,4185.15238701054,-0.280242426288262,-0.518033957980719,-0.435989759138024,-4.86568550814886,-1.1925100147406,"1" +"69",34.5950464399618,-4.02497038533371,0.994097384352272,-0.998749217771909,852.655430140497,-0.00208617190785308,-0.642079499792139,-0.468555167597986,"1" +"70",-0.102194322423417,0.452003571795249,1365.10776175588,-4.11677560596058,0.181527371808562,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"71",12.2367890208301,-0.723428206559773,43.0388305349108,0.452003571795249,90.482463369223,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"72",110.861163172682,1.03953114431459,-335.736685209061,-0.723428206559773,2.62515863609837,3.1108481117673,-1.68275051412008,-4.11677560596058,"1" +"73",-0.628177222139383,-1.04373592786303,13438.170725121,-0.959567210136544,69.8151336775747,-0.722079885453127,-0.492979223942957,-2.2068409681768,"1" +"74",101.488096090733,-0.702269515361112,6.71624042801051,-1.04629924020543,1808.73483338493,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"75",-0.709495555124432,-0.476696519712976,1997.24425309081,-0.702269515361112,43.5301338993869,-1.04117261552062,330.585991082502,9.03592456211912,"1" +"76",108.990842284676,-0.280242426288262,-0.723877646928655,-0.484837871827967,694.410428504501,1.74893298153631,13.601631975026,-1.0360459908358,"1" +"77",-0.969399869933159,0.998749217771909,4329.08746853102,-0.280242426288262,-0.581854490361957,-0.468555167597986,-4.86568550814886,-1.1925100147406,"1" +"78",23.3195149542336,-4.02497038533371,-0.456737401451324,-0.998749217771909,2176.05854078715,0.832382591233372,-0.724327087297537,-0.484837871827967,"1" +"79",-0.102194322423417,0.452003571795249,151.762525791659,-4.11677560596058,1.21966887048428,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"80",14.4620390558734,-0.723428206559773,14.1370305266948,-2.2068409681768,232.161967689894,-4.02497038533371,-0.551579958120463,0.998749217771909,"1" +"81",120.848655350401,1.03953114431459,23.6411954504269,-0.723877646928655,4.71748612965091,0.452003571795249,44.890096926931,-4.02497038533371,"1" +"82",-0.169151119005015,-1.04373592786303,1395.25265722736,1.03953114431459,349.640325584266,-0.722978766190891,0.508407086200864,3.1108481117673,"1" +"83",98.8292515507612,-0.702269515361112,3.9620838092043,-1.04629924020543,562.295809605585,3.03862949876572,51.4568208884677,-0.722079885453127,"1" +"84",-0.70904611475555,-0.476696519712976,3395.79648111611,-1.1925100147406,55.4648125808804,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"85",108.990842284676,-0.00208617190785308,0.774556542923784,-0.484837871827967,1678.18290829416,0.768451982777342,-4.20858082658746,-1.04629924020543,"1" +"86",-0.954019995878704,0.998749217771909,988.594118243174,-0.280242426288262,-0.639832297947729,-0.476696519712976,247.724545789196,3.21965447967477,"1" +"87",23.3195149542336,-4.02497038533371,9.48378786241147,0.998749217771909,1050.56616723116,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"88",-0.0126394491585226,0.452003571795249,1055.27576614805,-4.11677560596058,-0.410597779287958,2.99624765331573,362.87633329997,1.6668513543746,"1" +"89",11.6804765120693,-0.723428206559773,34.7916408424255,0.452003571795249,729.265834060692,-3.84135994407996,-1.04886255254784,-0.998749217771909,"1" +"90",128.838649092576,1.03953114431459,888.428990319117,-0.723877646928655,6.87494444012337,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"91",-1.08720332527375,-1.04373592786303,3930.07817193246,1.03953114431459,505.129671782914,-0.722529325822009,-0.492979223942957,-2.2068409681768,"1" +"92",144.029608730286,-0.702269515361112,758.233776479598,-1.04629924020543,384.518448842185,1.03953114431459,25.3101329767093,-0.723428206559773,"1" +"93",-0.688821298155862,-0.484837871827967,606.668558685423,-1.1925100147406,4.78833079484616,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"94",142.975514310345,-0.00208617190785308,-0.722978766190891,-0.484837871827967,388.643306407715,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"95",-0.887373874976066,-0.998749217771909,11.0350229165703,-0.00208617190785308,-0.677585288933813,-0.476696519712976,481.702865306737,0.278211483397858,"1" +"96",18.9073504598183,-4.02497038533371,-0.415724403972777,-0.998749217771909,1364.42460887999,0.554226336852964,-0.724327087297537,-0.484837871827967,"1" +"97",0.0606327198763911,-2.2068409681768,992.524982227475,-4.11677560596058,-0.633605953077557,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"98",8.3426014595044,-0.723877646928655,-2.73999240768031,0.452003571795249,492.96991335978,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"99",90.8861788172437,1.03953114431459,919.860647064103,-0.723877646928655,0.907333339835394,0.452003571795249,43.4193754287925,-3.93316516470684,"1" +"100",0.565290646009974,-1.04373592786303,5262.40962844019,-0.959567210136544,112.929353106538,-0.722529325822009,0.117622184681324,11.0873817316835,"1" +"101",114.782318790594,-0.702269515361112,93.0131478172718,-1.04629924020543,374.530956664466,5.03772785321685,-0.55839868066867,-0.723877646928655,"1" +"102",-0.704102270697849,-0.476696519712976,4063.16646064909,-0.702269515361112,27.7396359515646,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"103",61.0124817778486,-0.00208617190785308,-0.673989765982758,-0.484837871827967,1337.85080717774,-0.212029015981627,4.42110991233867,-1.04117261552062,"1" +"104",-0.951456683536295,0.998749217771909,8173.35360414055,-0.00208617190785308,-0.59398938032177,-0.468555167597986,460.43210898696,0.768451982777342,"1" +"105",27.731679448649,-4.02497038533371,16.609796174309,-0.998749217771909,2048.11624610227,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"106",-0.118477026653398,0.452003571795249,2503.93644181443,-4.11677560596058,-0.172209731443905,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"107",16.4091328365362,-0.723428206559773,13.6811148082554,-2.2068409681768,143.918677801587,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"108",62.9212007196303,1.03953114431459,217.516104753572,-0.723428206559773,7.02963013030818,5.76969265173935,-1.68275051412008,-4.11677560596058,"1" +"109",1.29973241102496,-1.04373592786303,989.760474811962,-0.959567210136544,204.999073306453,-0.722529325822009,1.00502956521528,0.452003571795249,"1" +"110",93.5115624708171,-0.702269515361112,106.692125690676,-1.04373592786303,3578.51844727675,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"111",-0.70679891291114,-0.476696519712976,2481.15395936572,-0.702269515361112,61.5239571422541,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"112",96.996252157969,-0.280242426288262,-0.298257617597443,-0.484837871827967,2146.13954732924,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"113",-0.946330058851477,0.998749217771909,-0.959567210136544,-0.280242426288262,-0.559382471917859,-0.476696519712976,1157.04937845964,3.70989497905425,"1" +"114",14.4951859654029,-4.11677560596058,-1.16421160795626,-0.998749217771909,2743.80247345127,0.832382591233372,-0.724327087297537,-0.484837871827967,"1" +"115",-0.0614875618484651,0.452003571795249,-255.627329192693,-4.02497038533371,0.0866848151394224,14.9812382665786,-2.95866556458768,-0.280242426288262,"1" +"116",18.6343828715795,-0.723428206559773,3.18691193203271,-2.2068409681768,181.667196253807,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"117",74.9061913328932,-0.959567210136544,16.6872890909167,-0.723877646928655,4.69306207330594,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"118",-0.903592884020004,-1.04373592786303,1978.52220040615,1.03953114431459,413.89442034614,-0.722529325822009,-0.492979223942957,-2.2068409681768,"1" +"119",85.5350288509009,-0.702269515361112,29.7593508053558,-1.04629924020543,889.885553034771,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"120",-0.703203389960085,-0.476696519712976,1699.45366461394,-0.702269515361112,5.79818822174177,-1.04117261552062,256.678548967381,3.03862949876572,"1" +"121",63.0115801322997,-0.00208617190785308,-0.51129235244749,-0.484837871827967,774.175764703663,0.278211483397858,-4.20858082658746,-1.04629924020543,"1" +"122",-0.951456683536295,0.998749217771909,1096.54542938354,-0.280242426288262,-0.616461398765867,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"123",18.9073504598183,-4.02497038533371,2.89607514241988,-0.998749217771909,1418.40026445017,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"124",-0.16732513934334,0.452003571795249,56.1656284126591,-4.02497038533371,1.59903909716084,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"125",6.11735142446113,-0.723428206559773,0.0524913677614007,0.452003571795249,680.732024622123,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"126",96.8786741238752,1.03953114431459,1003.30752337823,-0.723428206559773,1.56678286114962,0.452003571795249,81.1678938810128,-4.02497038533371,"1" +"127",2.67681072042807,-1.04373592786303,1337.32520259659,-0.959567210136544,73.7093212389004,-0.721630445084245,-0.492979223942957,-2.2068409681768,"1" +"128",149.34729781023,-0.702269515361112,967.457874288243,-1.04629924020543,588.263289267654,5.03772785321685,-0.55839868066867,-0.723877646928655,"1" +"129",-0.703203389960085,-0.476696519712976,891.164924462433,-0.702269515361112,19.7525817570266,-1.03860930317821,911.858035825753,1.03953114431459,"1" +"130",112.989038993578,-0.00208617190785308,-0.172414314310496,-0.484837871827967,614.645092305339,-0.702269515361112,-4.20858082658746,-1.04629924020543,"1" +"131",-0.920696935427385,0.998749217771909,20489.798565914,-0.00208617190785308,-0.680281931147105,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"132",17.4366289616798,-4.02497038533371,16.9097037183709,0.998749217771909,738.706823936783,-0.00208617190785308,-0.538708214949289,-0.476696519712976,"1" +"133",-0.0696289139634555,0.452003571795249,-954.220040808459,-4.02497038533371,0.550644349115484,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"134",18.0780703628187,-0.723428206559773,17.0109278232864,0.452003571795249,64.9899574014898,-3.93316516470684,-0.815601129388609,20.9737335732101,"1" +"135",60.9237022840865,1.03953114431459,723.760487725915,-0.723428206559773,1.79474072036935,8.42853719171141,-1.68275051412008,-4.11677560596058,"1" +"136",1.94236895541308,-1.04629924020543,104.86866786605,-0.959567210136544,156.043572535501,-0.722079885453127,-0.492979223942957,-2.2068409681768,"1" +"137",152.006142350202,-0.702269515361112,186.287251974175,-1.04629924020543,516.353345588077,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"138",-0.701405628484557,-0.476696519712976,830.011500043076,-1.1925100147406,83.8326257545844,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"139",61.0124817778486,-0.00208617190785308,1.1404010031937,-0.476696519712976,837.988033662992,0.278211483397858,4.97194123609991,-1.02322942912375,"1" +"140",-0.920696935427385,0.998749217771909,2184.05493420495,-0.00208617190785308,-0.581854490361957,-0.476696519712976,651.868915864948,0.768451982777342,"1" +"141",21.8487934560952,-4.02497038533371,1.54520953797025,-0.998749217771909,1614.31190318638,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"142",0.0362086635314198,0.452003571795249,1047.92215865736,-4.11677560596058,1.21197893345705,4.99374608885955,308.900677729789,0.276070082472555,"1" +"143",16.4091328365362,-0.723428206559773,-0.484837871827967,-2.2068409681768,583.664405744985,-4.02497038533371,0.00978544486714703,0.998749217771909,"1" +"144",114.85616004377,1.03953114431459,131.287665895645,-0.723877646928655,8.5439216236964,0.452003571795249,127.250500822684,-4.02497038533371,"1" +"145",-1.08720332527375,-1.04373592786303,358.550969180115,-0.959567210136544,155.765416281121,-0.723428206559773,3.05665029819286,5.76969265173935,"1" +"146",96.1704070107891,-1.1925100147406,40.6841720599538,-1.04373592786303,654.1807376406,1.03953114431459,237.265198814581,-0.722978766190891,"1" +"147",-0.702753949591203,-0.476696519712976,8160.44589674603,-1.1925100147406,56.7500856696567,-1.0360459908358,-2.99624765331573,-0.959567210136544,"1" +"148",122.984530765834,-0.00208617190785308,1.32646931591083,-0.476696519712976,444.479041747128,-0.702269515361112,-4.20858082658746,-1.04629924020543,"1" +"149",-0.889937187318475,0.998749217771909,6276.20926576642,-0.00208617190785308,-0.285673287268748,-0.452272463368005,258.359923949084,-0.702269515361112,"1" +"150",22.8292744548542,-4.11677560596058,26.1581346497832,-0.998749217771909,2449.93501534695,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"151",-0.134759730883379,0.452003571795249,1252.3524468986,-4.02497038533371,-0.62335270370792,2.99624765331573,652.745594695384,0.554226336852964,"1" +"152",19.7470078891011,-0.723428206559773,18.4112403870648,-2.2068409681768,107.150640348126,-3.93316516470684,-0.882247250291247,22.9712320087539,"1" +"153",66.9161975907179,1.03953114431459,241.437542630287,-0.723877646928655,13.168209625011,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"154",-0.53637200151251,-1.04373592786303,550.310818992322,-0.959567210136544,92.6239465367682,-0.722978766190891,-0.492979223942957,-2.2068409681768,"1" +"155",183.912276829867,-0.702269515361112,236.96373376021,-1.04629924020543,278.651031758363,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"156",-0.703203389960085,-0.476696519712976,1422.93383245684,-0.702269515361112,64.7371398641947,-1.04117261552062,-2.99624765331573,-0.959567210136544,"1" +"157",96.996252157969,-0.00208617190785308,0.950737167525511,-0.484837871827967,2324.28213150737,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"158",-0.925823560112203,0.998749217771909,2331.98821243434,-0.00208617190785308,-0.496910260643267,-0.476696519712976,253.04223486914,1.74893298153631,"1" +"159",38.5169704349977,-4.02497038533371,1.91176320293476,-0.998749217771909,460.832152668075,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"160",-0.191749195688312,0.452003571795249,1993.59608196038,-4.11677560596058,0.278933240820111,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"161",19.4688516347207,-0.723428206559773,35.23941520875,-2.2068409681768,438.062977429278,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"162",102.871169430507,1.03953114431459,-130.457369476319,-0.723877646928655,3.86264415757691,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"163",0.19806976350248,-1.04373592786303,7341.8054998413,-0.959567210136544,157.434353807403,-0.723428206559773,0.288590579096123,5.76969265173935,"1" +"164",106.805785170677,-0.702269515361112,282.590928411766,-1.04373592786303,2401.99186874144,3.03862949876572,37.2708519150669,-0.722978766190891,"1" +"165",-0.69511346332021,-0.476696519712976,1561.19374853539,-1.1925100147406,29.4839351434752,-1.04373592786303,180.773608416716,3.03862949876572,"1" +"166",118.986334056931,-0.00208617190785308,-0.285673287268748,-0.484837871827967,803.423054643355,-0.702269515361112,68.5929591305234,-1.04373592786303,"1" +"167",-0.964273245248341,0.998749217771909,1988.14329546874,-0.280242426288262,-0.575112884828728,-0.476696519712976,261.018768489056,-0.212029015981627,"1" +"168",14.0049454660234,-4.02497038533371,-0.805347880018972,-0.998749217771909,1156.51838001707,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"169",-0.0370635055034939,0.452003571795249,-185.032697282047,-4.11677560596058,1.04023700651564,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"170",10.0115389857869,-0.723428206559773,15.6187566116231,-2.2068409681768,163.038057277387,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"171",114.85616004377,1.03953114431459,929.317959713037,-0.723877646928655,2.31578725572874,11.0873817316835,-1.68275051412008,-4.11677560596058,"1" +"172",-0.903592884020004,-1.04373592786303,1219.4727948995,-0.959567210136544,84.5574151597363,-0.722978766190891,0.500265734085873,0.452003571795249,"1" +"173",96.1704070107891,-0.702269515361112,176.4640933671,-1.04629924020543,750.060662546704,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"174",-0.702304509222321,-0.484837871827967,218.477255849504,-0.702269515361112,96.9607723042273,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"175",71.0079735501043,-0.00208617190785308,-0.688821298155862,-0.476696519712976,7155.40266063659,3.21965447967477,-4.20858082658746,-1.04629924020543,"1" +"176",-0.925823560112203,0.998749217771909,2627.8547688931,-0.280242426288262,-0.576011765566492,-0.452272463368005,-4.86568550814886,-1.1925100147406,"1" +"177",15.9659074635414,-4.02497038533371,1.85793364374417,0.998749217771909,202.948464943879,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"178",-0.199890547803302,-2.2068409681768,662.593126145082,-4.02497038533371,0.840298643807721,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"179",12.2367890208301,-0.723877646928655,-3.39130057687954,0.452003571795249,301.776118601781,-4.02497038533371,-0.136323358650176,2.99624765331573,"1" +"180",50.9362101063674,1.03953114431459,1611.63525170818,-0.723428206559773,8.42180134197154,3.1108481117673,-1.68275051412008,-4.11677560596058,"1" +"181",-0.995398104646878,-1.04373592786303,2148.30956742738,1.03953114431459,37.5490081694473,-0.719383243239835,-0.492979223942957,-2.2068409681768,"1" +"182",90.852717930845,-0.702269515361112,5.24735689798053,-1.04373592786303,2663.66416379768,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"183",-0.704102270697849,-0.476696519712976,747.587319303942,-1.1925100147406,60.146878832851,-1.04117261552062,306.616009855976,5.03772785321685,"1" +"184",106.991743930225,-0.00208617190785308,0.267587806824939,-0.484837871827967,689.092739424557,-0.702269515361112,4.23749947108492,-1.0360459908358,"1" +"185",-0.959146620563523,0.998749217771909,-0.959567210136544,-0.00208617190785308,-0.654663830120834,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"186",17.4366289616798,-4.02497038533371,-0.587466330914192,-0.998749217771909,632.754611150873,0.832382591233372,-0.724327087297537,-0.484837871827967,"1" +"187",-0.0696289139634555,0.452003571795249,731.717036557589,-4.11677560596058,3.94190657812283,6.99124452440336,-2.95866556458768,-0.280242426288262,"1" +"188",19.4688516347207,-0.723428206559773,4.83146505926077,-2.2068409681768,178.72575325753,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"189",92.8836772527875,1.03953114431459,435.868764442192,-0.723428206559773,2.67400674878831,0.452003571795249,127.250500822684,-3.47413906157247,"1" +"190",0.748901087263722,-1.04373592786303,3958.04315003008,1.03953114431459,44.2247582745771,-0.723428206559773,-0.492979223942957,-2.2068409681768,"1" +"191",128.076541490454,-1.1925100147406,153.604593431008,-1.04629924020543,420.473420681974,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"192",-0.708596674386668,-0.476696519712976,978.906794281511,-1.1925100147406,75.5701558981658,-1.03860930317821,682.145715738214,7.03682620766799,"1" +"193",112.989038993578,-0.280242426288262,-0.512640673554136,-0.484837871827967,1274.03853821841,0.278211483397858,-4.20858082658746,-1.04629924020543,"1" +"194",-0.951456683536295,0.998749217771909,7441.68360641143,-0.280242426288262,-0.564775756344443,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"195",20.3780719579567,-4.02497038533371,18.9936766527495,-0.998749217771909,822.66895482373,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"196",-0.0696289139634555,0.452003571795249,1481.7850006082,-4.11677560596058,0.173837434781335,0.998749217771909,702.723053556662,-0.00208617190785308,"1" +"197",10.0115389857869,-0.723428206559773,1.05387767790522,-2.2068409681768,115.484728837577,-4.02497038533371,-0.772024819567653,2.99624765331573,"1" +"198",110.861163172682,1.03953114431459,2605.209392355,-0.723877646928655,2.12039480496897,3.1108481117673,48.3217804225874,-3.93316516470684,"1" +"199",0.0144593222487325,-1.04629924020543,1684.88993038121,-0.959567210136544,282.882824532968,-0.722529325822009,-0.492979223942957,-2.2068409681768,"1" +"200",165.300365050063,-0.702269515361112,354.290805721354,-1.04629924020543,718.100687578003,1.03953114431459,26.4227579942309,-0.722079885453127,"1" +"201",-0.710394435862196,-0.476696519712976,518.926688866346,-1.1925100147406,2.12597939666683,-1.04373592786303,200.748592772154,7.03682620766799,"0" +"202",67.009776841202,-0.00208617190785308,0.169609806408672,-0.476696519712976,138.711919650342,-0.702269515361112,20.3034130807878,-1.04117261552062,"0" +"203",-0.928386872454613,0.998749217771909,18628.63799792,-0.00208617190785308,-0.709944995493314,-0.427848407023034,-4.86568550814886,-1.1925100147406,"0" +"204",12.534223967885,-4.02497038533371,-1.02835605380857,0.998749217771909,450.83666089582,0.832382591233372,-0.724327087297537,-0.484837871827967,"0" +"205",-0.15104243511336,0.452003571795249,5115.93782250832,-4.02497038533371,-0.684872199925741,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"206",7.22997644198277,-0.723428206559773,7.2738706937579,0.452003571795249,165.489259774284,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"207",88.8886803816999,1.03953114431459,565.211422729082,-0.723877646928655,0.516548438315854,0.452003571795249,89.9922228698435,-3.74955472345309,"0" +"208",-0.81178766339313,-1.04373592786303,6245.17885872775,1.03953114431459,40.6087269676318,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"209",138.711919650342,-1.1925100147406,-80.6823296087732,-1.04373592786303,258.676047402924,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"210",-0.709495555124432,-0.476696519712976,566.785890585842,-0.702269515361112,13.3262163131454,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"211",73.0070719045554,-0.00208617190785308,0.0761262096812256,-0.476696519712976,542.856289726094,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"212",-0.961709932905932,0.998749217771909,3269.56534067192,-0.00208617190785308,-0.696012344057974,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"213",17.9268694610593,-4.02497038533371,8.45077298842057,0.998749217771909,152.971006082601,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"214",-0.15104243511336,0.452003571795249,-340.438935585344,-4.02497038533371,-0.725885197404287,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"215",13.9057265471126,-0.723428206559773,7.5262526093226,0.452003571795249,171.862386266217,-3.74955472345309,-1.04886255254784,-0.998749217771909,"0" +"216",104.86866786605,1.03953114431459,2399.3737641135,-0.723877646928655,0.0524913677614007,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"217",-1.82164509028874,-1.04373592786303,282.64602862945,1.03953114431459,86.5045089403992,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"218",93.5115624708171,-0.702269515361112,658.900527761321,-1.04373592786303,230.711069305311,7.03682620766799,42.5558207482946,-0.723428206559773,"0" +"219",-0.703203389960085,-0.476696519712976,5865.86305875014,-0.702269515361112,18.1918930063697,-1.03860930317821,-2.99624765331573,-0.959567210136544,"0" +"220",83.0025636768111,-0.00208617190785308,0.621297377135037,-0.476696519712976,359.396016468022,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"221",-0.933513497139431,0.998749217771909,6178.25344639832,-0.00208617190785308,-0.575112884828728,-0.452272463368005,-4.86568550814886,-1.1925100147406,"0" +"222",17.9268694610593,-4.02497038533371,-0.331135096673274,0.998749217771909,720.714938746722,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"223",-0.191749195688312,0.452003571795249,278.734815130945,-4.02497038533371,-0.928386872454613,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"224",12.7931015295909,-0.723428206559773,27.6842404460389,0.452003571795249,191.962246740776,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"225",68.9136960262617,1.03953114431459,336.845137882767,-0.723428206559773,1.42837987519478,0.452003571795249,171.372145766838,-4.02497038533371,"0" +"226",-1.2708137665275,-1.04373592786303,12259.6466481502,-0.959567210136544,66.7554148793902,-0.721630445084245,-0.492979223942957,-2.2068409681768,"0" +"227",106.805785170677,-1.1925100147406,-38.6355385616651,-1.04373592786303,839.948092146176,5.03772785321685,65.9209461162489,-0.719832683608717,"0" +"228",-0.708147234017786,-0.476696519712976,9043.18228401675,-1.1925100147406,20.5788287426684,-1.04373592786303,386.515947277729,3.03862949876572,"0" +"229",100.994448866871,-0.00208617190785308,-0.665450398974001,-0.484837871827967,1452.18112239654,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"230",-0.961709932905932,0.998749217771909,464.830349376977,-0.00208617190785308,-0.531517169047178,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"231",20.3780719579567,-4.02497038533371,0.819792145068448,0.998749217771909,41.0214982333372,2.77947637189623,-0.724327087297537,-0.484837871827967,"0" +"232",0.00364325507145819,0.452003571795249,409.629028465268,-4.11677560596058,-0.669492325871285,0.998749217771909,-2.95866556458768,-0.280242426288262,"0" +"233",8.62075771388481,-0.723428206559773,-0.75350249162265,0.452003571795249,113.523766840059,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"234",74.9061913328932,1.03953114431459,71.4840712038571,-0.723877646928655,0.988746860985298,8.42853719171141,-1.68275051412008,-4.11677560596058,"0" +"235",0.106264542875606,-1.04629924020543,2.99624765331573,1.03953114431459,51.7349771428481,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"236",88.193873390873,-0.702269515361112,36.185716249237,-1.04373592786303,692.133207915933,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"237",-0.705001151435613,-0.476696519712976,-2.2068409681768,-0.702269515361112,2.49320027917432,-1.04373592786303,306.616009855976,3.03862949876572,"0" +"238",73.0070719045554,-0.00208617190785308,-0.0978072130760911,-0.484837871827967,340.784104688218,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"239",-0.920696935427385,-0.998749217771909,59.0133834233974,-0.280242426288262,-0.526573324989476,-0.468555167597986,-4.86568550814886,-1.1925100147406,"0" +"240",14.4951859654029,-4.02497038533371,2.11170156564267,-0.998749217771909,192.952973171623,0.832382591233372,-0.724327087297537,-0.484837871827967,"0" +"241",-0.183607843573321,-2.2068409681768,17.9268694610593,-4.11677560596058,-0.856614126867156,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"242",15.0183515646342,-0.723428206559773,9.38248089154041,-2.2068409681768,116.955450335715,-4.02497038533371,-0.81047450470379,0.998749217771909,"0" +"243",82.8961850750684,1.03953114431459,131.287665895645,-0.723428206559773,1.27369418500996,11.0873817316835,-1.68275051412008,-4.11677560596058,"0" +"244",-1.08720332527375,-1.04373592786303,11266.8899256849,-0.959567210136544,46.4500083096204,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"245",154.664986890174,-0.702269515361112,-12.3792454623792,-1.04629924020543,80.8986866395246,9.03592456211912,-0.55839868066867,-0.723877646928655,"0" +"246",-0.705900032173377,-0.476696519712976,5190.51654559724,-0.702269515361112,6.16540910424927,-1.03348267849339,-2.99624765331573,-0.959567210136544,"0" +"247",63.0115801322997,-0.280242426288262,-0.723877646928655,-0.476696519712976,614.645092305339,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"248",-0.910443686057748,-0.998749217771909,1116.53641292805,-0.280242426288262,-0.716686601026544,-0.444131111253015,-4.86568550814886,-1.1925100147406,"0" +"249",23.8097554536131,-4.11677560596058,4.90827533121109,0.998749217771909,300.904284311985,2.50132011751582,-0.724327087297537,-0.484837871827967,"0" +"250",-0.224314604148273,0.452003571795249,64.0094764027309,-4.02497038533371,0.0328552559488296,0.998749217771909,-2.95866556458768,-0.280242426288262,"0" +"251",11.1241640033085,-0.723428206559773,21.635215824601,0.452003571795249,272.851929138391,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"252",84.8936835106123,-0.959567210136544,342.964575479136,-0.723428206559773,2.01455722747409,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"253",-1.54622942840812,-1.04373592786303,1602.99249452391,1.03953114431459,89.8423839929641,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"254",77.5584952309848,-0.702269515361112,63.7272824372991,-1.04629924020543,324.59349577587,3.03862949876572,-0.55839868066867,-0.723877646928655,"0" +"255",-0.706349472542259,-0.476696519712976,2919.86330846111,-1.1925100147406,15.2541259463098,-1.04117261552062,488.388367490464,11.0350229165703,"0" +"256",71.0079735501043,-0.00208617190785308,-0.605225389543819,-0.476696519712976,1279.35622729835,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"257",-0.900190436688112,0.998749217771909,3217.58878345619,-0.280242426288262,-0.657809912703007,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"258",15.9659074635414,-4.11677560596058,-0.777151444252471,0.998749217771909,234.934038615097,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"259",-0.0126394491585226,0.452003571795249,-0.212029015981627,-4.02497038533371,-0.323445159646046,16.9787367021225,-2.95866556458768,-0.280242426288262,"0" +"260",9.17707022264563,-0.723428206559773,-0.484837871827967,-2.2068409681768,58.6168309095565,-4.02497038533371,-0.861740751551974,0.998749217771909,"0" +"261",90.8861788172437,-0.959567210136544,5.83919517008072,-0.723877646928655,-0.419707054908043,24.3816044315437,-1.68275051412008,-4.11677560596058,"0" +"262",1.11612196977122,-1.04373592786303,13975.4978042823,-0.959567210136544,32.2640393362195,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"263",77.5584952309848,-0.702269515361112,1.57514807290558,-1.04629924020543,424.468417553061,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"264",-0.699158426640147,-0.476696519712976,5073.52738583847,-1.1925100147406,11.2146962387273,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"265",104.992645575774,-0.00208617190785308,2.48153106393745,-0.484837871827967,231.771478549364,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"266",-0.954019995878704,0.998749217771909,1904.18116458179,-0.00208617190785308,-0.564775756344443,-0.468555167597986,888.506079922461,0.278211483397858,"0" +"267",13.5147049666439,-4.02497038533371,2.38341267393805,0.998749217771909,160.967399500405,1.38869509999419,-0.724327087297537,-0.484837871827967,"0" +"268",-0.0126394491585226,0.452003571795249,713.087897581169,-4.02497038533371,-0.677182262898513,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"269",15.8528203277754,-0.723428206559773,38.9193063647256,-2.2068409681768,178.235512758151,-4.02497038533371,-1.04886255254784,-0.998749217771909,"0" +"270",112.858661608226,1.03953114431459,292.340137181901,-0.723428206559773,1.36324905827486,8.42853719171141,-1.68275051412008,-4.11677560596058,"0" +"271",-0.719982442766257,-1.04629924020543,-0.998749217771909,1.03953114431459,4.44841389817868,-0.720731564346481,-0.492979223942957,-2.2068409681768,"0" +"272",90.852717930845,-0.702269515361112,12.5917745481304,-1.04629924020543,756.053157853335,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"273",-0.700506747746793,-0.484837871827967,1319.23889539793,-1.1925100147406,21.3132705076834,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"274",102.993547221322,-0.280242426288262,-0.632191811676736,-0.484837871827967,354.078327388078,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"275",-0.948893371193886,0.998749217771909,1986.14419711429,-0.280242426288262,-0.670394243031702,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"276",17.4366289616798,-4.02497038533371,5.48502060825315,0.998749217771909,118.986334056931,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"277",-0.0696289139634555,-2.2068409681768,158.135652283592,-4.02497038533371,-0.395217905233503,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"278",10.5678514945477,-0.723877646928655,2.62515863609837,-2.2068409681768,97.3458303605358,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"279",106.866166301594,1.03953114431459,1102.60930619203,-0.723428206559773,0.63866872004071,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"280",-1.08720332527375,-1.04373592786303,-0.998749217771909,-0.959567210136544,58.4107272479779,-0.722529325822009,-0.492979223942957,-2.2068409681768,"0" +"281",80.2173397709568,-0.702269515361112,56.8418908902835,-1.04373592786303,1970.53220666398,5.03772785321685,-0.55839868066867,-0.723877646928655,"0" +"282",-0.704551711066731,-0.476696519712976,239.74801216928,-0.702269515361112,-0.719982442766257,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"283",77.0052686134576,-0.00208617190785308,-0.714888839551016,-0.476696519712976,287.607213888777,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"284",-0.913006998400158,-0.998749217771909,852.655430140497,-0.280242426288262,-0.707697793648904,-0.460413815482995,-4.86568550814886,-1.1925100147406,"0" +"285",23.8097554536131,-4.02497038533371,6.4744591724231,0.998749217771909,1452.38493647584,0.832382591233372,-0.724327087297537,-0.484837871827967,"0" +"286",-0.126618378768388,-2.2068409681768,-1.1925100147406,-4.11677560596058,-0.866867376236792,18.9762351376663,-2.95866556458768,-0.280242426288262,"0" +"287",13.0712577839713,-0.723428206559773,7.7786345248873,-2.2068409681768,47.8315399232079,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"288",122.846153785945,-0.959567210136544,450.054733415593,-0.723877646928655,3.86264415757691,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"289",-0.169151119005015,-1.04629924020543,48.9387116708235,1.03953114431459,65.3646336074881,-0.720731564346481,-0.492979223942957,-2.2068409681768,"0" +"290",122.75885241051,-0.702269515361112,72.9996097206133,-1.04629924020543,1015.72795447403,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"291",-0.698259545902383,-0.476696519712976,20949.4881340116,-1.1925100147406,25.0772845533852,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"292",67.009776841202,-0.00208617190785308,-0.747248546110516,-0.484837871827967,29.6992935114878,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"293",-0.943766746509068,-0.998749217771909,-0.959567210136544,-0.280242426288262,-0.693765142213564,-0.468555167597986,-4.86568550814886,-1.1925100147406,"0" +"294",20.8683124573362,-4.02497038533371,-0.928386872454613,0.998749217771909,466.829447731429,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"295",-0.0289221533885035,0.452003571795249,1542.57482253126,-4.11677560596058,-0.682308887583331,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"296",11.1241640033085,-0.723428206559773,14.837186808584,0.452003571795249,79.6971723828744,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"297",92.8836772527875,1.03953114431459,41.7213519851534,-0.723428206559773,1.06201903002021,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"298",-1.08720332527375,-1.04373592786303,985.765477940874,1.03953114431459,43.9466020201967,-0.723428206559773,2.33206995995872,3.1108481117673,"0" +"299",114.782318790594,-1.1925100147406,31.5036499972664,-1.04373592786303,490.385865926007,3.03862949876572,102.637571694463,-0.722978766190891,"0" +"300",-0.701405628484557,-0.484837871827967,354.078327388078,-1.1925100147406,20.0279974189072,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"301",132.980022538089,-0.280242426288262,0.0608452371392392,-0.484837871827967,354.078327388078,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"302",-0.938640121824249,0.998749217771909,29.0269081066305,-0.00208617190785308,-0.620506362085805,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"303",18.4171099604388,-4.02497038533371,-0.774588131910062,0.998749217771909,398.86010368009,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"304",-0.126618378768388,0.452003571795249,541.01348229897,-4.02497038533371,-0.590029643256601,8.98874295994718,236.933136969548,2.77947637189623,"0" +"305",15.0183515646342,-0.723428206559773,0.809637114455509,0.452003571795249,89.0117418710846,-4.02497038533371,-1.04886255254784,-0.998749217771909,"0" +"306",114.85616004377,1.03953114431459,37.5490081694473,-0.723428206559773,1.13529119905513,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"307",0.840706307890596,-1.04373592786303,152.808630319102,-0.959567210136544,49.7878833621853,-0.722079885453127,-0.492979223942957,-2.2068409681768,"0" +"308",120.100007870538,-0.702269515361112,12.0409432243692,-1.04629924020543,260.673545838468,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"309",-0.707697793648904,-0.476696519712976,58.9465834511804,-0.702269515361112,9.83761792932422,-1.04117261552062,692.133207915933,1.03953114431459,"0" +"310",73.0070719045554,-0.00208617190785308,-0.623652444667979,-0.484837871827967,61.6054279911525,3.70989497905425,-4.20858082658746,-1.04629924020543,"0" +"311",-0.964273245248341,0.998749217771909,5324.63844904768,-0.00208617190785308,-0.615113077659222,-0.427848407023034,-4.86568550814886,-1.1925100147406,"0" +"312",17.9268694610593,-4.02497038533371,3.04218394593721,0.998749217771909,202.948464943879,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"313",-0.16732513934334,0.452003571795249,703.773328092958,-4.02497038533371,-0.502877023614689,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"314",16.1309765821558,-0.723428206559773,8.13685401794688,-2.2068409681768,49.3022614213463,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"315",72.9086928973494,1.03953114431459,746.569300585108,-0.723877646928655,1.03759497367524,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"316",-1.2708137665275,-1.04373592786303,2653.67667161996,1.03953114431459,36.9926956606865,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"317",114.782318790594,-0.702269515361112,39.5825094124313,-1.04373592786303,943.818010794454,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"318",-0.706349472542259,-0.476696519712976,2055.73883297019,-0.702269515361112,-1.45442420778125,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"319",96.996252157969,-0.00208617190785308,-0.452865104492836,-0.476696519712976,1566.51143761533,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"320",-0.925823560112203,0.998749217771909,1832.21362382155,-0.280242426288262,-0.491067535847802,-0.460413815482995,561.468201505898,1.25869248215683,"0" +"321",20.3780719579567,-4.02497038533371,4.12390175443388,0.998749217771909,146.973711019247,0.554226336852964,-0.724327087297537,-0.484837871827967,"0" +"322",-0.110335674538408,0.452003571795249,6.16109747595167,-4.11677560596058,-0.487497149560234,8.98874295994718,-2.95866556458768,-0.280242426288262,"0" +"323",14.7401953102538,-0.723877646928655,21.9934353176606,-2.2068409681768,376.782915006842,-4.02497038533371,-1.04886255254784,-0.998749217771909,"0" +"324",50.9362101063674,1.03953114431459,793.855863829778,-0.723877646928655,-0.0614875618484651,5.76969265173935,-1.68275051412008,-4.11677560596058,"0" +"325",-0.995398104646878,-1.04373592786303,827.963101532913,-0.959567210136544,100.134165405039,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"326",149.34729781023,-0.702269515361112,155.716113505426,-1.04629924020543,726.090681320178,3.03862949876572,-0.55839868066867,-0.723877646928655,"0" +"327",-0.707248353280022,-0.476696519712976,-2.2068409681768,-0.702269515361112,20.8542444045491,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"328",87.0007603857133,-0.00208617190785308,-0.643877261267667,-0.484837871827967,332.807571068302,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"329",-0.913006998400158,0.998749217771909,-980.517760891192,-0.00208617190785308,-0.69061905963139,-0.403424350678062,-4.86568550814886,-1.1925100147406,"0" +"330",15.9659074635414,-4.11677560596058,-2.88675750205523,0.998749217771909,92.9980554490667,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"331",-0.0452048576184843,-2.2068409681768,3474.12239008643,-4.11677560596058,-0.943766746509068,0.998749217771909,-2.95866556458768,-0.280242426288262,"0" +"332",9.45522647702603,-0.723428206559773,-0.484837871827967,-2.2068409681768,236.08389168493,-3.74955472345309,-1.04886255254784,-0.998749217771909,"0" +"333",60.9237022840865,1.03953114431459,389.138513706284,-0.723428206559773,0.939898748295356,3.1108481117673,-0.212029015981627,-4.02497038533371,"0" +"334",-0.903592884020004,-1.04373592786303,2322.09193131969,1.03953114431459,78.7161338177477,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"335",93.5115624708171,-0.702269515361112,80.7112482532707,-1.04373592786303,344.568480131309,7.03682620766799,-0.55839868066867,-0.723877646928655,"0" +"336",-0.700057307377911,-0.476696519712976,-967.367408978032,-0.702269515361112,14.336073740041,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"337",87.0007603857133,-0.00208617190785308,-0.600730985854999,-0.476696519712976,1085.26057588039,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"338",-0.920696935427385,0.998749217771909,9420.79097731805,-0.00208617190785308,-0.70904611475555,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"339",28.712160447408,-4.02497038533371,-0.597719580283828,-0.998749217771909,156.969202791503,0.276070082472555,-0.641630059423257,-0.468555167597986,"0" +"340",-0.0533462097334747,0.452003571795249,99.7970328574332,-4.11677560596058,-0.823291066415836,10.986241395491,-2.95866556458768,-0.280242426288262,"0" +"341",8.06444520512399,-0.723428206559773,5.09198832694047,0.452003571795249,11.0635024697465,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"342",62.9212007196303,1.03953114431459,-127.675806932515,-0.723877646928655,0.679375480615662,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"343",-0.0773458983781412,-1.04373592786303,27670.3470783707,1.03953114431459,369.389419645275,-0.722529325822009,-0.492979223942957,-2.2068409681768,"0" +"344",112.123474250621,-1.1925100147406,111.5578023839,-1.04629924020543,32.958724186473,7.03682620766799,-0.55839868066867,-0.723877646928655,"0" +"345",-0.707248353280022,-0.476696519712976,1252.76778189863,-0.702269515361112,22.8739592583403,-1.03860930317821,-2.99624765331573,-0.959567210136544,"0" +"346",106.991743930225,-0.00208617190785308,-0.886125620095041,-0.476696519712976,601.350869605479,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"347",-0.964273245248341,0.998749217771909,5102.73853170361,-0.00208617190785308,-0.517584517611837,-0.427848407023034,-4.86568550814886,-1.1925100147406,"0" +"348",20.3780719579567,-4.11677560596058,0.332762800010704,-0.998749217771909,390.863710262286,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"349",-0.142901082998369,0.452003571795249,4.20013547843373,-4.11677560596058,-0.761771570198016,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"350",11.4023202576889,-0.723428206559773,1.7540339597944,0.452003571795249,66.9509193990078,-3.56594428219934,-1.04886255254784,-0.998749217771909,"0" +"351",78.9011882039808,1.03953114431459,156.321728789882,-0.723428206559773,4.52209367889114,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"352",-1.17900854590063,-1.04373592786303,3135.07379458602,-0.959567210136544,43.1121332570555,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"353",149.34729781023,-0.702269515361112,455.827379734677,-1.04629924020543,1137.5753590422,7.03682620766799,-0.55839868066867,-0.723877646928655,"0" +"354",-0.704551711066731,-0.484837871827967,420.54944088738,-1.1925100147406,6.16540910424927,-1.0360459908358,-2.99624765331573,-0.959567210136544,"0" +"355",81.0034653223599,-0.00208617190785308,-0.648371664956486,-0.476696519712976,494.997088006597,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"356",-0.895063812003293,0.998749217771909,1422.39846115907,-0.00208617190785308,-0.674888646720522,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"357",22.3390339554747,-4.02497038533371,2.9575946386377,-0.998749217771909,98.9953505124201,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"358",-0.102194322423417,0.452003571795249,974.386083750434,-4.02497038533371,-0.769461507225243,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"359",8.89891396826522,-0.723428206559773,-0.484837871827967,0.452003571795249,20.3780719579567,-3.38233384094559,-1.04886255254784,-0.998749217771909,"0" +"360",80.8986866395246,1.03953114431459,-4.45258624199439,-0.723428206559773,0.891050635605413,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"361",-0.903592884020004,-1.04373592786303,1421.22013688943,1.03953114431459,73.7093212389004,-0.722079885453127,-0.492979223942957,-2.2068409681768,"0" +"362",133.394230570398,-1.1925100147406,-0.0773458983781412,-1.04629924020543,272.658536451731,3.03862949876572,-0.55839868066867,-0.723877646928655,"0" +"363",-0.705900032173377,-0.476696519712976,819.376121883188,-0.702269515361112,16.1721781525785,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"364",75.0061702590065,-0.00208617190785308,-0.741855261683933,-0.476696519712976,601.350869605479,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"365",-0.90531706137293,0.998749217771909,1904.18116458179,-0.00208617190785308,-0.66769760081841,-0.460413815482995,-4.86568550814886,-1.1925100147406,"0" +"366",17.9268694610593,-4.11677560596058,14.0080341467637,-0.998749217771909,558.787972036181,-0.00208617190785308,-0.569270160033262,-0.468555167597986,"0" +"367",-0.126618378768388,0.452003571795249,30.6731224449259,-4.11677560596058,-0.910443686057748,20.9737335732101,-2.95866556458768,-0.280242426288262,"0" +"368",10.2896952401673,-0.723428206559773,54.1110694112977,-2.2068409681768,68.9118813965257,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"369",84.8936835106123,-0.959567210136544,316.261575058617,-0.723877646928655,0.0443500156464103,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"370",0.473485425383101,-1.04373592786303,1772.77986154514,-0.959567210136544,44.2247582745771,-0.722529325822009,-0.492979223942957,-2.2068409681768,"0" +"371",104.146940630705,-0.702269515361112,80.8030534738976,-1.04373592786303,42.9462163641921,15.0332196254725,-0.55839868066867,-0.723877646928655,"0" +"372",-0.699607867009029,-0.476696519712976,258.359923949084,-1.1925100147406,34.4414170573264,-1.0360459908358,-2.99624765331573,-0.959567210136544,"0" +"373",87.0007603857133,-0.00208617190785308,-0.120728671889071,-0.476696519712976,295.583747508693,-0.212029015981627,29.8511560259827,-1.04117261552062,"0" +"374",-0.910443686057748,0.998749217771909,270.917808995218,-0.280242426288262,-0.609719793232638,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"375",16.4561479629209,-4.02497038533371,3.05756381999166,0.998749217771909,324.893464565398,0.554226336852964,-0.724327087297537,-0.484837871827967,"0" +"376",-0.0452048576184843,0.452003571795249,416.002154957201,-4.02497038533371,-0.769461507225243,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"377",10.8460077489281,-0.723428206559773,-0.484837871827967,0.452003571795249,80.6776533816333,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"378",104.86866786605,1.03953114431459,-21.4201177591993,-0.723877646928655,0.923616044065375,0.452003571795249,79.2069318834949,-3.74955472345309,"0" +"379",0.473485425383101,-1.04373592786303,3093.1263274396,1.03953114431459,123.499290772993,-0.722978766190891,1.94942641055417,13.7462262716555,"0" +"380",130.735386030426,-0.702269515361112,168.293428731308,-1.04373592786303,88.8886803816999,3.03862949876572,50.0660396165657,-0.718933802870953,"0" +"381",-0.709495555124432,-0.476696519712976,-672.235665041134,-0.702269515361112,0.657095866636848,-1.0360459908358,-2.99624765331573,-0.959567210136544,"0" +"382",94.9971538035178,-0.00208617190785308,2.19254090674635,-0.476696519712976,697.069273044473,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"383",-0.930950184797022,0.998749217771909,4692.92336904112,-0.00208617190785308,-0.502752985438733,-0.476696519712976,391.302150947687,-0.702269515361112,"0" +"384",20.3780719579567,-4.02497038533371,8.34824049472421,-0.998749217771909,648.747397986482,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"385",-0.118477026653398,-2.2068409681768,10.0830214709875,-4.02497038533371,-0.887373874976066,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"386",7.78628895074358,-0.723428206559773,-1.41295201293687,0.452003571795249,63.5192359033514,-3.93316516470684,-0.126070109280539,2.99624765331573,"0" +"387",64.9186991551741,1.03953114431459,99.299696641898,-0.723877646928655,1.39581446673482,0.452003571795249,170.391664768079,-4.02497038533371,"0" +"388",-0.81178766339313,-1.04373592786303,2735.57410747726,-0.959567210136544,61.192289791782,-0.721630445084245,-0.492979223942957,-2.2068409681768,"0" +"389",96.1704070107891,-0.702269515361112,12.0409432243692,-1.04373592786303,292.633520807169,3.03862949876572,96.2399778437135,-0.723428206559773,"0" +"390",-0.705450591804495,-0.476696519712976,-2.2068409681768,-1.1925100147406,73.917661926882,-1.03860930317821,-2.99624765331573,-0.959567210136544,"0" +"391",61.0124817778486,-0.280242426288262,0.0702834848857602,-0.484837871827967,1048.03675232078,1.74893298153631,-4.20858082658746,-1.04629924020543,"0" +"392",-0.900190436688112,-0.998749217771909,1818.21993534039,-0.00208617190785308,-0.641630059423257,-0.468555167597986,-4.86568550814886,-1.1925100147406,"0" +"393",21.8487934560952,-4.11677560596058,-1.17190154498348,0.998749217771909,27.0278097521793,1.38869509999419,-0.576910646304255,-0.297586773183187,"0" +"394",-0.142901082998369,0.452003571795249,121.367614830131,-4.02497038533371,-0.928386872454613,30.9612257509292,-2.95866556458768,-0.280242426288262,"0" +"395",9.17707022264563,-0.723428206559773,19.7871288944982,0.452003571795249,89.5019823704641,-3.93316516470684,-0.584903018571783,0.998749217771909,"0" +"396",102.871169430507,1.03953114431459,228.086042420027,-0.723428206559773,2.98337812915795,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"397",-0.444566780885636,-1.04373592786303,7375.76297324555,1.03953114431459,15.574664073395,-0.719383243239835,-0.492979223942957,-2.2068409681768,"0" +"398",93.5115624708171,-0.702269515361112,267.351261787705,-1.04373592786303,384.518448842185,3.03862949876572,73.43116498452,-0.722529325822009,"0" +"399",-0.700506747746793,-0.484837871827967,526.903222486262,-0.702269515361112,30.1265716878633,-1.04373592786303,470.410881570569,3.03862949876572,"0" +"400",77.0052686134576,-0.00208617190785308,-0.455112306337246,-0.476696519712976,1178.32013477941,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" diff --git a/example/HamiltonianVI/hamiltonian_layer.jl b/example/HamiltonianVI/hamiltonian_layer.jl new file mode 100644 index 00000000..aabb4afe --- /dev/null +++ b/example/HamiltonianVI/hamiltonian_layer.jl @@ -0,0 +1,112 @@ +using Functors +using Flux +using Bijectors +using Bijectors: partition, combine, PartitionMask +using SimpleUnPack: @unpack + +struct LeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector + "dimention of the target space" + dim::I + "leapfrog step size" + ϵ::AbstractVector{T} + "number of leapfrog steps" + L::I + "score of the target distribution" + ∇logp + "score of the momentum distribution" + ∇logm +end +@functor LeapFrog (ϵ,) + +function LeapFrog(dim::Int, ϵ::T, L::Int, ∇logp, ∇logm) where {T<:Real} + return LeapFrog(dim, ϵ .* ones(T, dim), L, ∇logp, ∇logm) +end + +function Bijectors.inverse(lf::LeapFrog) + @unpack d, ϵ, L, ∇logp, ∇logm = lf + return LeapFrog(d, -ϵ, L, ∇logp, ∇logm) +end + +function Bijectors.transform(lf::LeapFrog, z::AbstractVector) + @unpack dim, ϵ, L, ∇logp, ∇logm = lf + @assert length(z) == 2dim "dimension of input must be even, z = [x, ρ]" + # mask = PartitionMask(n, 1:dim) + # x, ρ, emp = partition(mask, z) + x, ρ = z[1:dim], z[(dim + 1):end] + + ρ += ϵ ./ 2 .* ∇logp(x) + for i in 1:(L - 1) + x -= ϵ .* ∇logm(ρ) + ρ += ϵ .* ∇logp(x) + end + x -= ϵ .* ∇logm(ρ) + ρ += ϵ ./ 2 .* ∇logp(x) + # return combine(mask, x, ρ, emp) + return vcat(x, ρ) +end + +function Bijectors.with_logabsdet_jacobian(lf::LeapFrog, z::AbstractVector) + return Bijectors.transform(lf, z), zero(eltype(z)) +end + +abstract type TrainableScore end +struct CoresetScore{T<:AbstractVector} <: TrainableScore + "coreset weights" + w::T + "weighted coreset score function of the target, ∇logpw(x, w)" + ∇logpw +end +@functor CoresetScore (w,) +function CoresetScore(T, coresize::Int, datasize::Int, ∇logpw) + return CoresetScore(ones(T, coresize) .* N ./ coresize, ∇logpw) +end +(C::CoresetScore)(x::AbstractVector) = C.∇logpw(x, C.w) + +struct SurrogateLeapFrog{T<:Real,I<:Int,H<:Union{TrainableScore,Flux.Chain}} <: + Bijectors.Bijector + "dimention of the target space" + dim::I + "leapfrog step size" + ϵ::AbstractVector{T} + "number of leapfrog steps" + L::I + "trainable surrogate of the score of the target distribution, e.g., coreset score or some neural net" + ∇S::H + "score of the momentum distribution" + ∇logm +end + +@functor SurrogateLeapFrog (ϵ, ∇S) + +function SurrogateLeapFrog(dim::Int, ϵ::T, L::Int, ∇S, ∇logm) where {T<:Real} + return SurrogateLeapFrog(dim, ϵ .* ones(T, dims), L, ∇S, ∇logm) +end + +function Bijectors.inverse(slf::SurrogateLeapFrog) + @unpack dim, ϵ, L, ∇S, ∇logm = slf + return SurrogateLeapFrog(dim, -ϵ, L, ∇S, ∇logm) +end + +function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) + @unpack dim, ϵ, L, ∇S, ∇logm = slf + n = length(z) + @assert n == 2dim "dimension of input must be even, z = [x, ρ]" + # mask = PartitionMask(n, 1:dim) + x, ρ = z[1:dim], z[(dim + 1):end] + # x, ρ, emp = partition(mask, z) + + ρ += ϵ ./ 2 .* ∇S(x) + for i in 1:(L - 1) + x -= ϵ .* ∇logm(ρ) + ρ += ϵ .* ∇S(x) + end + x -= ϵ .* ∇logm(ρ) + ρ += ϵ ./ 2 .* ∇S(x) + # return combine(mask, x, ρ, emp) + return vcat(x, ρ) +end + +# leapfrog composes shear transformations, hence has unit jacobian +function Bijectors.with_logabsdet_jacobian(slf::SurrogateLeapFrog, z::AbstractVector) + return Bijectors.transform(slf, z), zero(eltype(z)) +end diff --git a/example/HamiltonianVI/main.jl b/example/HamiltonianVI/main.jl new file mode 100644 index 00000000..ca83c093 --- /dev/null +++ b/example/HamiltonianVI/main.jl @@ -0,0 +1,161 @@ +using CSV, DataFrames +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using Tullio: @tullio +using FunctionChains +using NormalizingFlows +using Zygote +using Zygote: @adjoint, Buffer +using Bijectors: Shift, Scale +include("../common.jl") +include("hamiltonian_layer.jl") + +##############################33 +# model for posterior inference (logistic regression) +################################# + +######################################################################################### +# Example of Bayesian Logistic Regression (the same setting as Gershman et al. 2012): +# The observed data D = {X, y} consist of N binary class labels, +# y_t \in {-1,+1}, and d covariates for each datapoint, X_t \in R^d. +# The hidden variables \theta = {w, \alpha} consist of d regression coefficients w_k \in R, +# and a precision parameter \alpha \in R_+. We assume the following model: +# p(α) = Gamma(α ; a, b) , τ = log α ∈ R +# p(w_k | τ) = N(w_k; 0, exp(-τ)) +# p(y_t = 1| x_t, w) = 1 / (1+exp(-w^T x_t)), y ∈ {1, 0} +######################################################################################### +df = DataFrame(CSV.File("example/HamiltonianVI/data/bank_dat.csv")) +xs = Matrix(df)[:, 2:end] +X_raw = xs[:, 1:(end - 1)] +const X = (X_raw .- mean(X_raw; dims=1)) ./ std(X_raw; dims=1) +const Y = xs[:, end] +const a, b = 1.0, 0.01 +(N, p) = size(X) +idx = sample(1:N, 20; replace=false) +const Xc, Yc = X[idx, :], Y[idx] + +function log_sigmoid(x) + if x < -300 + return x + else + return -log1p(exp(-x)) + end +end + +function neg_sigmoid(x) + return -1.0 / (1.0 + exp(-x)) +end + +# z = (τ, w1, ..., wd) +function logp(θ, X, Y, w) + τ = θ[1] + W = @view(θ[2:end]) + Z = X * W + logpτ = a * τ - b * exp(τ) + logpW = 0.5 * p * τ - 0.5 * exp(τ) * sum(abs2, W) + @tullio llh := w[n] * ((Y[n] - 1.0) * Z[n] + log_sigmoid(Z[n])) + # llh = sum((Y .- 1.) .* Z .- log1p.(exp.(-Z))) + return logpτ + logpW + llh +end + +function logp_subsample(θ; batch_size=10) + (N, p) = size(X) + idx = sample(1:N, batch_size; replace=false) + w = N / batch_size .* ones(batch_size) + return logp(θ, X[idx, :], Y[idx], w) +end + +function ∇logp(z, X, Y, w) + τ = z[1] + W = @view(z[2:end]) + grad = similar(z) + grad[1] = a - b * exp(τ) + 0.5 * p - 0.5 * exp(τ) * sum(abs2, W) + S = neg_sigmoid.(X * W) + @tullio M[j] := w[n] * X[n, j] * (S[n] + Y[n]) + grad[2:end] .= -exp(τ) .* W .+ M + return grad +end + +function ∇logp_subsample(z; batch_size=10) + (N, p) = size(X) + idx = sample(1:N, batch_size; replace=false) + w = N / batch_size .* ones(batch_size) + return ∇logp(z, X[idx, :], Y[idx], w) +end + +function ∇logp_coreset(z, w) + @assert length(w) == size(Xc, 1) + τ = z[1] + W = z[2:end] + dim = length(z) + grad = Buffer(z) + grad[1] = a - b * exp(τ) + 0.5 * p - 0.5 * exp(τ) * sum(abs2, W) + S = neg_sigmoid.(Xc * W) + @tullio M[j] := w[n] * Xc[n, j] * (S[n] + Yc[n]) + grad[2:dim] = -exp(τ) .* W .+ M + return copy(grad) +end + +# customize gradient for logp_subsample (in each iteration logp evaluation only uses a small batch of the full dataset) +Zygote.refresh() +@adjoint function logp_subsample(z; batch_size=10) + return logp_subsample(z; batch_size=batch_size), + Δ -> (Δ * ∇logp_subsample(z; batch_size=batch_size),) +end + +function logp_joint(z; batch_size=10) + dim = div(length(z), 2) + x, ρ = z[1:dim], z[(dim + 1):end] + return logp_subsample(x; batch_size=10) + logpdf(MvNormal(zeros(dims), I), ρ) +end + +function ∇logp_joint(z; batch_size=10) + dim = div(length(z), 2) + gx = ∇logp_subsample(z[1:dim]; batch_size=batch_size) + gρ = -z[(dim + 1):end] + return vcat(gx, gρ) +end + +@adjoint function logp_joint(z; batch_size=10) + return logp_joint(z; batch_size=batch_size), + Δ -> (Δ * ∇logp_joint(z; batch_size=batch_size),) +end + +#################################################33 +# train sparse Hamiltonian flow (https://arxiv.org/pdf/2203.05723.pdf) +# note: +# - SHF operates on a joint space (target space × momentum space) +# - instead of using the full score in the Hamiltonain flow, we use coreset score +# - instead of using the momentum refreshment as used in the paper (only perform normalization on the mometnum), +# we just stack a general shift and scaling layer after the leapfrog step +###################################################3 + +∇S = CoresetScore(Float64, 20, 400, ∇logp_coreset) +dims = 9 +L = 20 +∇logm(x) = -x +Ls = [ + Scale(ones(2dims)) ∘ Shift(ones(2dims)) ∘ SurrogateLeapFrog(dims, 0.02, L, ∇S, ∇logm) + for i in 1:5 +] +q0 = MvNormal(zeros(Float64, 2dims), I) +flow = Bijectors.transformed(q0, ∘(Ls...)) +# flow = Bijectors.transformed(q0, trans) +flow_untrained = deepcopy(flow) + +sample_per_iter = 5 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp_joint, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(1e-3), + callback=cb, + ADbackend=AutoZygote(), + hasconverged=checkconv, +) +losses = map(x -> x.loss, stats) diff --git a/example/Project.toml b/example/Project.toml index d462c5ee..a1d4a818 100644 --- a/example/Project.toml +++ b/example/Project.toml @@ -2,7 +2,10 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" FunctionChains = "8e6b2b91-af83-483e-ba35-d00930e4cf9b" @@ -13,5 +16,8 @@ NormalizingFlows = "50e4474d-9f12-44b7-af7a-91ab30ff6256" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +SimpleUnPack = "ce78b400-467f-4804-87d8-8f486da07d0a" +Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl new file mode 100644 index 00000000..1efd01ed --- /dev/null +++ b/example/RealNVP/AffineCoupling.jl @@ -0,0 +1,98 @@ +using Flux +using Functors +using Bijectors +using Bijectors: partition, combine, PartitionMask + +include("../util.jl") + +""" +Affinecoupling layer +""" +struct AffineCoupling <: Bijectors.Bijector + dim::Int + mask::Bijectors.PartitionMask + s::Flux.Chain + t::Flux.Chain +end + +# let params track field s and t +@functor AffineCoupling (s, t) + +function AffineCoupling( + dim::Int, # dimension of input + hdims::Int, # dimension of hidden units for s and t + mask_idx::AbstractVector, # index of dimensione that one wants to apply transformations on +) + cdims = length(mask_idx) # dimension of parts used to construct coupling law + s = MLP_3layer(cdims, hdims, cdims) + t = MLP_3layer(cdims, hdims, cdims) + mask = PartitionMask(dim, mask_idx) + return AffineCoupling(dim, mask, s, t) +end + +function Bijectors.transform(af::AffineCoupling, x::AbstractVector) + # partition vector using 'af.mask::PartitionMask` + x₁, x₂, x₃ = partition(af.mask, x) + y₁ = x₁ .* af.s(x₂) .+ af.t(x₂) + return combine(af.mask, y₁, x₂, x₃) +end + +function (af::AffineCoupling)(x::AbstractArray) + return transform(af, x) +end + +function Bijectors.with_logabsdet_jacobian(af::AffineCoupling, x::AbstractVector) + x_1, x_2, x_3 = Bijectors.partition(af.mask, x) + y_1 = af.s(x_2) .* x_1 .+ af.t(x_2) + logjac = sum(log ∘ abs, af.s(x_2)) + return combine(af.mask, y_1, x_2, x_3), logjac +end + +function Bijectors.with_logabsdet_jacobian( + iaf::Inverse{<:AffineCoupling}, y::AbstractVector +) + af = iaf.orig + # partition vector using `af.mask::PartitionMask` + y_1, y_2, y_3 = partition(af.mask, y) + # inverse transformation + x_1 = (y_1 .- af.t(y_2)) ./ af.s(y_2) + logjac = -sum(log ∘ abs, af.s(y_2)) + return combine(af.mask, x_1, y_2, y_3), logjac +end + +function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) + x_1, x_2, x_3 = partition(af.mask, x) + logjac = sum(log ∘ abs, af.s(x_2)) + return logjac +end + +################### +# an equivalent definition of AffineCoupling using Bijectors.Coupling +# (see https://github.com/TuringLang/Bijectors.jl/blob/74d52d4eda72a6149b1a89b72524545525419b3f/src/bijectors/coupling.jl#L188C1-L188C1) +################### + +# struct AffineCoupling <: Bijectors.Bijector +# dim::Int +# mask::Bijectors.PartitionMask +# s::Flux.Chain +# t::Flux.Chain +# end + +# # let params track field s and t +# @functor AffineCoupling (s, t) + +# function AffineCoupling(dim, mask, s, t) +# return Bijectors.Coupling(θ -> Bijectors.Shift(t(θ)) ∘ Bijectors.Scale(s(θ)), mask) +# end + +# function AffineCoupling( +# dim::Int, # dimension of input +# hdims::Int, # dimension of hidden units for s and t +# mask_idx::AbstractVector, # index of dimensione that one wants to apply transformations on +# ) +# cdims = length(mask_idx) # dimension of parts used to construct coupling law +# s = MLP_3layer(cdims, hdims, cdims) +# t = MLP_3layer(cdims, hdims, cdims) +# mask = PartitionMask(dim, mask_idx) +# return AffineCoupling(dim, mask, s, t) +# end diff --git a/example/RealNVP/main.jl b/example/RealNVP/main.jl new file mode 100644 index 00000000..2eb2bb98 --- /dev/null +++ b/example/RealNVP/main.jl @@ -0,0 +1,55 @@ +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using FunctionChains +using NormalizingFlows +using Zygote +using Flux: f32 +using Plots +include("../common.jl") +include("AffineCoupling.jl") + +Random.seed!(123) +rng = Random.default_rng() + +###################################### +# 2d Banana as the target distribution +###################################### +include("../targets/banana.jl") + +# create target p +p = Banana(2, 1.0f-1, 100.0f0) +logp = Base.Fix1(logpdf, p) + +###################################### +# learn the target using Affine coupling flow +###################################### +d = 2 +hdims = 20 +Ls = [AffineCoupling(d, hdims, [1]) ∘ AffineCoupling(d, hdims, [2]) for i in 1:2] +q0 = MvNormal(zeros(Float32, 2), I) +flow = create_flow(Ls, q0) +flow_untrained = deepcopy(flow) + +# train the flow +sample_per_iter = 10 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(), + callback=cb, + ADbackend=AutoZygote(), + hasconverged=checkconv, +) +losses = map(x -> x.loss, stats) + +###################################### +# evaluate trained flow (one can see that the performance is better than the plannar flow) +###################################### +plot(losses; label="Loss", linewidth=2) # plot the loss +compare_trained_and_untrained_flow(flow_trained, flow_untrained, p, 1000) diff --git a/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv new file mode 100644 index 00000000..7f8dda94 --- /dev/null +++ b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv @@ -0,0 +1,144 @@ +away away bullseye bullseye circle circle dino dino dots dots h_lines h_lines high_lines high_lines slant_down slant_down slant_up slant_up star star v_lines v_lines wide_lines wide_lines x_shape x_shape +x y x y x y x y x y x y x y x y x y x y x y x y x y +32.3311102266 61.411101248 51.2038911373 83.3397766053 55.9930301513 79.2772636977 55.3846 97.1795 51.1479167122 90.8674123313 53.366566866 90.2080300059 57.6132335477 83.9051707998 52.8720214902 97.3432226575 47.6952008891 95.2411869282 58.2136082599 91.881891513 50.4815081703 93.2227013657 65.8155400946 95.5883740582 38.3377571839 92.472719051 +53.4214628807 26.1868803879 58.9744699018 85.499817612 50.0322537946 79.013071199 51.5385 96.0256 50.5171258092 89.1023945866 52.8019793617 90.0880645063 51.2743918205 82.8179829379 59.0141444945 93.5748748733 44.6099759113 93.075835032 58.1960536923 92.2149886482 50.2824056687 97.6099835723 65.6722651618 91.9334018119 35.7518707905 94.1167680276 +63.92020226 30.8321939163 51.8720726696 85.8297376348 51.2884586593 82.4359398425 46.1538 94.4872 50.2074801993 85.4600473951 47.0541298828 90.458936026 50.7538984114 76.7541289533 56.3751090389 96.30514763 43.8563814822 94.0858717713 58.7182307185 90.310532087 50.1867033389 99.6946801425 39.002716451 92.26183816 32.7672179591 88.5182945794 +70.2895057187 82.5336485877 48.1799307865 85.0451167372 51.1705368873 79.1652941091 42.8205 91.4103 50.0694819181 83.0576695262 42.4484337816 89.5077001153 37.0211840116 81.954469997 37.8391996844 94.359443503 41.5789293984 90.3035665387 57.27837287 89.9076067166 50.3269108629 90.0220534916 37.7953028026 93.5324553607 33.7296067755 88.622265789 +34.1188302357 45.7345513203 41.6832004024 84.0179406019 44.3779145287 78.1646280036 40.7692 88.3333 50.5628463386 82.9378217764 42.7040363241 90.4426288607 42.8817550897 80.1847700514 39.8853734176 90.6346620817 49.1774185559 96.6105324538 58.082020494 92.0081450101 50.4562073315 89.9874101286 35.5139005857 89.5991901138 37.238249327 83.7249284144 +67.6707164012 37.110947969 37.8904155015 82.5674929978 45.0102700721 77.8808631153 38.7179 84.8718 50.288527801 82.9752535659 32.3789386326 90.144142631 37.1557104777 84.9541114129 44.0774051326 84.1258533023 42.6522509813 90.5606397307 57.4894477748 88.0852855629 30.4648469308 82.0892320845 39.2194529062 83.5434818364 36.0271982243 82.0407806505 +53.2591294055 97.4757710964 39.5489736947 80.812601774 48.559816719 78.7883706035 35.641 79.8718 25.5834750822 82.9148911324 32.5307027362 70.1573965085 38.7318581261 83.7893564281 31.4970254888 67.9125621568 30.1233318217 81.1442953304 28.088741319 63.5107944263 30.5015964737 82.3071263462 31.5882013862 77.0280524208 39.2392807768 79.2637238398 +63.5149808671 25.1000785788 39.6495738769 82.6645338713 42.1422682922 76.8806345544 33.0769 77.5641 25.4835833896 82.929084979 33.3652644305 70.4593354392 31.0005254108 82.5794805962 25.9525965456 73.0331803541 31.915519986 79.7904134869 28.0854682136 63.5901969481 30.489569227 77.7298801366 32.6577782482 80.1138612983 39.7845249391 82.2605658962 +67.9805388133 80.9571652197 34.7505970472 80.0110909897 41.0269715735 76.4095884613 28.9744 74.4872 25.4435257049 82.8742004973 32.6560991775 70.0510704992 25.9854996226 74.3493875478 36.4347239148 62.2940454639 27.8877193574 75.4455759934 28.087273053 63.1232828116 30.45024833 79.8348944011 31.6415992679 77.6108331874 35.1660293896 84.1564919839 +67.3724659005 29.720400203 27.560835291 72.8478255927 34.5753090115 72.7248411845 26.1538 71.4103 25.5651134237 82.9240977704 22.9550932697 70.4268951523 23.7145662909 75.0980490792 24.997482691 75.4415616626 27.5482344526 75.1647590544 27.5780252176 62.8210386566 30.5183947985 68.2525829388 30.9871426099 63.9440280336 40.622115916 78.5421042089 +15.5607495229 80.0656402858 24.6355399754 71.6107148256 31.1686007236 69.2454213193 23.0769 66.4103 25.9288442731 82.8211841087 27.1498436318 70.2108058428 23.077319507 71.7205396747 32.2462751352 70.1105868383 20.2097781637 67.5166420988 27.7799191139 63.518147517 30.5116612138 66.0937181831 27.4396322054 65.7444629564 39.181907086 79.8190371976 +71.7907676942 71.0654666627 20.9594648125 66.0414983826 32.6442540714 70.7325550334 22.3077 61.7949 27.5514782612 51.4873865306 26.1669092779 70.5003151212 24.9391368744 72.5802858084 18.1094722949 53.7081103324 24.4434528292 54.4736551228 28.5889998149 63.0240805653 30.4980777147 62.3849850338 30.5783699449 65.5219454667 42.4308889944 75.1336346406 +70.2425464362 24.1095975542 20.6891490538 62.7213052064 26.6666456539 62.9809712052 22.3077 57.1795 27.5304663693 51.4148465629 26.3314570795 50.494556525 17.8934987136 71.1096873431 21.7960992263 49.4995295248 24.6308301331 53.3957388998 28.7391414969 62.7208638859 30.6202277519 60.8600821544 28.809401637 59.6210044666 43.0812558617 75.3467016413 +64.9374355444 81.5542049945 19.2882047369 62.0630593582 26.4592623274 62.6346733709 23.3333 52.9487 27.0955703605 52.0767994399 22.0037091415 50.4994401238 23.6730818572 64.8708391385 24.150491069 47.1885335266 21.0379688969 49.829299645 27.0246032407 62.9018588649 30.4921854231 55.8789168923 27.9395185786 50.9850621313 44.3607194344 70.4742070546 +62.2135245453 21.4758389969 20.0245005746 61.3426238695 25.5380076465 60.6302361093 25.8974 51.0256 27.4392496133 51.7120790527 23.2847479782 50.4743528321 23.7490719056 63.557171895 28.6031956015 44.2237227 27.106242932 54.7195531488 28.8013366963 63.389040388 30.474634333 52.3221596388 33.8499497887 46.2844744465 44.3247788435 71.0459044019 +67.2694004772 18.7089683725 35.4695229971 43.1158849517 26.3207111933 62.3962811186 29.4872 51.0256 27.8782642594 50.70890793 27.6340383923 50.5036668574 32.2151786343 65.6009580191 20.9319996761 51.6462354868 28.0896176553 60.512212549 27.186463838 63.5587296478 30.500998079 48.0696996767 32.628463049 54.0637204929 47.552160202 66.4604037217 +40.5701970446 79.3729634752 36.8943276507 47.7065559687 26.1633874648 62.0268148985 32.8205 51.0256 27.3388689181 51.6530467544 32.0371080007 50.5000661229 29.4368412489 67.5545281141 35.4355325251 34.9610381566 31.1357349152 61.4319127116 29.2851466002 63.3836058254 30.4944213623 45.7734755346 32.4386500903 49.9030194916 48.6649246215 64.9610049438 +74.7411813341 21.1016372041 39.0555497751 55.5469737131 25.2137904611 60.2733986018 35.3846 51.4103 27.6784029741 51.1819891691 29.3365106528 50.461893969 40.0500951983 71.8890335708 44.3773828236 57.6847369215 37.700837521 68.4184052804 39.4029453026 51.1508571967 30.4995445502 49.3820997506 35.8382920833 53.3514883396 46.280631958 68.0284231525 +71.7683189223 20.0110618423 46.9570801494 65.2404073935 26.8595295604 63.1418346727 40.2564 51.4103 52.6356576791 51.4185522584 40.816953133 50.4877601957 37.5747950188 69.9887444412 46.313692406 55.6855917703 43.3926884016 43.5975940692 28.8113284396 61.3578540576 50.144392979 47.0142836417 37.1420429198 49.7419879519 50.5632444849 62.6238279659 +76.1669198143 75.9361704048 37.3104527352 45.2587508999 31.7606691716 69.7976553936 44.1026 52.9487 52.0252141055 52.1230110467 42.7390751173 50.483155992 49.3104686263 67.1687603206 46.3860739231 51.0021645381 51.2623974761 49.6665051258 34.3039579069 56.5421259093 49.9128985877 55.5878389121 34.6445467746 54.4441077764 52.6309642318 59.9639357915 +65.6236574431 15.5828033531 40.0096720019 60.9865825077 39.7932905744 75.8079642168 46.6667 54.1026 52.8811647904 50.6215547554 51.8678937691 50.4932732621 47.3480185524 72.5916115408 42.4954400876 56.9036815315 43.2867129203 43.0568603208 29.6027609801 60.157346722 50.3374274859 57.9553142818 35.675404696 49.6256433471 54.6362064446 57.4938424367 +50.8506336394 13.9876016304 48.0143866846 65.7128195934 45.6174391854 78.1199702277 50.0 55.2564 52.9526073133 50.0747390118 43.3710018924 50.5008373234 50.157414434 71.1617258246 44.3005150328 58.1329592781 55.9059162338 54.2526725342 49.1161568603 63.6600006211 50.4189791992 60.5249117346 64.99458547 49.4621728312 51.346027139 61.6005530364 +33.0240700249 24.4678303872 53.7037759268 66.3894820363 53.7714606827 79.3029316435 53.0769 55.641 52.5205524936 51.5024421035 53.0427983006 50.4875038691 58.2100082697 68.0445380719 46.9882833458 55.4206367313 50.198958949 52.2250295948 39.6175458263 62.9251879574 50.2052589714 57.9242171676 66.1291498342 61.6437178853 57.134855354 53.8779658045 +39.7063261674 84.2752871038 63.0674998921 64.0350004581 56.4514336733 79.2471070208 56.6667 56.0256 52.3428220582 51.8619520908 56.6189102967 50.5033740178 56.6089076561 70.4089460888 52.4215231623 47.867005488 55.761587372 54.0305172396 43.23308466 63.1652187223 50.4353151633 56.4393224734 65.790651882 55.9453737171 54.1712415737 58.0598078952 +45.5964849542 9.76334884943 62.0480325076 63.8458632494 66.0937201989 77.0178182606 59.2308 57.9487 51.9275902141 52.2577906065 70.568289819 50.4962361243 52.8338205303 72.5518862725 64.090998189 68.6509850107 58.3231396505 60.0310591646 64.8927879422 65.8141767574 69.5057889042 59.2849248279 67.5017806138 60.8502274659 50.9423823071 62.0973939351 +42.9680469104 17.9454583961 59.8399667124 64.4767644437 56.9258394194 79.2063186226 61.2821 62.1795 52.7137744894 51.1979443188 67.870887292 70.0680261345 61.8185854576 76.1128240228 63.0168717563 68.2252589114 70.171457528 70.2822212385 62.4901493154 74.5842896072 69.5340709833 64.8929773346 67.6295142176 61.4291060701 66.512924456 59.0712818478 +52.4944067819 16.0511142003 55.1609418175 65.2373081694 58.9882538467 78.9172589554 61.5385 66.4103 50.4438027888 82.9418288173 58.8215465891 70.4557817483 61.7276334143 77.0692384058 67.3025779226 64.4134627173 60.600276471 59.7533088802 68.9880844294 63.232147305 69.5101069302 61.3296244282 63.9100640599 62.9723349305 64.3979095023 56.1748754008 +46.0822757831 23.1104578154 61.2797865792 65.766402504 57.9022660533 79.0808678187 61.7949 69.1026 50.2166950326 83.7523429676 60.1869278445 70.4835512458 62.1922520815 76.1568192103 67.2259568904 62.4764105301 63.7602750188 61.1856411696 62.1056186306 75.9908707599 69.5263228876 56.9952791254 66.5493579687 67.7255886599 68.686593294 62.2373439902 +74.2477082092 20.314187812 60.8349175348 64.6037697063 64.8128669568 77.4869389235 57.4359 55.2564 52.1841801076 51.9752506714 59.4355174849 50.4206326073 58.1187658587 73.1189614684 50.309988715 47.2678818553 54.3894952619 53.811901225 32.4618467439 62.8819029189 49.9472399619 48.2295767261 64.0515417112 48.8200939835 65.0277937057 57.0962573525 +64.5682641863 83.6396338956 61.5205906458 64.7918550418 60.3497935166 78.7294458673 54.8718 49.8718 52.7920973459 51.0733956509 49.5858434795 50.4349813825 58.2704123843 35.5968950425 45.4126389249 53.8707976895 55.2412969947 52.5935746267 41.3272006535 49.0702512739 50.0631041291 58.085037556 68.1932895833 46.5435408096 53.3045031712 40.883844549 +74.0216939058 76.1282745076 36.916543863 41.0952474383 48.890555756 16.8065435154 52.5641 46.0256 52.5897198565 51.338090196 41.7797417848 50.4706422808 59.7665303129 32.6212516009 54.0206071815 47.2698972057 49.0701267502 48.8969446811 44.0071499323 46.4496737846 50.489902373 39.2054119682 65.2783891952 44.8203529292 52.9451780881 40.3846256942 +62.3911805626 5.62307076073 38.5021996714 41.5671556174 47.549506348 17.0714045245 48.2051 38.3333 52.0288486671 52.1768375002 46.1276863394 30.4494107792 52.5355347207 27.1741229419 49.8420222483 50.5765522936 46.0181190216 46.2577233718 44.0740606888 34.5532038906 49.6888638379 42.018514414 34.9976613883 28.6329939351 45.225738199 30.5552259226 +74.189036683 68.1335832223 48.6643707344 30.6806668519 30.8894308257 26.3290372348 49.4872 42.1795 52.729246581 51.2017650475 38.0337837705 50.4908995189 50.8925215809 29.1452839382 50.7132995657 48.7722867825 44.7595874356 41.9746066134 44.0013152386 33.9042073544 50.3334811097 43.588671479 33.9237661692 53.5586885227 50.8433046202 37.7446280981 +28.2367819396 56.1395964513 50.2852524042 30.3379221051 28.972835724 29.0504689185 51.0256 44.1026 52.8843132864 50.4414354501 44.5468412844 50.4159133758 49.9675511775 28.4602033343 55.0455549106 45.9862727459 51.6006241332 50.0324668853 45.0063004454 38.2990195507 50.4467514196 57.2305889423 66.0044543688 52.6846691452 49.6592939398 36.273442735 +75.7719387944 69.8292300322 42.276332666 34.5276361236 35.1613607523 22.7868496473 45.3846 36.4103 52.5093008862 51.4162051508 46.2381175975 30.4956129408 51.3478832298 27.3642602259 37.5437012597 32.8840244657 36.4658820182 35.5241876075 44.4438406096 36.0190833012 50.4209091963 43.3479568967 37.0232562464 32.1184816794 47.1427889348 32.0250641734 +75.8552294691 62.5170442862 54.0317756204 29.6723483149 45.8703691685 17.4893933655 42.8205 32.5641 50.8626843321 17.1456310914 49.7397453317 30.4727248868 37.5551162883 24.5376638386 39.7958978066 28.4432191156 33.3947987327 35.0168013145 42.1787133986 26.492119482 50.3740520508 32.9182406167 34.6982700852 36.4753541349 41.2700565259 21.7218171465 +65.9708570175 72.7448559954 37.3293552552 39.6020423074 32.8314206076 24.7975381459 38.7179 31.4103 50.8914922458 17.1413237277 39.1102518936 30.4786951789 39.1995941778 23.6804956588 28.5233910215 39.7966662481 37.5800715749 38.8315722007 44.0445656189 35.6622382764 30.4768299051 39.4246055913 34.8634441715 30.9392745218 47.1301271555 31.3099029222 +21.7780404779 6.61662530728 41.389522551 37.2960562328 39.4855747899 20.0330500489 35.1282 30.2564 25.8551275976 17.0819086886 39.1428377913 30.7928559394 36.3827221955 25.2380052325 31.3996460012 39.2652632229 36.619722059 36.8114141094 41.6404540171 27.0930954213 30.5034743497 38.3485561381 31.0900650789 32.3837547196 40.9159715783 23.5415923929 +67.7597962473 72.4212015285 40.0746666572 34.6236852027 33.4461829314 24.1906434075 32.5641 32.1795 26.0256445518 16.9250135319 34.6254702671 30.4997410297 41.1006943822 26.5526132161 39.2415652162 33.3614166846 36.1634664356 33.819314199 41.9383300069 24.9915229793 30.5048349283 28.7714879939 34.9213061465 35.4184380529 38.6601639737 21.6895496369 +78.6171953363 52.5752573142 35.3496806211 47.1410731332 21.8635812756 46.0525933457 30.0 36.7949 27.8931727187 50.6619634101 34.7542074707 30.4958650002 28.2178118248 28.2802924847 27.7140565712 42.2500516253 33.1677570349 31.2477411672 44.0539275103 33.5563924949 30.4622184647 32.7504720275 30.343361183 32.5245631742 42.6125508262 24.195053774 +68.5077081898 15.4569189652 34.7637004189 47.6247999245 25.1658708449 35.7666139539 33.5897 41.4103 27.6399679354 51.3990974838 36.7537002386 50.5293043035 33.7970720433 28.6997507433 32.9607035213 35.8231112903 42.6449829318 40.1543216386 39.2067193308 51.5337156971 30.491302622 43.5613740753 35.4833288685 44.4916349303 41.2090395008 24.1287091891 +74.8850211598 25.4166063231 37.0266294472 44.4622930512 22.9594789762 51.2877067934 36.6667 45.641 27.892658895 50.7952815163 37.0193512929 50.4926666972 43.9193873055 27.7978445882 35.4047362397 31.9391621952 47.2250252695 44.5756466185 28.7044492315 61.7775254006 30.449653838 44.9951230008 33.2349104271 38.502483721 50.6492211499 37.4837907435 +66.4549036599 19.8366286542 36.4555695327 40.791843032 25.5860552415 60.8465833585 38.2051 49.1026 52.7977329399 50.6860370901 41.4171152753 50.4697734181 44.4164011084 28.9476923345 51.6422264225 50.8383406788 48.3923630032 50.3315285645 31.7086628996 58.837754374 30.4859942157 47.3479464653 35.4399436182 47.7975176939 52.1319019406 39.3646020815 +77.3178020985 48.3983464352 35.5376642131 48.7293868661 26.3143100672 33.2971681038 29.7436 36.0256 27.5806388053 51.5247612607 22.8171074784 30.4360403387 28.0376027706 26.8038243613 34.207620843 33.5938054385 27.2451932001 29.8195074488 42.8117114739 30.0204484197 30.5020767935 36.4412129198 33.8735095227 37.056028453 39.7972712705 16.083549649 +58.9124603193 75.6677562173 20.4089478892 32.2030304205 33.037301251 24.5644624921 29.7436 32.1795 26.4913985349 17.4053909738 34.800701463 30.4979313851 31.6895761296 26.6664002089 36.5234892198 35.6710483891 30.4043143388 29.0795399241 43.300614891 31.5264261979 30.4780588886 28.020580489 28.5641377966 34.6263997171 43.5498814807 24.3072109395 +57.617447817 8.19480060319 23.4957104659 25.3224681542 36.4688894772 21.8761144725 30.0 29.1026 25.9853178159 17.203722132 27.8918897953 30.4849702595 27.2796128523 19.2572809685 28.4192021281 21.3157482481 30.0807992865 27.5880255173 40.3986329069 16.3470083822 30.5011608787 38.3141385582 31.3195486706 31.6639905316 36.7146633371 17.3818098572 +76.0882257967 59.6799300235 29.5575433634 21.3647774591 26.5135531009 32.9187498178 32.0513 26.7949 26.2014192753 17.0938239128 32.8706391587 30.5023417488 31.2218783581 26.4408557812 26.1604969417 23.0422314726 33.9806339675 35.0330581489 40.4356915763 20.2326706762 30.5047730174 26.4166256707 30.796174379 23.5190343602 37.2246101278 17.9800644828 +57.4660505497 1.50441817488 33.0082339121 15.9850714584 36.0270912125 22.1954385395 35.8974 25.2564 25.8575694746 17.113842659 37.7940494081 30.4926383364 33.5458051034 20.0807081014 40.435327397 27.6400711421 34.2506117649 34.4415865654 40.9365466658 16.9130048364 30.509115326 22.8255471196 34.2333554347 22.4959116015 40.5124478537 22.9622145725 +79.4283834934 45.2107942872 53.9803991821 29.3509867092 42.6810404129 18.5425361977 41.0256 25.2564 50.7046843629 17.0237445436 35.6673163042 30.522035054 39.7363306065 23.4948069772 44.544955314 27.3622426221 51.1411835714 20.6183237784 39.6615736653 15.609355577 50.1156321728 16.6940065479 36.6307877065 31.4768312621 40.2220627884 20.6146217044 +76.3565221496 10.4182411281 52.2343086004 29.7116729865 45.0410771273 17.7345177303 44.1026 25.641 50.8119753465 17.1149252621 48.4422995026 30.5017227753 42.8418426525 20.3339004396 40.1237272008 30.0731619402 49.899592066 19.5640835356 40.899259175 20.7985289466 50.4483013392 21.6402564599 34.5486191018 26.1110878654 44.1249612057 23.3662452858 +64.4050752632 78.5841760758 59.5030766063 30.6696739448 47.3524164557 17.1347454264 47.1795 28.718 50.5648455577 17.0777773216 58.6693732631 30.5068378948 47.0937298879 26.9934706312 42.1848064503 28.5736747331 55.5898773424 26.2992451143 41.9686168283 26.4970725985 50.17317743 29.177864393 37.5196356 29.2676058936 42.3462616039 24.5583269437 +40.6350418091 73.3947503698 41.1637810689 34.3157582514 53.8461940322 16.3873180961 49.4872 31.4103 50.9393039127 16.981021883 47.9169554375 30.4867257545 51.4536583539 26.0229417521 40.0026592892 27.4875100027 55.4820563581 22.8274662445 40.3834058238 21.3912255247 49.9227419194 38.4404729124 62.4256973267 34.0442062707 44.1890843708 28.772625473 +43.9498645857 75.9587156671 48.9930401201 32.0303588363 55.1296393835 16.3532468711 51.5385 34.8718 50.4588548392 17.0385789686 51.8941663939 30.494755684 49.3677068167 22.005580703 57.1252310509 42.7390191388 63.5729393821 32.9386176053 56.5381264538 32.4442454697 50.3112993746 26.8604882737 64.320383794 31.7856294846 45.8142453101 31.056979672 +30.9962205791 71.694404938 59.2692803248 29.6407017654 52.1881199617 16.4182174652 53.5897 37.5641 52.9013640729 50.6905627231 51.7490891643 30.4729933399 48.2534809736 29.2708555331 60.375744968 41.3564579961 59.0250770848 28.3220858801 52.9706912828 29.0401966941 50.3994096114 38.9638389234 65.4455325117 43.2734991936 46.6431192691 31.4072190618 +68.2307689907 80.8725016628 45.4691770035 33.8311927323 58.6726630845 16.6633724322 55.1282 40.641 52.6849534438 51.2944692191 58.2465530013 50.4428744949 61.0029597643 30.0153144324 55.1058961252 46.1490510561 62.4237785929 33.4865952868 54.6209525885 30.3445244537 50.3373626448 40.913386144 64.3256315887 41.6159763773 52.1898883047 39.3677964938 +72.0463894612 12.9180067349 62.6912665406 30.2903738312 82.1443116037 33.1952218669 56.6667 42.1795 52.500088943 51.594356165 57.3106968674 50.4825132895 60.8065608049 27.8683134861 61.7372710245 38.5848440052 47.849907703 48.556624645 65.0990443941 27.2415575588 50.2023180965 47.2129480045 65.264338195 35.7175447772 51.1003173587 38.095699984 +46.5927679682 84.9723827774 73.4286708654 48.5778551303 75.6865711383 24.7415103908 59.2308 44.4872 51.8356372562 52.3357655339 67.9680481127 50.4521589097 59.0674270249 26.7882840217 55.7412443246 46.6535769763 52.2685616825 52.8052401961 63.0559909076 29.7090956748 50.2749326091 49.6234993439 66.388704175 38.8447443366 65.4008341477 42.6140320392 +49.2572183396 81.8814032306 70.846426107 52.282253331 85.344964503 52.886419866 62.3077 46.0256 76.9954121013 52.0455286534 65.9493192681 50.494240794 61.4146337175 31.0393750583 58.2964084158 41.5831977346 68.8510215758 39.573434273 70.9601362323 41.2595012867 69.4943803061 52.6124107059 68.1914228192 45.1694864796 64.8520398267 43.1337348404 +42.7817612539 12.9911884302 71.5390198495 45.5218061588 83.8446535127 58.6160602305 64.8718 46.7949 77.310600475 51.7467300374 55.7470928102 50.4789748003 68.9343694692 31.6198930866 57.0425482213 42.418757739 65.6763960251 35.1093154301 69.8958192404 43.4537592655 69.5193397556 47.384533707 68.5406624515 48.4385263389 63.4038060118 44.847493688 +65.475952195 14.2745856444 67.6208658884 38.0065584658 85.66476066 45.5427527692 67.9487 47.9487 77.9260443352 50.3186604237 68.4030100436 50.4986272318 63.0652438592 34.2357465852 76.7756112073 55.2545987009 77.7468851148 47.3718241139 70.5958928563 41.9647438672 69.5291826079 57.8412666804 67.6614312657 51.0685845873 65.3715488709 42.6959713164 +71.9650826544 17.7102359443 72.4709525633 51.1221348241 77.931004934 68.6900618808 70.5128 53.718 77.2543883433 51.4618248152 76.696465628 50.4938305609 72.2725228706 67.6444437851 71.8870772602 59.2669926152 80.352934115 47.8002988852 69.6470214273 44.044445022 69.495755496 58.785073162 68.6755160534 50.465234811 61.9080397046 53.1903520694 +32.1464623358 43.4817094425 64.8122375621 62.8109155873 73.0318330209 73.12057473 71.5385 60.641 76.2431578028 52.1236898503 67.2148371159 70.2600284356 75.3598819664 75.4216145799 73.7883730663 58.6208121164 60.3375864858 63.0349593272 77.392982494 63.3714590552 69.199057706 60.4189573202 69.0817099788 55.4740530987 71.8266529973 65.6862303604 +31.8384976954 71.8121653901 60.8536798727 65.4991470327 70.1028044999 75.6832068542 71.5385 64.4872 77.0844843669 51.9671367003 72.3020057291 70.2204373812 72.8336258709 72.1146938095 75.1396847343 54.2445521781 61.6587233282 64.1000234779 64.4007871926 67.4487184472 69.5510914402 58.4010982686 69.048716493 66.3660457477 72.6546231621 66.8782776113 +31.0052582572 40.682503007 67.7894961571 61.3637015467 57.1446946889 79.1941025922 69.4872 69.4872 75.2280531954 82.9856620201 66.7632573944 70.4738617113 70.8104148392 79.3504618327 64.5963189058 65.952512925 71.5450389858 72.657716331 63.8689598271 70.2137388333 69.5066843151 68.0844667893 67.3082233775 71.5902965807 76.8885781632 72.1654720777 +80.4708943189 49.5021483467 41.6095572691 83.8486865559 46.9624619942 78.6665589963 46.9231 79.8718 50.6583547714 83.1144793358 51.6803004439 89.9970275485 45.8019347617 81.6582677456 46.9095386323 85.6022611586 49.0409566406 77.3625832678 56.5944213157 86.9270062202 50.4441175504 75.0132995059 38.714899408 76.3969778172 43.7446196844 71.2679610751 +71.9641671122 41.8742826668 53.0030253202 84.6747986012 50.5649338128 79.0772985529 48.2051 84.1026 50.2033658114 82.9826568621 53.0965573571 89.540243712 45.835020171 82.7425919919 46.9051031994 81.5656424136 51.8868880222 80.1136076892 56.5313372853 87.4998110713 50.4672293846 90.281464505 35.5480948795 89.3049758407 39.0542484428 83.1190207043 +78.0794214417 93.1333167652 54.7141710591 84.0431280734 53.7196765696 79.306936167 50.0 85.2564 50.9295476993 82.8460411252 42.2422929389 89.7085785936 40.0788064496 80.2471852579 49.8069877356 82.085025592 57.968981435 85.3560457326 59.6521583697 87.8094615921 50.210881979 80.9417717084 36.7106729424 86.0749670003 39.9316746322 79.9391880033 +41.6775957748 30.2012640846 44.2916687157 82.9094412144 54.2002323226 79.3250023354 53.0769 85.2564 50.1786718465 83.1846223323 56.2338756577 89.635308157 51.9829247003 82.2043834675 48.8632418189 83.1583574877 54.855664646 83.7833976656 56.6365087005 85.637495556 50.205406955 90.6405359009 64.9126624396 87.6457960566 39.762826107 76.4059674459 +65.953595185 31.1474060835 49.1917219555 85.8762291224 47.875198343 78.6758534295 55.3846 86.0256 50.422698058 82.9099014671 54.6809865825 89.9645994589 57.337906053 84.9995188696 66.7481899024 84.0048341677 54.1331954613 83.7939960824 58.6722880025 90.077160307 50.4382906648 92.3547602661 66.0641357377 89.2086685815 41.323383287 80.2118685969 +62.9344593731 31.9163906992 53.1013817819 84.5476586881 51.4734585149 79.1598669963 56.6667 86.0256 50.4642248341 82.9353284131 57.9026866088 89.6184017294 62.3362327624 83.2465678988 62.3597662841 90.2436599196 51.3088252568 80.1363911553 58.2216127264 90.4110187715 50.3735188462 82.2192057218 64.9359294757 87.0721381174 41.7873232275 83.9880779935 +64.3737979844 28.8625834061 51.599848153 84.8198214898 52.7512222162 79.2936599344 56.1538 82.9487 50.4492703306 83.9699203811 51.8202625467 89.8341303987 58.5059100905 81.7455863409 47.3748964954 83.4715555182 61.1203943844 88.7040529069 57.9146644768 89.9538027677 50.253683689 81.7438287242 67.0265507063 76.2225153324 39.7556490651 79.6570966361 +72.5093283599 39.5401302526 54.3797219484 84.2403555494 48.5622230047 78.7823835275 53.8462 80.641 49.9283802752 82.9936654888 56.8283953362 89.5836647251 56.2479351377 83.513949493 47.4464667165 82.622124014 55.5638650304 85.8103751441 55.3155090581 80.2518606944 50.2504708592 71.9513852104 65.3140322257 77.9387440331 40.7498793985 77.204657286 +30.0522898741 96.6175423534 46.4807681047 83.518211666 53.0812416908 79.2972507654 51.2821 78.718 50.4880136436 83.0995191232 58.2722785867 69.9002729578 55.4065208295 80.2666719669 53.385280634 78.8711362558 55.044123361 83.5414106324 54.5757285877 77.5362884724 50.0870583609 73.9518736268 64.2952177764 84.1258572133 43.8568984983 77.6098436482 +28.0033242354 46.6721919544 53.174656268 84.2605679876 56.475383374 79.1678400979 50.0 78.718 49.9649053793 83.7083177013 51.8281274487 70.3526492053 53.2839688423 84.2635992213 49.4527077733 79.9306028004 50.4441853247 78.1958883115 54.4130936504 78.2290865878 50.2310718829 80.6473870602 38.7980121423 74.6106304299 42.3678224956 73.6382718294 +75.4012268619 88.6390766207 45.272002941 85.2370794195 49.570911405 79.0381238831 47.9487 77.5641 50.7521082559 82.9019500961 44.8499842618 70.3952029719 44.6767615455 80.2382855483 54.8026825363 75.3639849893 43.8277089981 71.0470522123 55.0745059041 79.8175464166 50.2077411201 73.1145035177 34.6757573144 79.6375987743 39.4629598364 78.459973898 +38.9800154218 87.322160691 36.03340215 53.371687755 37.6277032991 74.5575043418 29.7436 59.8718 27.4224292143 51.4388762325 24.2703823928 50.4855676918 30.0112642472 68.1467563743 37.0882451339 60.857227495 37.7914323802 66.8546410275 29.4329605156 60.8017765444 30.4995061735 53.6125276703 30.7192619427 58.812060082 44.5173339492 70.3693498252 +65.2199135479 84.6829549336 28.2711941653 72.840231257 32.2637000863 70.3166389012 29.7436 62.1795 27.6740833991 51.3041121452 34.7424714403 70.3399348879 29.7452539206 74.4045268347 32.3759619238 68.5108204552 29.292783111 59.79199693 29.4226860665 63.0684648229 30.5203612407 65.1004090686 34.1051509268 64.1291972267 41.6360929963 76.2492229205 +73.0539899616 29.3808085571 25.0548060847 71.5485979221 32.542380336 70.7024172063 31.2821 62.5641 27.5373953151 51.5936540762 37.7993149267 70.4695358763 37.3099483549 72.5861552003 37.4770138847 63.7886855744 32.1920969515 60.2006509147 29.0056141576 63.3907513334 30.4549558873 63.5017787814 33.4233718511 67.0957433613 45.4229321817 69.0955016709 +34.3983616372 59.5444469033 64.7588870014 82.3152236405 52.8250171923 85.5781338795 57.9487 99.4872 52.2633473764 94.2493278274 49.7869410649 89.6580469945 61.1359469093 82.5375693646 52.4066488967 96.4254150057 51.4339355233 99.2568672885 58.4618385916 90.2653263904 50.4164772292 97.2696919618 66.3138096741 98.9310270371 42.0028562742 97.8376147235 +43.4904501336 40.782542065 63.1445274767 85.2366963264 58.5282895407 78.9787042541 61.7949 99.1026 51.7372816586 92.9791175339 59.751497414 89.7319325216 61.7206191907 85.7056934492 50.2970432655 99.6441791736 67.4641398118 95.999963905 57.997804739 92.1599086113 69.5250693831 93.5980502089 65.3096314498 98.1651568396 76.1056290282 95.3049805754 +55.138737967 30.7257603575 50.4246786898 85.1747475891 56.265834263 79.2529394854 64.8718 97.5641 75.8709636929 88.3864417377 64.7747992444 90.132316736 63.6093798555 83.3600288362 61.5633243835 89.8641388605 65.1507052757 92.5026337368 57.5494740761 90.7489065581 69.5202566002 97.1938760167 65.7140760682 99.2837639519 79.2025626772 92.4072156264 +43.6843934333 32.8230098696 70.6449962629 82.4309187567 57.0758924193 79.1939225382 68.4615 94.1026 75.2443262052 83.9034973788 70.8444476004 89.9242090159 70.7237162322 87.1522075165 61.0865603919 89.2243556475 65.5834276962 93.5170609365 59.5299284566 88.3272741451 69.4989700668 92.1204899292 67.5934531877 90.9519098246 84.8482440854 95.4248045304 +35.9036097344 91.1118630801 63.1490490834 83.9468553485 54.2564023808 79.3138648504 70.7692 91.0256 75.1982952889 83.4623033365 70.6059455116 90.042061963 75.0063926418 82.4258351563 59.2364904011 91.4811005766 69.2889731377 99.5795911254 58.2493910631 92.1296814778 69.5286156637 91.6220882409 66.3880173202 89.2812304744 81.5644753207 83.7929072262 +45.3780188805 29.1692166544 62.8240245172 84.9661859507 62.6856712475 78.1747432719 72.0513 86.4103 75.7010415321 82.9194588559 78.6424883907 89.5512032307 75.0925497814 83.4373907092 66.1540050863 84.1119370252 81.2694405511 80.635944933 58.0245140126 91.6944211685 69.4997669273 84.47968329 67.8061881135 91.6881773501 85.4461864009 83.0782938448 +39.7774828713 43.75581895 70.236869515 82.1711510621 68.0856058616 76.1123175001 73.8462 83.3333 75.4793396586 82.8840513891 70.810692254 89.5926540118 70.3279291834 82.0293546968 67.2313347767 85.055640808 78.5757448361 76.8157030942 58.3821244904 90.5534760692 69.5038013697 81.8154056166 70.9626103129 86.5181213517 80.3124913784 76.5162389518 +38.6644611569 33.3172384774 70.0427352449 80.3850213547 65.1181219632 77.3991884543 75.1282 79.1026 75.1945668739 82.9321157846 66.7699466213 70.4233752108 70.8633308324 79.4081747457 73.7099068702 78.2706108976 74.1867734997 74.3442041467 62.5667590405 77.7439347591 69.4875893963 81.8359875567 71.9936371714 78.8209480695 80.7662322969 79.8960656796 +39.0440366877 84.6760108316 72.5706234458 80.9712184346 68.7456312971 75.7900554548 76.6667 75.2564 74.8202539551 82.9623887856 73.3363681612 70.3169271705 75.7799048195 79.8861994199 76.5804513576 72.8915336827 72.622209414 74.5269656511 72.1758243064 63.1289294176 69.5191157281 71.1159672213 72.4798343312 80.8006830564 82.6701124763 81.711479845 +91.6399614428 79.4066030605 75.1307160373 79.9840931403 63.0902251412 78.041409495 77.6923 71.4103 75.1643404933 83.0349971693 73.1051481627 70.300511881 78.0936590733 77.759558184 78.8765811639 71.5625276804 75.1415195918 73.8196684648 79.4727615693 63.4086861199 69.5043762947 70.4934438205 74.0784292286 72.9515308522 77.1636254828 73.5237189623 +47.4881326771 85.3899333808 83.2939057345 70.7784317876 77.2045893204 69.3121497658 79.7436 66.7949 75.2633555474 82.9452793018 74.6782541432 70.4453141903 76.7457612243 71.9733600919 77.4490021363 72.4473094857 75.168667839 73.9004448762 80.3577008812 63.2954375404 69.5125730109 61.497563947 72.0362880189 69.7160484161 74.0707000309 68.4795642895 +44.5902125769 22.0340116412 79.6642622775 73.9323097157 82.0339084758 62.6818697859 81.7949 60.2564 77.7564189331 51.1517703255 80.240250951 70.3543089613 78.4646034437 69.7493008209 86.8537073476 65.5433240194 85.3242028911 55.3885788252 78.7572361375 53.3326200111 89.4942005622 62.2343269676 72.9981862717 59.6656447092 74.1142308687 71.9263635185 +39.0896145478 70.4661940802 88.4321025254 64.6242470023 81.5779459183 63.8949255005 83.3333 55.2564 77.9544350549 50.4755789715 70.9468490729 50.4727017878 76.7456848486 70.0440572588 79.806531301 52.7231797883 84.090507598 56.301968006 82.5402395934 56.5410522935 89.5001240511 53.2163306165 74.8121312339 57.5898837631 68.942075458 62.635150577 +42.2293783752 19.9140684075 89.1155590082 64.0015066388 85.1801432805 53.7816766607 85.1282 51.4103 77.0833377719 52.1577992662 88.6626638086 50.4553260208 85.161682187 65.5124047088 76.9882734745 54.2027848022 86.2562238205 55.8309303793 86.4358971909 59.7927618087 89.490246254 48.890796551 73.0273050243 46.2741878725 73.2420023309 67.3532711288 +37.0003871448 60.264279248 89.0921976305 57.768193047 84.2868249936 57.2561680896 86.4103 47.5641 76.0635502513 52.1046520613 84.405952832 50.4948658866 91.8083525691 34.9655944844 79.4793575017 49.1410154455 92.5487933153 42.9019079116 79.4886844186 53.6516742641 89.49778562 46.6299066746 77.067111934 51.4869181896 71.8670131631 34.3291575887 +39.0520864793 70.6525028457 91.7260057727 52.6233532573 85.6224867126 45.0241658646 87.9487 46.0256 77.6820163247 51.1656378096 75.9871047068 50.4919669557 91.8874485924 34.7692538555 82.3041595613 46.9909036215 90.3426345089 37.7803861004 81.5304203188 56.0253645659 89.5048511306 48.4234081578 75.0171368727 46.6224426489 71.5691888532 34.699218607 +37.4884147432 60.8144048511 91.7355387598 48.9702108877 85.6044990156 50.7623379055 89.4872 42.5641 76.8780819814 51.8675622958 91.1120854191 50.489389393 92.2484016622 32.3771544295 90.700691247 41.4667030063 85.8191629042 37.5713861819 79.1867885665 53.2347918468 89.5010526463 43.3473436951 76.6653077326 38.440250966 71.4829096934 34.6805169926 +69.3595594592 65.5213545959 91.5078881747 53.3126520851 84.4247858276 38.761410164 93.3333 39.8718 76.9485027211 51.9075165439 98.2881232748 30.6039193719 91.782532741 34.3842946521 95.4434878077 36.1897016148 90.4418181137 40.1331134204 77.8990579454 51.8224583343 89.4888412549 40.6250912209 77.9158742329 45.9268433826 78.1285220512 23.139231835 +43.542775926 62.4603112824 88.239001904 31.4774348786 85.570985078 44.5600964397 95.3846 36.7949 77.8640547124 49.6625455341 95.0652748396 30.5004028652 96.0805193672 28.0536005018 94.9974880471 35.0953011066 92.0175590822 39.9179311324 75.1307142075 23.3724419733 89.5016235276 45.8151791904 73.7420460836 39.1209853231 78.789825254 25.5971933466 +39.8112302539 65.3348328092 88.5305192041 30.4760310098 84.6267242006 39.4870775084 98.2051 33.718 75.7714500936 17.1112512057 95.2492339568 30.4594542144 92.2478988278 28.0707514904 91.517847246 36.8902631263 95.2605278366 46.008830272 76.0580137544 16.3837496882 89.497529662 33.901372671 75.3298157891 32.8303519164 77.7815467962 23.4472144745 +70.0689259404 7.59346560899 55.3651603364 30.4458502777 73.3961549252 22.8162479426 56.6667 40.641 52.3315691301 51.8788603463 49.6561262881 50.4313198677 57.662276619 24.0606079328 62.2926421901 40.0295333141 41.2902998033 42.3115524113 57.6146743876 33.8224476502 50.4390920216 37.670771798 63.4104355141 38.377735614 48.5306293681 34.1524500077 +70.0405435824 77.1438066024 62.5602588694 30.4471356661 73.5074477338 22.8569887644 59.2308 38.3333 52.5928183721 51.3915915185 63.0186009279 30.4925819262 59.8469248931 26.1777124879 59.2034756073 41.2706776667 66.4313243161 36.5361097353 56.1713975295 32.1179887719 50.179850128 36.8075593352 68.856486669 43.084147214 65.5307814724 42.2810847015 +39.505789079 74.8516272173 58.0066691227 30.2537212987 63.3564881135 17.7240688424 60.7692 33.718 50.4770477213 17.0482894119 70.5382006004 30.4835582526 64.0670962137 27.7765005568 65.2563927704 32.8541041237 63.9863468858 32.7134316129 66.2878905956 26.1171097453 69.5050706711 33.7401525035 66.3377850519 33.3065100022 70.1180341426 36.4481847905 +62.5168908529 66.4847322418 55.0671179917 29.0115351997 72.9590468722 22.5114998325 63.0769 29.1026 75.2964750909 17.0156531888 70.0868697658 30.5100448173 60.9815013975 23.9342981582 62.4447424979 35.4149992261 61.2696820905 29.3857240482 67.8817196169 24.2360184109 69.5798670452 21.9352367455 64.2037185014 26.6441143003 70.4722626065 36.0880530553 +72.1399254065 0.0151193251552 61.6147759612 29.9943911942 64.0208416469 17.8835812521 64.1026 25.2564 75.573954129 17.0621921415 62.6856904424 30.5464245575 59.9670579092 26.8025832185 72.4624062907 27.6242638254 60.7493891909 28.0517363383 64.0280813016 27.6726855125 69.5064719492 22.7286628748 64.4986348346 22.863501327 76.7911300701 28.0720081685 +45.2515760666 70.0034213192 68.5431435378 35.6578321706 61.124463241 17.1163533592 64.359 24.1026 75.4005271583 17.0411068868 69.239417959 30.5060052447 62.7074541119 21.9955085787 69.9194262552 29.3110927191 64.6552305634 16.6122765086 77.4966517463 14.9485235577 69.4994788728 16.3374699821 68.8909906096 27.2962056672 72.4093545935 33.0392569664 +42.0633045627 2.33519661206 77.7061096487 20.3042601852 75.6742073297 24.7825009657 74.359 22.9487 75.8709908356 17.1348939075 80.4002336791 30.4850063746 69.1952352018 22.709792166 77.7628590782 23.4913735562 68.6382807138 18.5080128963 77.6346517616 14.461853927 69.500264751 27.1360239942 72.371523338 21.9616397473 76.271299815 25.5695441452 +36.3556951539 6.0058486497 68.4530459999 13.0355285908 69.5135831574 20.2793578927 71.2821 22.9487 75.6058847579 17.0677230553 74.6478782284 30.6738628584 65.5923626471 20.6941731019 72.7784101997 26.9005353695 65.7766651568 16.7876560655 77.8637264289 14.6106776491 69.5041565157 23.6689181006 69.7654249704 19.999850454 76.9532591384 27.7706107405 +30.3918276596 42.75961287 68.2572064402 12.3846391542 58.5721354669 16.642703808 67.9487 22.1795 75.8955770466 17.1699497051 65.9952727194 30.4750241915 64.6170985343 22.4431455531 65.198316667 33.0271717859 71.1341263421 19.3859575313 77.3381581659 15.890054658 69.5322795847 21.8509008151 68.6213124429 18.9156764428 75.1077300904 30.0169834157 +36.4490038543 50.5462690659 70.2554746739 13.2503849689 70.9819409865 21.1458881039 65.8974 20.2564 75.7465632023 17.1057165115 67.0885590036 30.4945239508 59.191159132 17.767901807 69.4598588916 30.1970315053 63.7231835039 14.5214211957 76.1804165329 15.9125737458 69.5001114942 11.7916260054 64.2977371716 20.4287496884 75.9692852004 29.0388653753 +40.467576002 60.0275120878 65.0443252778 11.0008414785 65.2230303973 18.3971951223 63.0769 19.1026 75.1423414811 16.7549238932 59.8525770283 10.5431877763 55.8396313724 21.6185696663 59.3150555205 12.3543136666 73.0014940701 24.0738125371 77.2526510945 15.1515170246 69.5048154243 15.6143582625 66.6992650902 18.5910853404 75.2838870131 24.8747531721 +81.7246168002 6.03754484635 60.0922466078 11.8721117093 65.3150439093 18.3337482986 61.2821 19.1026 50.6617795563 17.0781405153 53.7489900744 10.5225745323 59.0007586238 21.1710074885 51.487713355 20.8436106875 72.436657497 20.5783451136 77.4133752817 15.2219279762 69.5034745234 17.1690897689 67.5445276811 16.4479381344 77.0522406385 27.9056390894 +48.8231974964 76.6353305783 52.9920289667 9.90666848335 64.0102406778 17.9244781912 58.7179 18.3333 50.6998506392 17.0851843836 58.2390955164 10.4948990284 56.4593711284 25.1445642082 57.1124475509 15.3217851726 68.4127671225 19.9131265443 76.7318493982 16.2168461441 50.3774019862 16.5060150613 63.9469519836 18.6928454476 42.8267563374 25.5569594065 +35.6205617651 57.2860155789 50.1446289773 12.2115430946 57.1409359294 16.5986105437 55.1282 18.3333 50.9189408654 17.1476047571 54.810300495 10.7249292845 52.3581053558 19.4170541222 50.2244178157 19.1709698793 52.8583150204 20.2350610035 49.4711054118 25.0630193062 50.3394254813 18.7234506861 64.3881920705 15.7728122808 42.0806606852 24.9465264454 +50.5839631148 71.8066161014 46.5086141932 11.2071344881 56.3651532552 16.3265463742 52.3077 18.3333 50.7252585405 16.9074698083 52.9767123211 10.859490276 51.8116637782 17.0334945954 50.4338611907 18.5348708515 53.5299526734 24.2419155845 42.4765399431 18.3384735636 50.0936501016 11.6077134241 65.570045691 23.7657582226 39.8783046498 18.3249292157 +61.8564651063 71.7927431642 43.8070319612 11.3189448865 43.046183241 18.4025593948 49.7436 17.5641 51.2638712274 17.1623497098 52.2038939272 10.4956369212 54.222845621 19.503665544 56.3803054046 14.7884597101 50.7597410528 18.0893946004 43.5951158622 19.9942009773 49.9341271714 22.2561397401 38.4028368753 19.0468586722 36.4754991726 17.7189477884 +39.3237560262 59.3008196656 57.8178546865 10.9451424306 52.0817555582 16.4538745312 47.4359 16.0256 51.2509196468 17.2404558616 50.7451622429 10.5249055723 49.0187553996 23.4892740197 54.8635045999 14.009611915 42.53480844 10.1698684594 50.3399696652 26.4713966117 50.2888186122 13.436410466 37.8323600164 14.4694894463 37.1030736419 15.4456116477 +42.1856791429 66.0348978235 50.9404926595 9.69154713447 42.8790365154 18.4894110252 44.8718 13.718 50.7851572052 17.1801964842 42.7131936166 10.4875472428 35.6535812004 17.5145080595 45.4998576493 3.98542629287 41.5303978512 13.3962907452 40.7489802561 16.1821416622 50.2879944267 12.8722822086 36.9041611715 13.5838157511 34.1489682077 13.5238698677 +30.8469189898 37.3416401041 63.4973230835 11.9140691707 51.5669948027 16.4857136372 48.7179 14.8718 50.5013965822 17.1057707188 45.3172255098 10.4958733944 46.965785757 16.7892016247 49.9367453057 21.5900969196 46.7145179915 15.6132744651 38.3865255803 14.5802151464 50.2347207 20.4044729196 36.286143052 17.1057707266 37.5794219113 15.0158901333 +29.3462004281 42.1487418312 50.0164829541 11.9338520864 58.8391251216 16.6376352154 51.2821 14.8718 50.7336745372 16.9929634144 53.116915041 10.5068605973 49.5842392555 16.7579160147 53.8098729164 17.7664144512 48.4506176492 16.2122782224 38.4040152085 14.4519484496 50.2797016367 15.608737571 62.7866325947 13.9189931024 41.0643057894 12.2144689321 +82.1105579783 1.21055166293 58.6367650809 11.9747210652 60.7557891374 17.0332315064 54.1026 14.8718 50.7113785398 17.0883158538 57.049801823 10.5081369414 51.8562999655 21.969084007 59.8526782804 9.03994992133 59.6326101247 7.05803719901 38.7642788913 14.3655904744 50.2530067222 11.3020842468 66.8176792234 11.4124971575 42.195266147 20.9193721857 +38.3020058088 60.0177857932 54.7302890863 11.4128826732 54.7611561181 16.3661890954 56.1538 14.1026 50.8127448984 16.5727180506 54.8546309465 10.4639151924 55.9528886329 16.7897576048 48.1756567942 19.5212153543 61.6185682473 13.4108227184 41.4701423265 17.2780334445 50.4741411162 12.4619966676 66.755021412 18.0853051378 36.6055782383 15.748875885 +56.5841530218 70.512514809 65.8755478023 11.7324363627 53.5699050003 16.3397125506 52.0513 12.5641 51.014232955 17.2210955293 54.9248425638 10.498331942 51.6637862607 19.9251251977 50.7417190667 17.8384056087 48.8339891275 17.7259558674 47.1554048146 22.3779325323 50.1084399725 18.2590443442 65.4155283864 10.4635122068 39.4311612098 19.3238407541 +33.3393742865 0.5091067352 57.0609827131 9.92056085444 48.9146182416 16.7986370826 48.7179 11.0256 50.3535214052 17.0647430794 63.3095540306 10.4888142392 45.8596693166 17.1289036754 58.4597612975 14.1486628546 45.3426531889 13.9088123935 39.5825667453 17.6484528361 49.919058249 3.7168538855 36.9463314417 13.5143774996 37.5620763361 11.9284457044 +78.7742390407 35.4841012146 46.819907946 10.4946526804 52.6461373069 16.3848363836 47.1795 9.8718 50.4355295722 17.0651685025 49.7494984129 10.4858029097 39.7403795202 18.7410514146 50.1145975193 1.74146171252 42.4150122932 11.6937810961 41.7402438167 17.8293243101 50.0836645085 10.2358819024 37.8254347391 9.60103429534 34.2381100444 13.7131582305 +27.9226442446 25.9868781844 38.3593948725 13.431322624 49.8384929905 16.6763302777 46.1538 6.0256 50.6309819633 17.0765223454 41.1693104232 10.4973002433 52.8112216438 17.8549881176 45.4203820755 5.37240899542 58.9347650647 9.36920121911 39.3118718889 15.6407169743 50.3522608355 12.4686005273 36.722837943 9.33330210686 34.1409410386 4.57766135024 +71.6978651182 10.8681445111 47.3154157773 12.853451784 52.6840664009 16.4079627826 50.5128 9.4872 51.0668072036 17.2088597095 46.6883404976 10.5075321353 56.7891043639 17.3061155207 51.3532681954 0.30387242055 60.6836260407 10.6326415569 41.6798476925 17.7459190133 50.4975722411 7.40962523419 67.0733217806 6.04921458162 36.6592676972 17.6819644553 +74.1383313856 49.1739189791 55.0519165396 11.9499886181 65.8072190346 18.1407649473 53.8462 10.2564 50.7923547263 17.2042143387 52.4189219215 10.4959063596 54.1145317806 17.6735343896 53.091704424 19.2900304453 54.8117983683 6.56964870325 39.0874644519 15.1223039378 50.0163058346 16.2536847911 64.601818107 12.0019169643 40.6437192795 20.2408613716 +32.579020066 1.80811559665 50.5159602579 9.76559162187 60.6739209233 17.0518023644 57.4359 10.2564 50.5512780561 17.0846551833 50.8784883796 10.5030898746 55.571119732 17.369574581 58.9431665444 11.6128777918 61.5868424253 11.0275858356 41.4815028637 18.0474374446 50.4682212165 13.3567024555 65.4372821806 15.5453860785 36.7924815919 13.3921347304 +59.83218542 69.1525081443 49.6774146514 10.3831325083 63.394712866 17.7005351354 60.0 10.641 50.5597580562 17.0938837744 57.0533061394 10.5140692392 65.3064624975 18.1973548277 59.3109904219 10.5721363739 59.7160077628 11.5190968617 77.6060865492 15.1628725383 69.5506789842 9.03298756904 67.0040223792 15.3458266393 75.9483006427 22.7447959146 +35.0306285457 12.5366493416 67.2806595188 14.1286515323 64.8020094467 18.2168786335 64.1026 10.641 75.3259785471 15.7718919896 70.7864511597 10.5136950409 66.9907491546 17.2228706509 68.2221986595 1.13388036584 58.4163840114 5.64577748328 75.9826615205 16.3069223786 69.5186727475 15.1804864298 66.7241920702 5.24980548075 76.8406924717 20.9185472197 +74.3001198284 42.4770945921 66.1730182606 12.0379170217 65.0259398087 18.2878245002 66.9231 10.641 75.04472578 17.0042622562 56.7721495656 10.5032723508 72.1567189391 16.8142747914 62.5117719862 7.49123248279 59.243809528 10.5401420943 76.9457572424 15.858478334 69.5034574098 16.8783132661 68.3076187632 13.2809165227 81.9504776395 19.8745582085 +63.2501970628 65.9524861966 61.0885441374 10.0845344144 65.7555515404 18.5162158303 71.2821 10.641 75.2870877232 16.1749349097 75.4772220231 10.5185680355 70.9805804021 17.15575228 69.3800563617 1.48813233299 56.5943646385 8.91734404523 77.5437200743 15.253949149 69.4994868291 10.2078266798 68.7680482759 13.5214565521 81.6437056853 16.5917599845 +34.1730737648 25.6936743092 66.0530857731 13.3802260073 69.9677341245 20.5389887758 74.359 10.641 75.2399699777 17.0318474866 76.7659121498 10.496999064 72.2220822073 19.3692816955 82.8002478704 18.5293762368 68.4925132247 17.9873634289 77.5847398447 15.8300393854 69.5275296829 9.99229666551 74.1672717461 5.34988087254 84.6102197758 14.2477619017 +40.9138319319 38.5590195509 72.6699892655 15.2342259409 68.8927840241 20.0324488429 78.2051 10.641 75.1524592023 17.0049424015 77.621049129 10.4958742318 79.7001583481 20.740218365 85.1603321777 15.0183695439 65.0059431717 15.9126686464 76.8223042562 15.5951653195 69.5190952617 11.4226857018 64.9003579699 16.2452583689 84.1028794336 8.98065579006 +62.8332930874 62.5108942269 61.503472503 10.8284144754 61.8265465579 17.2696554452 67.9487 8.718 75.9618400861 16.6948422289 65.4144870523 10.6818401251 64.2491606461 14.9139624607 67.095847697 0.601490941965 63.9210367078 15.6209610332 77.3485716647 15.7745292395 69.5618994268 15.9998037858 68.7634335463 8.70057294385 80.18002545 19.0888441297 +42.4767923803 56.7312899691 68.995028632 13.9943105293 60.7879700808 17.0462048583 68.4615 5.2564 75.4480625064 17.0451494125 69.9647805002 10.5072329326 66.9432914334 20.1891325034 62.307750573 5.74997471945 53.5129705438 5.87510952836 77.5731526867 14.7806458332 69.5010445197 5.95583353945 66.8169139163 12.2732943361 80.5732729943 8.36563890168 +52.0334562787 64.5666620298 78.2499161658 17.8832409068 61.9156096372 17.2821912228 68.2051 2.9487 75.7593838168 16.9429296545 64.9697382623 10.4821595117 66.9416766634 18.2435093914 59.9704150458 7.9133104735 57.8328062743 9.22238462333 77.9726106776 14.9556987461 69.524635976 2.73476016988 67.3093466467 0.217006270014 79.0043268649 10.6281977654 +48.9070429644 74.2877488252 36.4819805699 15.1627600944 45.0289975331 17.7244473049 37.6923 25.7692 50.378262298 16.9462798113 38.0502407785 30.4079460343 39.6482666468 22.7313663718 39.9116351739 32.1440226145 31.0329174395 28.9281418896 41.5289197586 24.9164251896 30.4995937106 26.0855615691 34.7318293093 19.6017950724 40.0481864651 24.2614879334 +59.8518383854 72.9583909677 50.9677483784 29.6797742875 39.9213628084 19.7018495281 39.4872 25.3846 50.5336350094 17.0195813691 42.2241202189 30.4980371582 37.9497836858 26.5217492344 39.5965009783 32.1361317349 31.3332440932 32.5385694785 43.7225508034 19.0773278036 30.487391709 19.7794703733 33.6744421497 26.09049021 34.7945935378 13.9696834611 +48.9604602016 72.6295257275 91.1910536059 46.6743428399 84.7942778751 55.5686503735 91.2821 41.5385 77.5009073165 50.1669859475 87.6984642949 50.4899365296 86.5043905611 34.8940252874 95.5934163965 33.2341293953 86.4015498466 38.7469331743 79.3260781751 52.9003912897 89.5001804719 31.9789174357 75.6272550236 37.128751949 79.221764441 22.0945914837 +46.8448551673 36.7917136918 55.8637684939 85.3364867573 55.6629587449 83.3564796243 50.0 95.7692 50.6911241884 87.5139604241 52.3603310701 89.7043170068 50.8154853559 78.950838805 52.772288113 99.6134716833 47.4421116354 98.1843018589 56.6639740815 87.9401250067 50.4102716314 98.6283694405 40.6101254225 89.1362398699 36.0308797708 93.1217332374 +39.9630215796 42.9449148487 49.2805947987 84.0488228306 50.4922475132 78.997532065 47.9487 95.0 49.9903949511 83.997356924 51.4067867898 89.7121385499 51.1834666638 85.1288546497 53.1786144535 98.6038611855 46.2647413368 94.1161921613 57.8217892319 90.6931665489 50.3259243927 94.9946308528 39.1143664549 96.4817513202 34.4995583102 86.6099850511 +66.704943997 32.0150954299 43.3685015447 84.3321772039 51.4671011416 79.2018445773 44.1026 92.6923 50.1271820272 82.990749996 43.588468468 89.7455365536 40.8638552293 82.9261452983 37.6270534612 93.060672952 40.1638157437 87.4486723469 58.2431718991 92.1043278652 50.1040315041 95.0885380179 34.5838289299 89.5889019877 31.1068665551 89.4616352367 diff --git a/example/neural_spline_flow/data/data_proc.jl b/example/neural_spline_flow/data/data_proc.jl new file mode 100644 index 00000000..8d572ca1 --- /dev/null +++ b/example/neural_spline_flow/data/data_proc.jl @@ -0,0 +1,7 @@ +using CSV, DataFrames + +# Read the .tsv file +dat = CSV.read("example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv", DataFrame) +dat_dino = permutedims(hcat(dat.dino, dat.dino_1)[2:end, :]) +dat_dino = map(x -> parse(Float32, x), dat_dino) +dat = vcat(dat_dino, dat_dino) diff --git a/example/neural_spline_flow/main.jl b/example/neural_spline_flow/main.jl new file mode 100644 index 00000000..76abee5c --- /dev/null +++ b/example/neural_spline_flow/main.jl @@ -0,0 +1,56 @@ +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using FunctionChains +using NormalizingFlows +using Zygote +using Flux: f32 +using Plots + +const NF = NormalizingFlows +include("../common.jl") +include("nsf_layer.jl") + +Random.seed!(123) +rng = Random.default_rng() + +###################################### +# load dataset +###################################### +include("../targets/banana.jl") +p = Banana(2, 1.0f-1, 100.0f0) +logp = Base.Fix1(logpdf, p) + +d = 2 +hdims = 10 +K = 8 +B = 3 +Ls = [ + NeuralSplineLayer(d, hdims, K, B, [1]) ∘ NeuralSplineLayer(d, hdims, K, B, [2]) for + i in 1:2 +] +q0 = MvNormal(zeros(Float32, d), I) +flow = Bijectors.transformed(q0, ∘(Ls...)) +# flow = Bijectors.transformed(q0, trans) +flow_untrained = deepcopy(flow) + +sample_per_iter = 10 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(1e-2), + callback=cb, + ADbackend=AutoZygote(), + hasconverged=checkconv, +) +losses = map(x -> x.loss, stats) + +###################################### +# evaluate trained flow (one can see that the performance is better than the plannar flow) +###################################### +plot(losses; label="Loss", linewidth=2) # plot the loss \ No newline at end of file diff --git a/example/neural_spline_flow/nsf_layer.jl b/example/neural_spline_flow/nsf_layer.jl index 6f3b2a7f..2353c634 100644 --- a/example/neural_spline_flow/nsf_layer.jl +++ b/example/neural_spline_flow/nsf_layer.jl @@ -3,93 +3,114 @@ using Functors using Bijectors using Bijectors: partition, PartitionMask +include("../util.jl") """ Neural Rational quadratic Spline layer # References [1] Durkan, C., Bekasov, A., Murray, I., & Papamakarios, G., Neural Spline Flows, CoRR, arXiv:1906.04032 [stat.ML], (2019). """ -struct NeuralSplineLayer{T1,T2,A<:AbstractVecOrMat{T1}} <: Bijectors.Bijector +# struct NeuralSplineLayer{T1,T2,A<:AbstractVecOrMat{T1}} <: Bijectors.Bijector +# dim::Int +# mask::Bijectors.PartitionMask +# w::A # width +# h::A # height +# d::A # derivative of the knots +# B::T2 # bound of the knots +# end + +# function NeuralSplineLayer( +# dim::Int, # dimension of input +# hdims::Int, # dimension of hidden units for s and t +# K::Int, # number of knots +# B::T2, # bound of the knots +# mask_idx::AbstractVector{<:Int}, # index of dimensione that one wants to apply transformations on +# ) where {T2<:Real} +# num_of_transformed_dims = length(mask_idx) +# input_dims = dim - num_of_transformed_dims +# w = fill(MLP_3layer(input_dims, hdims, K), num_of_transformed_dims) +# h = fill(MLP_3layer(input_dims, hdims, K), num_of_transformed_dims) +# d = fill(MLP_3layer(input_dims, hdims, K - 1), num_of_transformed_dims) +# mask = Bijectors.PartitionMask(dim, mask_idx) +# return NeuralSplineLayer(dim, mask, w, h, d, B) +# end + +# @functor NeuralSplineLayer (w, h, d) + +# # define forward and inverse transformation +# function instantiate_rqs(nsl::NeuralSplineLayer, x::AbstractVector) +# # instantiate rqs knots and derivatives +# ws = permutedims(reduce(hcat, [w(x) for w in nsl.w])) +# hs = permutedims(reduce(hcat, [h(x) for h in nsl.h])) +# ds = permutedims(reduce(hcat, [d(x) for d in nsl.d])) +# return Bijectors.RationalQuadraticSpline(ws, hs, ds, nsl.B) +# end + +## Question: which one is better, the struct below or the struct above? +struct NeuralSplineLayer{T,A<:Flux.Chain} <: Bijectors.Bijector dim::Int + K::Int + nn::AbstractVector{A} # networks that parmaterize the knots and derivatives + B::T # bound of the knots mask::Bijectors.PartitionMask - w::A # width - h::A # height - d::A # derivative of the knots - B::T2 # bound of the knots -end - -function MLP_3layer(input_dim::Int, hdims::Int, output_dim::Int; activation=Flux.leakyrelu) - return Chain( - Flux.Dense(input_dim, hdims, activation), - Flux.Dense(hdims, hdims, activation), - Flux.Dense(hdims, output_dim), - ) end function NeuralSplineLayer( - dim::Int, # dimension of input - hdims::Int, # dimension of hidden units for s and t - K::Int, # number of knots - mask_idx::AbstractVector{<:Int}, # index of dimensione that one wants to apply transformations on + dim::T1, # dimension of input + hdims::T1, # dimension of hidden units for s and t + K::T1, # number of knots B::T2, # bound of the knots -) where {T2<:Real} + mask_idx::AbstractVector{<:Int}, # index of dimensione that one wants to apply transformations on +) where {T1<:Int,T2<:Real} num_of_transformed_dims = length(mask_idx) input_dims = dim - num_of_transformed_dims - w = [MLP_3layer(input_dims, hdims, K) for i in 1:num_of_transformed_dims] - h = [MLP_3layer(input_dims, hdims, K) for i in 1:num_of_transformed_dims] - d = [MLP_3layer(input_dims, hdims, K - 1) for i in 1:num_of_transformed_dims] - mask = Bijectors.PartitionMask(D, mask_idx) - return NeuralSplineLayer(D, mask, w, h, d, B) + nn = fill(MLP_3layer(input_dims, hdims, 3K - 1), num_of_transformed_dims) + mask = Bijectors.PartitionMask(dim, mask_idx) + return NeuralSplineLayer(dim, K, nn, B, mask) end -@functor NeuralSplineLayer (w, h, d) +@functor NeuralSplineLayer (nn,) # define forward and inverse transformation -function instantiate_rqs(nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector) +function instantiate_rqs(nsl::NeuralSplineLayer, x::AbstractVector) # instantiate rqs knots and derivatives - ws = permutedims(reduce(hcat, [w(x) for w in nsl.w])) - hs = permutedims(reduce(hcat, [h(x) for h in nsl.h])) - ds = permutedims(reduce(hcat, [d(x) for d in nsl.d])) - return Bijectors.RationalQuadraticSpline(ws, hs, ds, nsl.B) + T = permutedims(reduce(hcat, map(nn -> nn(x), nsl.nn))) + K, B = nsl.K, nsl.B + ws = T[:, 1:K] + hs = T[:, (K + 1):(2K)] + ds = T[:, (2K + 1):(3K - 1)] + return Bijectors.RationalQuadraticSpline(ws, hs, ds, B) end -function Bijectors.transform( - nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector -) +function Bijectors.transform(nsl::NeuralSplineLayer, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) # instantiate rqs knots and derivatives rqs = instantiate_rqs(nsl, x_2) - y_1 = transform(rqs, x_1) + y_1 = Bijectors.transform(rqs, x_1) return Bijectors.combine(nsl.mask, y_1, x_2, x_3) end -function Bijectors.transform( - insl::Inverse{<:NeuralSplineLayer{<:Vector{<:Flux.Chain}}}, y::AbstractVector -) +function Bijectors.transform(insl::Inverse{<:NeuralSplineLayer}, y::AbstractVector) nsl = insl.orig y1, y2, y3 = partition(nsl.mask, y) rqs = instantiate_rqs(nsl, y2) - x1 = transform(Inverse(rqs), y1) - return combine(nsl.mask, x1, y2, y3) + x1 = Bijectors.transform(Inverse(rqs), y1) + return Bijectors.combine(nsl.mask, x1, y2, y3) end -function (nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}})(x::AbstractVector) +function (nsl::NeuralSplineLayer)(x::AbstractVector) return Bijectors.transform(nsl, x) end # define logabsdetjac -function Bijectors.logabsdetjac( - nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector -) +function Bijectors.logabsdetjac(nsl::NeuralSplineLayer, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) - Rqs = instantiate_rqs(nsl, x_2) - logjac = logabsdetjac(Rqs, x_1) + rqs = instantiate_rqs(nsl, x_2) + logjac = logabsdetjac(rqs, x_1) return logjac end -function Bijectors.logabsdetjac( - insl::Inverse{<:NeuralSplineLayer{<:Vector{<:Flux.Chain}}}, y::AbstractVector -) +function Bijectors.logabsdetjac(insl::Inverse{<:NeuralSplineLayer}, y::AbstractVector) nsl = insl.orig y1, y2, y3 = partition(nsl.mask, y) rqs = instantiate_rqs(nsl, y2) @@ -97,9 +118,7 @@ function Bijectors.logabsdetjac( return logjac end -function Bijectors.with_logabsdet_jacobian( - nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector -) +function Bijectors.with_logabsdet_jacobian(nsl::NeuralSplineLayer, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) rqs = instantiate_rqs(nsl, x_2) y_1, logjac = with_logabsdet_jacobian(rqs, x_1) diff --git a/example/planar_radial_flow/planar_flow.jl b/example/planar_radial_flow/planar_flow_main.jl similarity index 92% rename from example/planar_radial_flow/planar_flow.jl rename to example/planar_radial_flow/planar_flow_main.jl index d1ccff81..c79077d9 100644 --- a/example/planar_radial_flow/planar_flow.jl +++ b/example/planar_radial_flow/planar_flow_main.jl @@ -37,15 +37,17 @@ flow_untrained = deepcopy(flow) # train the flow sample_per_iter = 10 cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 flow_trained, stats, _ = train_flow( elbo, flow, logp, sample_per_iter; max_iters=200_00, - optimiser=Optimisers.ADAM(), + optimiser=Optimisers.Adam(), callback=cb, ADbackend=AutoZygote(), + hasconverged=checkconv, ) losses = map(x -> x.loss, stats) diff --git a/example/planar_radial_flow/radial_flow.jl b/example/planar_radial_flow/radial_flow_main.jl similarity index 97% rename from example/planar_radial_flow/radial_flow.jl rename to example/planar_radial_flow/radial_flow_main.jl index 4d51cd24..70085d11 100644 --- a/example/planar_radial_flow/radial_flow.jl +++ b/example/planar_radial_flow/radial_flow_main.jl @@ -43,7 +43,7 @@ flow_trained, stats, _ = train_flow( logp, sample_per_iter; max_iters=200_00, - optimiser=Optimisers.ADAM(), + optimiser=Optimisers.Adam(), callback=cb, ) losses = map(x -> x.loss, stats) diff --git a/example/util.jl b/example/util.jl new file mode 100644 index 00000000..d98bff82 --- /dev/null +++ b/example/util.jl @@ -0,0 +1,9 @@ +using Flux + +function MLP_3layer(input_dim::Int, hdims::Int, output_dim::Int; activation=Flux.leakyrelu) + return Chain( + Flux.Dense(input_dim, hdims, activation), + Flux.Dense(hdims, hdims, activation), + Flux.Dense(hdims, output_dim), + ) +end \ No newline at end of file diff --git a/src/objectives/loglikelihood.jl b/src/objectives/loglikelihood.jl index 8564793d..fc2f7369 100644 --- a/src/objectives/loglikelihood.jl +++ b/src/objectives/loglikelihood.jl @@ -21,4 +21,30 @@ function loglikelihood( ) llhs = map(x -> logpdf(flow, x), eachcol(xs)) return mean(llhs) +end + +function loglikelihood( + rng::AbstractRNG, + flow::Bijectors.UnivariateTransformed, + fulldata::AbstractVector, + batchsize::Int, +) + N = length(fulldata) + @assert batchsize <= N + idx = sample(rng, 1:N, batchsize; replace=false) + xs = fulldata[idx] + return loglikelihood(flow, xs) +end + +function loglikelihood( + rng::AbstractRNG, + flow::Bijectors.MultivariateTransformed, + fulldata::AbstractMatrix, + batchsize::Int, +) + N = size(fulldata, 2) + @assert batchsize <= N + idx = sample(rng, 1:N, batchsize; replace=false) + xs = fulldata[:, idx] + return loglikelihood(flow, xs) end \ No newline at end of file diff --git a/src/train.jl b/src/train.jl index 5edd9a02..3c236bf6 100644 --- a/src/train.jl +++ b/src/train.jl @@ -159,5 +159,5 @@ function optimize( end # return status of the optimiser for potential continuation of training - return θ, map(identity, opt_stats), st + return θ, map(identity, opt_stats[1:(i - 1)]), st end diff --git a/src/util.jl b/src/util.jl new file mode 100644 index 00000000..e63c5895 --- /dev/null +++ b/src/util.jl @@ -0,0 +1,61 @@ +using Functors +using Bijectors, Flux + +# falttening and unflattening approach from: https://github.com/TuringLang/AdvancedVI.jl/discussions/46#discussioncomment-5543111 +inner_flatten(x::Real) = [x], only +inner_flatten(x::AbstractVector{<:Real}) = x, identity +inner_flatten(x::AbstractArray{<:Real}) = vec(x), Base.Fix2(reshape, size(x)) + +function inner_flatten(original::Tuple) + vecs_and_unflattens = map(inner_flatten, original) + vecs = map(first, vecs_and_unflattens) + unflattens = map(last, vecs_and_unflattens) + + lengths = map(length, vecs) + end_indices = cumsum(lengths) + + N = length(original) + + function unflatten_Tuple(x) + ntuple(N) do i + v = original[i] + unflatten = unflattens[i] + l = lengths[i] + end_idx = end_indices[i] + start_idx = end_idx - l + 1 + return unflatten(@view(x[start_idx:end_idx])) + end + end + + return reduce(vcat, vecs), unflatten_Tuple +end + +function inner_flatten(x::NamedTuple{names}) where {names} + vec, re = inner_flatten(values(x)) + unflatten_NamedTuple(x) = NamedTuple{names}(re(x)) + return vec, unflatten_NamedTuple +end + +""" + flatten(x) + +Return a the flattened vector of parameters and a function to reconstruct `x` from the vector. + +This uses `Functors.functor` to get the parameters of `x` and `inner_flatten` to flatten them. +""" +function flatten(x) + params, re = Functors.functor(x) + params_vec, re_vec = inner_flatten(params) + return params_vec, re ∘ re_vec +end + +# Functors for Composed type +function Functors.functor(::Type{<:ComposedFunction}, f) + outer, re_outer = Functors.functor(f.outer) + inner, re_inner = Functors.functor(f.inner) + function reconstruct_ComposedFunction(x) + return ComposedFunction(re_outer(x.outer), re_inner(x.inner)) + end + + return (; outer, inner), reconstruct_ComposedFunction +end