diff --git a/core/src/main/java/zingg/util/DSUtil.java b/core/src/main/java/zingg/util/DSUtil.java index ac02c154b..4d970d965 100644 --- a/core/src/main/java/zingg/util/DSUtil.java +++ b/core/src/main/java/zingg/util/DSUtil.java @@ -235,7 +235,7 @@ private static Dataset getTraining(SparkSession spark, Arguments args, Pipe Dataset trSamples = PipeUtil.read(spark, true, false, args.getTrainingSamples()); LOG.warn("Read all training samples "); - trFile = (trFile == null) ? trSamples : trFile.unionByName(trSamples); + trFile = (trFile == null) ? trSamples : trFile.unionByName(trSamples, true); } else { LOG.warn("No configured training samples"); diff --git a/examples/febrl/configWithTrainingSamples.json b/examples/febrl/configWithTrainingSamples.json index 130e87749..4f110d830 100644 --- a/examples/febrl/configWithTrainingSamples.json +++ b/examples/febrl/configWithTrainingSamples.json @@ -14,7 +14,6 @@ {\"name\" : \"z_cluster\",\"type\" : \"string\",\"nullable\" : false, \"metadata\" : {}}, {\"name\":\"z_ismatch\",\"type\":\"integer\",\"nullable\":true,\"metadata\" : {}}, - {\"name\":\"id\", \"type\":\"string\", \"nullable\":false,\"metadata\" : {}}, {\"name\":\"fname\", \"type\":\"string\", \"nullable\":true,\"metadata\" : {}}, {\"name\":\"lname\",\"type\":\"string\",\"nullable\":true,\"metadata\" : {}}, {\"name\":\"stNo\", \"type\":\"string\", \"nullable\":true,\"metadata\" : {}}, diff --git a/examples/febrl/training.csv b/examples/febrl/training.csv index e0f439650..4cf9cc652 100644 --- a/examples/febrl/training.csv +++ b/examples/febrl/training.csv @@ -1,70 +1,70 @@ -10,1,rec-10-dup-0, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769 -10,1,rec-10-dup-1, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769 -10,1,rec-10-dup-2, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769 -10,1,rec-10-dup-3, mikhayla, hannagan,, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769 -10,1,rec-10-org, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769 -10,1,rec-10-org, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,,1030769 -100,1,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843 -100,1,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,, q,19161017,4975843 -100,1,rec-100-dup-1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843 -100,1,rec-100-dup-2, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161027,4975843 -100,1,rec-100-dup-3, domenique, paterson,15, lubum a place, merewether pub lic school, greensborough,3138, qld,19161017,4975843 -100,1,rec-100-dup-4, domenique, paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843 -100,1,rec-100-dup-4, , paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843 -100,1,rec-100-org, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161017,4975843 -100,1,rec-100-org, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,,4975843 -1000,1,rec-1000-dup-0, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902 -1000,1,rec-1000-dup-1, sophie, waller,24, suunybrae, goldfinch circuit, mona vale,2285, vic,19380705,2972902 -1000,1,rec-1000-dup-2, sophie, waller,24, suunybrae, , mona vale,2285, vic,19380705,2972902 -1000,1,rec-1000-org, sophie, waller,24, goldfinch circuit, suunybrae, mona vale,2285, vic,19380705,2972902 -1001,1,rec-1001-dup-0, , pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19991009,1859104 -1001,1,rec-1001-org, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104 -1001,1,rec-1001-org, casey, ,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104 -1002,1,rec-1002-dup-0, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819 -1002,1,rec-1002-org, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819 -1002,1,rec-1002-org, , matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819 -1002,1,rec-1002-dup-0, xavier, ,10, , rank in park, kellyville,3039, vdc,19620202,6169819 -1004,1,rec-1004-dup-0, tayla, green,41, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673 -1004,1,rec-1004-dup-1, tayla, greecn,38, kambalda crescent, ambevrale, bonville,2905, vic,19310175,2230673 -1004,1,rec-1004-dup-2, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673 -1004,1,rec-1004-org, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673 -1004,1,rec-1004-org, tayla, gren,38, kambaldcrescent, ambervale, bonville,2905, ,19310715,2230673 -1006,1,rec-1006-dup-0, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819 -1006,1,rec-1006-dup-1, sam, bentham,1, narran street, lake chatsworth, westlake,2306, vic, ,8928819 -1006,1,rec-1006-dup-2, sam, bentham,1, narranstreet, lake chagsworth, westlake,2036, vic, ,8928819 -1006,1,rec-1006-dup-3, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic,19330165,8928819 -1006,1,rec-1006-dup-4, sam, bentham,1, narranystreet, , westl ake,2036, vic,19330165,8928819 -1006,1,rec-1006-org, , bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819 -1007,1,rec-1007-dup-0, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023 -1007,1,rec-1007-dup-1, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023 -1007,1,rec-1007-dup-2, rebekah, modystach,34, tausx place, pyramid cara park, iluka,2216, nsp,19341113,8796023 -1007,1,rec-1007-dup-3, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, nsp,19341113,8796023 -1007,1,rec-1007-dup-4, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, np,19341113,8796023 -1007,1,rec-1007-org, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023 -n0,0,rec-0-org, jinni, dreyer,11, were street, marriott downs, south melbourne,3172, nsw,19420127,3787407 -n0,0,rec-1-org, jonah, browne,6, bingley crescent, , pindimar,3182, nsw,19250413,8328406 -n10,0,rec-10-dup-0, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769 -n10,0,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843 -n11,0,rec-10-dup-1, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769 -n11,0,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843 -n12,0,rec-10-dup-2, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769 -n12,0,rec-100-dup-1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843 -n13,0,rec-100-dup-1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843 -n13,0,rec-13-org, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902 -n1001,0,rec-1001-org, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104 -n1001,0,rec-1002-dup-0, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819 -n1002,0,rec-1002-org, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819 -n1002,0,rec-1003-org, connor, clarke,39, de chair street, wilding, peakhurst,5204, vic,19450907,1230316 -n1004,0,rec-1004-dup-2, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673 -n1004,0,rec-1005-org, jamie, kranz,8, , rmbh 884, port macquarie,5085, vic,19921217,4602627 -n1005,0,rec-1004-org, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673 -n1005,0,rec-1006-dup-0, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819 -n1006,0,rec-1006-org, sam, bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819 -n1006,0,rec-1007-dup-0, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023 -n1007,0,rec-1007-dup-1, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023 -n1007,0,rec-1008-org, alex, ballantyne,212, aronson crescent, logan central plaza, semaphore park,2153, wa,19450621,4145113 -n1008,0,rec-1007-org, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023 -n1008,0,rec-1085-org, schkirra, walpole,4, namadgi circuit, rydellan, bibra lake,2234, nsw,19560330,1950180 -n511,0,rec-511-org-0, mason, ashlie,6, sinclair street, young retirement village, ,3201, nsw,19540731,9955736 -n511,0,rec-1093-org, dylan, wilkins,18, denovan circuit, two tree hill, brighton-le-sands,2320, nsw, ,8735875 -n511,0,rec-74-org, jordan, gold,18, sharwood crescent, sunforest therapy centre, belmont,2155, nsw,19941117,1139906 +10,1, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769 +10,1, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769 +10,1, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769 +10,1, mikhayla, hannagan,, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769 +10,1, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769 +10,1, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,,1030769 +100,1, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843 +100,1, domenique, paterson,15, , merewether pufblic school, greensborough,, q,19161017,4975843 +100,1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843 +100,1, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161027,4975843 +100,1, domenique, paterson,15, lubum a place, merewether pub lic school, greensborough,3138, qld,19161017,4975843 +100,1, domenique, paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843 +100,1, , paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843 +100,1, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161017,4975843 +100,1, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,,4975843 +1000,1, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902 +1000,1, sophie, waller,24, suunybrae, goldfinch circuit, mona vale,2285, vic,19380705,2972902 +1000,1, sophie, waller,24, suunybrae, , mona vale,2285, vic,19380705,2972902 +1000,1, sophie, waller,24, goldfinch circuit, suunybrae, mona vale,2285, vic,19380705,2972902 +1001,1, , pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19991009,1859104 +1001,1, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104 +1001,1, casey, ,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104 +1002,1, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819 +1002,1, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819 +1002,1, , matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819 +1002,1, xavier, ,10, , rank in park, kellyville,3039, vdc,19620202,6169819 +1004,1, tayla, green,41, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673 +1004,1, tayla, greecn,38, kambalda crescent, ambevrale, bonville,2905, vic,19310175,2230673 +1004,1, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673 +1004,1, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673 +1004,1, tayla, gren,38, kambaldcrescent, ambervale, bonville,2905, ,19310715,2230673 +1006,1, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819 +1006,1, sam, bentham,1, narran street, lake chatsworth, westlake,2306, vic, ,8928819 +1006,1, sam, bentham,1, narranstreet, lake chagsworth, westlake,2036, vic, ,8928819 +1006,1, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic,19330165,8928819 +1006,1, sam, bentham,1, narranystreet, , westl ake,2036, vic,19330165,8928819 +1006,1, , bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819 +1007,1, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023 +1007,1, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023 +1007,1, rebekah, modystach,34, tausx place, pyramid cara park, iluka,2216, nsp,19341113,8796023 +1007,1, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, nsp,19341113,8796023 +1007,1, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, np,19341113,8796023 +1007,1, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023 +n0,0, jinni, dreyer,11, were street, marriott downs, south melbourne,3172, nsw,19420127,3787407 +n0,0, jonah, browne,6, bingley crescent, , pindimar,3182, nsw,19250413,8328406 +n10,0, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769 +n10,0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843 +n11,0, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769 +n11,0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843 +n12,0, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769 +n12,0, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843 +n13,0, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843 +n13,0, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902 +n1001,0, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104 +n1001,0, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819 +n1002,0, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819 +n1002,0, connor, clarke,39, de chair street, wilding, peakhurst,5204, vic,19450907,1230316 +n1004,0, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673 +n1004,0, jamie, kranz,8, , rmbh 884, port macquarie,5085, vic,19921217,4602627 +n1005,0, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673 +n1005,0, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819 +n1006,0, sam, bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819 +n1006,0, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023 +n1007,0, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023 +n1007,0, alex, ballantyne,212, aronson crescent, logan central plaza, semaphore park,2153, wa,19450621,4145113 +n1008,0, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023 +n1008,0, schkirra, walpole,4, namadgi circuit, rydellan, bibra lake,2234, nsw,19560330,1950180 +n511,0, mason, ashlie,6, sinclair street, young retirement village, ,3201, nsw,19540731,9955736 +n511,0, dylan, wilkins,18, denovan circuit, two tree hill, brighton-le-sands,2320, nsw, ,8735875 +n511,0, jordan, gold,18, sharwood crescent, sunforest therapy centre, belmont,2155, nsw,19941117,1139906