HIVE-18908: FULL OUTER JOIN to MapJoin (Matt McCline, reviewed by Ted…

…dy Choi)

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -1679,7 +1679,15 @@ public static enum ConfVars {
         "joins unnecessary memory will be allocated and then trimmed."),
     HIVEHYBRIDGRACEHASHJOINBLOOMFILTER("hive.mapjoin.hybridgrace.bloomfilter", true, "Whether to " +
         "use BloomFilter in Hybrid grace hash join to minimize unnecessary spilling."),
-
+    HIVEMAPJOINFULLOUTER("hive.mapjoin.full.outer", true,
+        "Whether to use MapJoin for FULL OUTER JOINs."),
+    HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE(
+        "hive.test.mapjoin.full.outer.override",
+        "none", new StringSet("none", "enable", "disable"),
+        "internal use only, used to override the hive.mapjoin.full.outer\n" +
+        "setting.  Using enable will force it on and disable will force it off.\n" +
+        "The default none is do nothing, of course",
+        true),
     HIVESMBJOINCACHEROWS("hive.smbjoin.cache.rows", 10000,
         "How many rows with the same key value should be cached in memory per smb joined table."),
     HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000,

data/files/fullouter_long_big_1a.txt

@@ -0,0 +1,11 @@
+-5310365297525168078
+-6187919478609154811
+968819023021777205
+3313583664488247651
+-5206670856103795573
+\N
+-6187919478609154811
+1569543799237464101
+-6187919478609154811
+-8460550397108077433
+-6187919478609154811

data/files/fullouter_long_big_1a_nonull.txt

@@ -0,0 +1,10 @@
+1569543799237464101
+-6187919478609154811
+968819023021777205
+-8460550397108077433
+-6187919478609154811
+-5310365297525168078
+-6187919478609154811
+-5206670856103795573
+3313583664488247651
+-6187919478609154811

data/files/fullouter_long_big_1b.txt

@@ -0,0 +1,13 @@
+\N
+31713
+31713
+31713
+31713
+32030
+31713
+-25394
+31713
+31713
+31713
+31713
+31713

data/files/fullouter_long_big_1c.txt

@@ -0,0 +1,11 @@
+1928928239,\N
+-1437463633,YYXPPCH
+-1437463633,TKTKGVGFW
+1725068083,MKSCCE
+1928928239,\N
+\N,ABBZ
+1928928239,AMKTIWQ
+-1437463633,JU
+1928928239,VAQHVRI
+-1437463633,SOWDWMS
+-1437463633,\N

data/files/fullouter_long_big_1d.txt

@@ -0,0 +1,12 @@
+-702028721
+-702028721
+-1780951928
+-670834064
+-814597051
+\N
+-814597051
+-814597051
+-702028721
+-2038654700
+\N
+-814597051

data/files/fullouter_long_small_1a.txt

@@ -0,0 +1,54 @@
+-1339636982994067311,2000-06-20
+-2575185053386712613,2105-01-21
+\N,2098-02-10
+-6784441713807772877,1845-02-16
+\N,2024-01-23
+-4224290881682877258,2185-07-08
+-614848861623872247,2101-05-25
+-2098090254092150988,2163-05-26
+434940853096155515,2275-02-08
+3873405809071478736,2034-06-09
+-2184423060953067642,1880-10-06
+7297177530102477725,1921-05-11
+7937120928560087303,2083-03-14
+\N,2242-02-08
+-2688622006344936758,2129-01-11
+214451696109242839,1977-01-04
+-4961171400048338491,2196-08-10
+4436884039838843341,2031-05-23
+2438535236662373438,1916-01-10
+6049335087268933751,2282-06-09
+8755921538765428593,1827-05-01
+5252407779338300447,2039-03-10
+-2184423060953067642,1853-07-06
+7297177530102477725,1926-04-12
+-2098090254092150988,1817-03-12
+-5754527700632192146,1958-07-15
+-614848861623872247,2112-11-09
+5246983111579595707,1817-07-01
+-2098090254092150988,2219-12-23
+-5706981533666803767,2151-06-09
+7297177530102477725,2125-08-26
+-7707546703881534780,2134-08-20
+214451696109242839,2179-04-18
+3845554233155411208,1805-11-10
+3905351789241845882,2045-12-05
+2438535236662373438,2026-06-23
+-2688622006344936758,1948-10-15
+6049335087268933751,2086-12-17
+-2575185053386712613,1809-07-12
+-327698348664467755,2222-10-15
+-4224290881682877258,1813-05-17
+3873405809071478736,2164-04-23
+-5706981533666803767,1800-09-20
+214451696109242839,1855-05-12
+2438535236662373438,1881-09-16
+5252407779338300447,2042-04-26
+-3655445881497026796,2108-08-16
+3905351789241845882,1866-07-28
+-6784441713807772877,2054-06-17
+5246983111579595707,2260-05-11
+-1339636982994067311,2008-12-03
+3873405809071478736,1918-11-20
+-4224290881682877258,2120-01-16
+3845554233155411208,2264-04-05

data/files/fullouter_long_small_1a_nonull.txt

@@ -0,0 +1,51 @@
+5246983111579595707,1817-07-01
+4436884039838843341,2031-05-23
+-4224290881682877258,1813-05-17
+-4961171400048338491,2196-08-10
+-2575185053386712613,2105-01-21
+5252407779338300447,2042-04-26
+-614848861623872247,2101-05-25
+-2098090254092150988,2163-05-26
+2438535236662373438,1881-09-16
+214451696109242839,2179-04-18
+2438535236662373438,2026-06-23
+-2184423060953067642,1853-07-06
+3873405809071478736,2164-04-23
+214451696109242839,1855-05-12
+-6784441713807772877,1845-02-16
+-2688622006344936758,1948-10-15
+7297177530102477725,1921-05-11
+-2575185053386712613,1809-07-12
+3905351789241845882,2045-12-05
+3845554233155411208,1805-11-10
+-3655445881497026796,2108-08-16
+3905351789241845882,1866-07-28
+-1339636982994067311,2008-12-03
+7297177530102477725,2125-08-26
+7297177530102477725,1926-04-12
+-5706981533666803767,1800-09-20
+6049335087268933751,2282-06-09
+3845554233155411208,2264-04-05
+8755921538765428593,1827-05-01
+-1339636982994067311,2000-06-20
+-2098090254092150988,1817-03-12
+3873405809071478736,2034-06-09
+2438535236662373438,1916-01-10
+5246983111579595707,2260-05-11
+-5706981533666803767,2151-06-09
+-614848861623872247,2112-11-09
+-327698348664467755,2222-10-15
+-2184423060953067642,1880-10-06
+434940853096155515,2275-02-08
+-4224290881682877258,2120-01-16
+-5754527700632192146,1958-07-15
+-4224290881682877258,2185-07-08
+-2098090254092150988,2219-12-23
+-7707546703881534780,2134-08-20
+214451696109242839,1977-01-04
+-2688622006344936758,2129-01-11
+7937120928560087303,2083-03-14
+-6784441713807772877,2054-06-17
+3873405809071478736,1918-11-20
+6049335087268933751,2086-12-17
+5252407779338300447,2039-03-10

data/files/fullouter_long_small_1b.txt

@@ -0,0 +1,72 @@
+2748,2298-06-20 21:01:24
+11232,2533-11-26 12:22:18
+\N,2124-05-07 15:01:19.021
+3198,2428-06-13 16:21:33.955
+-7624,2219-12-03 17:07:19
+24870,2752-12-26 12:32:23.03685163
+14865,2943-03-21 00:42:10.505
+-8624,2644-05-04 04:45:07.839
+-30059,2269-05-04 21:23:44.000339209
+14865,2079-10-06 16:54:35.117
+-8435,2834-12-06 16:38:18.901
+10553,2168-05-05 21:10:59.000152113
+-8624,2282-03-28 07:58:16
+-15361,2219-09-15 20:15:03.000169887
+-14172,1918-09-13 11:44:24.496926711
+26484,1919-03-04 07:32:37.519
+-14172,2355-01-14 23:23:34
+-24775,2920-08-06 15:58:28.261059449
+-23117,2037-01-05 21:52:30.685952759
+17125,2236-07-14 01:54:40.927230276
+21181,2253-03-12 11:55:48.332
+-7373,2662-10-28 12:07:02.000526564
+-8087,2550-06-26 23:57:42.588007617
+29407,2385-12-14 06:03:39.597
+21181,2434-02-20 00:46:29.633
+-14172,2809-06-07 02:10:58
+13598,2421-05-20 14:18:31.000264698
+2748,2759-02-13 18:04:36.000307355
+-22422,1949-03-13 00:07:53.075
+26484,2953-03-10 02:05:26.508953676
+4510,2777-03-24 03:44:28.000169723
+-24775,2035-03-26 08:11:23.375224153
+-30059,2713-10-13 09:28:49
+-20517,2774-06-23 12:04:06.5
+11232,2038-04-06 14:53:59
+32030,2101-09-09 07:35:05.145
+-29600,2333-11-02 15:06:30
+-30306,2619-05-24 10:35:58.000774018
+-7624,2289-08-28 00:14:34
+-4279,2470-08-12 11:21:14.000955747
+-4279,2214-09-10 03:53:06
+-26998,2428-12-26 07:53:45.96925825
+17125,2629-11-15 15:34:52
+-8087,2923-07-02 11:40:26.115
+2632,2561-12-15 15:42:27
+21436,2696-05-08 05:19:24.112
+\N,2971-08-07 12:02:11.000948152
+-7624,2623-03-20 03:18:45.00006465
+-26998,2926-07-18 09:02:46.077
+11232,2507-01-27 22:04:22.49661421
+-30059,2420-12-10 22:12:30
+-15427,2355-01-08 12:34:11.617
+3198,2223-04-14 13:20:49
+-19167,2319-08-26 11:07:11.268
+14865,2220-02-28 03:41:36
+-20517,2233-12-20 04:06:56.666522799
+-15427,2046-06-07 22:58:40.728
+2748,2862-04-20 13:12:39.482805897
+-8435,2642-02-07 11:45:04.353231638
+-19167,2230-12-22 20:25:39.000242111
+-15427,2023-11-09 19:31:21
+13598,2909-06-25 23:22:50
+21436,2526-09-22 23:44:55
+-15361,2434-08-13 20:37:07.000172979
+4510,2293-01-17 13:47:41.00001006
+-8624,2120-02-15 15:36:40.000758423
+-22422,2337-07-19 06:33:02.000353352
+-26998,2268-08-04 12:48:11.848006292
+-22422,2982-12-28 06:30:26.000883228
+\N,2933-06-20 11:48:09.000839488
+3198,2736-12-20 03:59:50.343550301
+-20824,2478-11-05 00:28:05

data/files/fullouter_long_small_1c.txt

@@ -0,0 +1,81 @@
+-1093006502,-69.55665828
+452719211,83003.43722
+1242586043,71.1485
+-934092157,-7843850349.57130038
+294598722,-3542.6
+284554389,5.727146
+90660785,12590.288613
+-99948814,-38076694.3981
+466567142,-9763217822.129028
+1909136587,-8610.078036935181
+1242586043,-4
+\N,1.089120893565337
+1039864870,987601.57
+-466171792,0
+-1681455031,-6.4543
+1755897735,-39.965207
+1585021913,745222.66808954
+448130683,-4302.485366846491
+193709887,0.8
+-424713789,0.48
+1585021913,607.22747
+-1250662632,5454127198.951479
+294598722,-9377326244.444
+193709887,-19889.83
+1039864870,0.7
+1242586043,-749975924224.63
+-1250662632,-544.554649
+-1740848088,-9.157
+-369457052,7.7
+-369457052,560.11907883090455
+90660785,-4564.517185
+466567142,-58810.60586
+466567142,196.5785295398584
+1738753776,1525.280459649262
+1816559437,-1035.7009
+-1490239076,92253.232096
+1039864870,94.04
+560745412,678.25
+-466171792,4227.5344
+1561921421,53050.55
+-99948814,-96386.438
+1519948464,152
+1719049112,-7888197
+-793950320,-16
+-466171792,69.9
+1738753776,-99817635066320.2416
+1091836730,0.02
+891262439,-0.04
+452719211,3020.2938930744636
+-2048404259,3939387044.1
+698032489,-330457.4292625839
+-1197550983,-0.5588796922
+-2123273881,-55.89198
+-2048404259,-0.3222960446251
+1585021913,-5762331.06697112
+1785750809,47443.115
+1909136587,181.07681535944
+1801735854,-1760956929364.267
+\N,4.26165227
+1801735854,-438541294.7
+150678276,-8278
+1479580778,92077343080.7
+1091836730,-5017.14
+193709887,-0.5663
+-1681455031,-11105.372477
+-1250662632,93104
+-1197550983,0.1
+\N,682070836.2649603
+-1197550983,71852.8338674412613
+1561921421,-5.405
+-1740848088,0.506394259
+150678276,15989394.8436
+-793950320,-0.1
+-1740848088,901.441
+-477147437,6
+-1264372462,0.883
+-2123273881,3.959
+-1264372462,-6993985240226
+-1264372462,-899
+-243940373,-97176129669.654953
+-243940373,-583.258

data/files/fullouter_long_small_1d.txt

@@ -0,0 +1,39 @@
+533298451
+1164387380
+1614287784
+1635405412
+-1912571616
+-894799664
+-1210744742
+-1014271154
+-747044796
+-1003639073
+436878811
+-1323620496
+-1379355738
+-1712018127
+246169862
+1431997749
+670834064
+1780951928
+-707688773
+1997943409
+1372592319
+-932176731
+162858059
+-683339273
+-497171161
+699863556
+1685473722
+41376947
+-1036083124
+1825107160
+-2038654700
+2119085509
+260588085
+-1792852276
+1831520491
+103640700
+\N
+699007128
+1840266070

data/files/fullouter_multikey_big_1a.txt

@@ -0,0 +1,13 @@
+22767,-1969080993
+-17582,-1730236061
+3556,\N
+-17582,1082230084
+-17582,827141667
+1499,371855128
+-17582,9637312
+\N,1082230084
+-6131,-1969080993
+3556,-1969080993
+\N,\N
+-18222,-1969080993
+-17582,267529350