From 95ce46a68dfb57da93746ad41f440bfc7f301995 Mon Sep 17 00:00:00 2001 From: Rajasekhar-Vuppala Date: Tue, 26 Mar 2024 05:26:06 -0400 Subject: [PATCH 1/8] added recipe file --- reciep.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 reciep.yml diff --git a/reciep.yml b/reciep.yml new file mode 100644 index 00000000000000..605380cc9a8948 --- /dev/null +++ b/reciep.yml @@ -0,0 +1,26 @@ +source: + type: vertica + config: + host_port: '10.20.73.210:5433' + database: Vertica_db + username: "dbadmin" + password: "vdb" + schema_pattern: + deny: + + - '^staging.*' + include_tables: true + include_views: true + include_projections: false + include_models: false + include_view_lineage: false + include_projection_lineage: false + profiling: + enabled: false + stateful_ingestion: + enabled: false +sink: + type: datahub-rest + config: + server: 'http://localhost:8080' + From 9e169b9c0fffa38af826baf4bca3e0a5c9904ab6 Mon Sep 17 00:00:00 2001 From: Rajasekhar-Vuppala <122261609+Rajasekhar-Vuppala@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:00:09 +0530 Subject: [PATCH 2/8] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 15289f663f7a85..d59aa60602fe64 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ export const Logo = (props) => { +

From add39df5d1808f08329d6b9516c340ea0667ee0d Mon Sep 17 00:00:00 2001 From: Rajasekhar-Vuppala <122261609+Rajasekhar-Vuppala@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:56:26 +0530 Subject: [PATCH 3/8] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d59aa60602fe64..15289f663f7a85 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ export const Logo = (props) => { -

From 98c938a8d876cf7335561611f36539b7fafce35b Mon Sep 17 00:00:00 2001 From: rajasekhar-vuppala Date: Mon, 3 Jun 2024 14:13:34 +0530 Subject: [PATCH 4/8] changes related to three part and some fixes --- .../src/images/verticalogo copy.png | Bin 25763 -> 0 bytes metadata-ingestion/setup.py | 2 +- .../datahub/ingestion/source/sql/vertica.py | 18 ++++++++++++------ .../integration/vertica/docker-compose.yml | 1 - .../integration/vertica/vertica_to_file.yml | 3 ++- 5 files changed, 15 insertions(+), 9 deletions(-) delete mode 100644 datahub-web-react/src/images/verticalogo copy.png diff --git a/datahub-web-react/src/images/verticalogo copy.png b/datahub-web-react/src/images/verticalogo copy.png deleted file mode 100644 index 5da38f4e67c7d4ea72c02983bc8cf01c9b10752f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25763 zcmeEug;$%+)^7+Y1Sdd?1$QV~io3fMDOR*tvEWcNxJz-jwou%qKylZW7KbKyacgn7 z>3i<|zH{!n|H1v%nyfsTJkQKz&+NT_+Y+s%p@@e~feiov@RXJ0UH|~-Pyhf83dTV7 zsAtyGq2AEkUnt4|swbhpP#@&14U}!v)dB3NYcK#U(jEZ%dkgACfqJ1HFGK^Nqu$Z} z_7wvE&$sB%LiGQ0jfR>50Fbx6wgv#C0LpUGx;|(Jd5>D5KduHt!=+$zj3Doj$1J*q zoF6z(hv-_Q*alXZd|;(Yu1+|dS|+lEt;A3wlNi6_hnwD&dhf&BtY@~9!lDf~fm3HI zoqiiPU(RwC7K8Pd<>kc67<$+`0)Zli1u#{J`K9=?P-`+hzzZvjPP0<`7^Y)F<5c%--9ER z%>BPpg7aE#Jl#4z+6TQ)3=LeaqG7rZqB$xEl|a8EVaL)m4{B99T=BuPKa=&%Wz5H_ z?c)WCl-jm4ySLDSXB)=@&lOOSM)$;2ym6nZgtQ-5clx{(v8?00_ z75_WHH88cr+W!ul!YCm!)KEq=`7WIpq5gG7Si3Uh*2!%&b>yCu-As2-D(Cwv!UF@~ z=%l>t_TU4tYyg(xf*JfYQ>S1zFo3LvMusU6j2n{0Qk3#OQ}l^_`}_ACBk1rZ^H3IF z+&kPrn+w7QwZ?QQ$<`nU#d>aX;#%&5o$p(da;u#?sEuu?Aa+o=GYIbon*pHNEPVIu z>ng5(H}7N$7YW#x8m3iGY|X|0R&|GITCj+Ye0$-Z;{GUv8uSWF1fsj9fl$$X&b$3Q zw*mT23Z&3ejrHZ|JQlr4+GZUx&$4#4c5}N2^4o>=x92b9YC@Hl2t*>c6L)Cm*DGVx zaoKe=HFjY>_tUR4*RuohyYn#v$c;9qaBcN#;D?|*O!hjKV+WE6(PoB&iBGS!fvUy5 zEz=#L2!?{dbS{d*(U%e3YEfZ(ngO{u^^k}X5vWCLy11F|!8$^N^!prmGp=@E^NF>U z&fOb6-AHs6Z*#*Zf7-s^ys~>GiJ5yh4;;N?AbN(z734?##ywY9mh(i({MOl+0P)*D z(W^nt^(T04Sb0|&p%qpYYb2hSSM1D*W_qI^mXE%fvD&P~rTLV>rRY>^C(k;rtQzb= zhBhg1NDzQOn)AD=a7Jp6eV;!MYx5iTe-7(F*3gKGzt#>z88+LJ8SO@X+jq?-*qmXOXoFe(TkVLV&>1;o^I8 z{M_%<)?mk181YTTeHXurqr~yw>|^JVuwPjyb6h}3Ey{Xa6KyrDB0`zXLaBd!^~(R$ zryV?F{O+1ih@wcK;MwnJljInw=U6c1@4u$~3S%?CC+^>0VUFVM5kUyy`JnW^C5Ivc z>A8sX14N(PW;nx@qV$lAcZ8c(ampr${|hVnq|6x_6%hgGonWGv@XurOxxs!YYd@am zySIdwRK2%`;xc-v^jf08%r@!;dPW zntX#*lt;U7?&{U27g4-6Se>SKqzD|r$8_fkb6ez%P(x$E9D*^@d;lM>@oTc-l%wR1 zxoWT`-o|qu^(E|Rx4HFzad=6XU*}_ZIwX@MpCDWUV{<>^y&&8$$%E)znziQn0oO9C z#LE!{YWQ`BSz0P>(kv1&Og48j@QxFq73>sENSFGD{x^T2@A0?PnNLJ7u^l}Zdocz52Po|00=hcgEyn9VqrXyn&rt+Qli9} zc=0!a=PIl&JWPK;J`V$ETVJQWav>t*2kf7TLOoDxX~_;BJVSdMq5tS<7d;}mo1*qNkB4ERfOWYb zd?d+3IQ3@e^pMzZ&e>^{w)LV;d-;1zR&vqar)Uj@&lAxe%)63l7Jar)f>LBPz=t?I*Bg*t0Tnv2pZtkTrZ^r&y@kvNki zuFz4S6)bu?`U;Tzf*nU%XtXr$m5|LLbef?YELkr^Bw}Xs=`w5I4hCRuwnO(tTay=o^83-AmjM5a zk-WlA6J)oN^2C84;=VmRE-&gut+I-_78^KUD@9mPv~T!mw4w~$&)l8BZ!4f5b6{{^ z2_6r5!BB)_l$M~gBiEY6m{4JA+N=;mT|a}PdK6YnmGICjH;y-OOO>RbC)a7;@rF|qO-HId z@TByUGyP?hJMghl2iiGsY}!n&t)kGV+7e4`pco>Ak-ytmk7Vn=Av@1<-Yh9C2;K5z zZzT^@+0oTHj5erL-tKvj#|+hG`V1vsv2uwt0S1~JZS1g(^{w;l_UkvHhJx& zyOwH;dj- zN&EE$CPtjP=WQE)YR@EFSVgjW35)LYPM7BSqwjJrhR!?gKwhKD%LUo2^v~ObfC^gT z+HkC8Omz~2@5GCWtO}T#%)+B1>l7NqlDSUoxEg6w_dBa4 zh()T(mGTEND(JX5_V|jDyvtC7`Ueu$J6rLB^o74^OG>l9mk)loQ*ZY3g z-HApyQRS55E8dm}Kui+g5x1v)&jK5A#mf7TVO1|baJ>%`D?rUujSH=aMi-1|*&|}- z=hSHhZkq43Xc5!sTnA~DVl$k_{UovaG2<(~WfjMik2Q|vx7W{Zt{+!=vlt|adswc)domyw~G4mjkcBLX$< z%Yxs2)uRDmX7FeozD{;O!d7j66L@RNL0PXK{vQ<-3lj-gQJEi7ev&y71t=Ub*Tnjm zP=|0Qlq;31OHn1vLTTHH|b?YBrDz!zU7EkqtKb82A&&$U{UJa{L2NfIzcnJN$gIXE+3171P zQt$pa(-Uc|i?lE7QiR$MH7D5X8RpO4#6_hN6>mS#S^ngHaqHm@ZVZwO@^pWW29#j7 zoX+T*D~r0{Pyy{KAy&Jk=PZ=<%t3r+5MT;jLLU8)8};o0^@8oIW!FM`s~tgdw4>;b zi)W`+uU+-`TgejWuMF@5McxLB$wh_^9jdVwfFZIr@Dy=Nq4H0OoREukSPIF+~t&!Uh5Iim@dkm9>WdK-wE>7jOtb0%!#5b;)3=LDr-9aT0+8CODz)O zA0M2Uup&iW-q-W{&249-fs9$hpqoB}By>}Z$FhT^o3!AD6Q$vYnF&Alef^}lG4io` zc^-7#V=?ZBw))gvnC?iLaRD^{N$GF~ky6$boQ#1vMF_R# zaMkp6+c#7@woC^eFMj8JH8|w78z#SuBnYr?I?kA2Ngx5WpqY@2kgnWfi7*y-jkaPh zAk7&TWG9SRc8Ec9j9kbW^9iR3_+?A%d;~gTFZ5r#phkO6L@bey0`pajJdoU>Ma3S- zm3mQ4Q+?0#^9k$e{r8<{){#y$b`{p@pyP@t>3ssT+#g7VE3)1j=gFt>q7O?9U@X zOAnN9BSHbf>O(24=}3+iCvxc{8Tf+Nae3(u@$zy z$sn}-&Z#>%zGd@mX{6hI+F286S|dDv{+k^Eds^hvV8e(YA@j6N}< zzGQz83!!_-rJ|ks;(f7V@}xU^x+9jNtflZCqom0$iB(LT#AWF7884#c!)CX;^@?eO z+=p-QxTQ}cMlXJ3m$Z3@P1#klpr_H5G3ijek!pec8v3sc@|GoOHi# z=vtg8fe@Us&` z-$0+uE$+YR*W>Zms7`WyGJzaxniD@T`x*MIJ~ zT9VRBZh5Bq2`>hs`|jgomQHgI3r~NkX9+8^@+Yr==k-AtGIR%$>R@h`xQeHAKqt9i z247wqQ>zKamaj$HI%1GafYcbP%_&g##xNh zU&z?)#{OsU1@j{vRsuWRyYTJH3cG}ye&3V$o)n4unixV79SdMavBI#sjU6&-u^m`I zP5REe@6X!vo-%Q0AinNP#Mv7;I8O#P6ldy$w1`-m={HL^i-|zBagkY@Zu!n?^E$)e z(4=}L=6$!Gf3$dC&yz3M;(V4m!ec$!8D3ASgW{Tg?KodvD~k5%F3HgX;1;!QI)HZN zlmc4h*Vo0Xu~Lk{3b|u<5`X{4J&7h^OtnwOANVR`sb>kg+XemJZ**DYwVcNKvi01Y zq!iU1ZgzlPVKUr7HXNp>#TO$2B;HoOvZms%{5nZivy5Q0Cje$6quik{gL7P`$qF>@ zxintMJ%a`&=#q|yiM7Ox!%S#|o>V7f;AP1G9Tod3Yh;mQa9g>!mM3!&Md`jC!jxFpGbM%`4=v4`P@_32!|Kk5@Frl zr}9i-+32BgpIy5-^G2KX5^|m!2CeA{tkAdaZt5q7PmdjDOX5JovKDlf>jaayk!**f znfAXUFT9dk#fahqkBQx{w0CUk(7Vn0G zx9u@g;F!+Jw?4JQvoqVHIDF(Q3j>~0dlRRXwGkH2(_1o{xP0<#fWTX#E2Qm)fKK1i zw`^clA~ZEZIH{7YZ9tF5c5x15%kQyk>~r-{ey$4eIL-XY1pHvk-vDTGNPb2-pxF0w zggWOO;&_1ZyH0M5x$=Y7PU7@4N9>9?aX5o^qc6qNjR)_gNWF!3f^r+b!TwEenJeot zi?`sfC{Dz%ud>n)BmxA=k!>jH7Km^35;1iGj^(*2Q5*glvStNiX^E1RFIPpt!k(NN zb-th_DWPv(r@G7*&=M`qsTz&nu&@*E8BDT9yuKq#0A5`_->;?^_R2QA$1eL=H#5wt z4l0)p7Ca<<$v`6iwFtB&-7d@Z-~;$0d8pNVFzuL6BdX{)EPp(ZAPru4qHx6!z#Eo_ z=k25}T-Aa!Lax48ytL`~_dx3Tf|Wgl-6pC@6{^6CoM*)$AcZ7 zM1}86Ml3kAtThr5K-bx6S?NO7Q2P&^vTwR38364oR$AN{a#i{@2ghSj)=^Gnvh4_Z|B`N8`@N zB|MwZup5tGBklWnkjyr;kEm_vxf^OqWA_qL2(_u)#@7Ux@s{40Xr`8E)zvs`hWyT# z^}R>Lp>R=QHmJ4ZwbhRI4_r-cPg?j0u-S0z6R;Q0E?YIna;3!|*Q_1GF?Qs9eHd|i zW>8NKb@kQb{18KhAF^Tks5&{8uA&!<=DD$+Avyd`!V=crOK;P6N^v=nrc& zzn1`AGkmvVh!~4sV^KKI@EGW!aefrVq}uy6y)O6aT;~Tq=8_K)|EvP9#G5~w9!9+6 z5k3)eF~5v2ktHGjteLTUNcXJ86@}NCfONKk^BSH)+j-I=!s@3&A zyp58MCf1+R7OZ?ufpu@v%G@H?aS9j3>8p{)&#mQxe5-|2x=V@r-?<}pk_{#|6jAsy z8(=GN`C+uvga}ebfrc8_OU{UtASjV(t9z5_Sg_u6;s7aZGi3Y5N1{0Lf}-P-TV&`Z zSL7;r^t#7!U%szi*wkEs@C%-TUd^;c>Q4)948Sy&?&Nm$!ctv{7mwyYfxaVmzBBx6 zUopx0*5d7!L^A1rK+#{H^L0o&{veSlr|0W86cvtjexs-`yvnC|X*ui#;}eP~fzsqA zEHjnHP3`wt?Bf++SgWFr1wY)iw)V++@wfEXQlMT4IDn>S^UJ$k;!DY}%h)K3)04Xz z#dF*BH$vD0i&7HQ{y*NJ`|jX`PoRx*oB^;+l3ot$Nce3jzOC0A#4Tuox)GbYJJul7 z$MH-jzWnY&xeZ0&ElIbkJ6$skY3#r>x?@*_lFU>g2)~PAd}x*Rp0O-nOS*npirRhF z8tZbYJOuhewp=zc6n=z$_q6HJJ@H(`$m83brsiR`db~>24DR6%#swLI3^=Hxmk++h z@p=+6cn{&wM2?}%DwI*#hwr1}exbPN$o8w~o#kN$>mtUaQX;$2CplkyB>o@zVkX^+ z?yyg1e2Zn(1F>r&)`KCFWYf)R84R^t-;|`zp%5mQECr4I zD@7#UWDW3<1jtJ;#FT9K%Ri)>#=*@99O!NzvuE!+k1X3kS~-oM##3Y-Mk5`xN3G`!BM!CZjfoJF=m_xqye3wzR1&!FP4^S6C_I41FbznlFWM>Jh@<5c!C}40tc_K+YGf?_>?vO0(GLCIY*`NT`z(Sn(-__GXWNg zspM&&z?XocYu?+Mz?-D2RS>i6i}6mITsF}!-egWGr@ThJezOLApajcRcZ#JSTvQ1x z)4k8)s?Kw?y&5jQf>~x8H^?qyu5N9+-lM!f<;B>p>})AZGzN)Y9{CBP!-G%8=9|ZEPj3{f>ae{>ocT)KlnkhntHghDkoCHnGx> zB|R>rBxul$S(1Kw3zRQ&Gl@MvsAq9Qigqy82=1pPd~5il(XQW!mx7+=C$SKDkX6<2 zCCQgXl*Ig{yEdAO5)s0ptsQKcWWNr+BWY11Q&6Hq{hwkSup=CvrN%cou8Cgmz9(>o z_Z1e?Bh<`=-nfwnrb5yBmN!^XAyuN*=Q4;~r|;X0VmrT)z1B>T4zDqb)+(yg9E@Cq z))Ly_xm3egMg!$5B2Y5`V+Le^$obDkM7dZN^pT}At~kv-^)Qag73vU$gO6kBHl8vG zUbsp|q9Us*!I~a83v4VI@`bz^%rmy7SVx8l(erlBB$8akUFEIUWbl3{bW2J&!>vhf zu?l4fWXQJpaz>2fviKPYtwoK;J=ydpK@J3((L}ud8yzP_s}9{fozUuqW8D|mqh?SF z4h?)M5B4trONcByy6B%If^>Tw|2U)^P#5Bs9#qn?3HF#|oFMFWt{(B)ht+-+m2RtK zji~h&@rx*d4_k@Vg$c^l{EPM$AR+nz94}Vn?+{IUl-2hNE0>1u_yf|a2^=W$bpUAR zfr8Kv!Cg6`5D%Rk--*N~xWCG!Nco6&W6)AlU(co-sa&xT)-5z0=>R&tth^&T70HEC zp*&$xQkcX-R8Bz><~PD`$4XFG||9{kN6A zl=@r%4sT{t}l_ZTI3bDMp8uo^m|=mL&f5W%{pg0k={e zuxX))<070=nG$DOmu>tZhhTDt#VAbfwyacpUO{PbP*9zXCqjiDNTnV;1~|k~3f9?2 z2mWO(urlqeH{^oDjlB517@W;~E%nr{A?`W|4SEVKm(+mjscV|$tjf)$*?HoBYvl01l2@R02 zh`15Kf1qO<{Tb9dR>qpY@mK3g2N$$eP`Us++&*m;qwSBrN3j;n3zy}%4#Q(l>C57e ziEO+1+Nk(;+%L`vJ2e{B^N<^n7v0)|NiBXTSALvMJAu52PbkvXk5y25zzF|@$$vpC zn{~4<0_se=jh%=1hDDWdB$YfCZ;2)7cgoC|4imY$`j=f$h=~>`oV&y@aFnBG zaY!1dKAH`dqdUgx=>P1xHX70;%aZu5{*P;~pVXwBI!&Fj3H!AQDwg3f@Ij+aASp#N z6>lSN8c92@t#wapj*BGu@tPOfGt{nCb(R1!c9!dwG7owmg-2{py}VmVPR=crHk(j>a) z4#*oW*mU4U^Cafy2OcrL)Dx2ST)iwdrWNBCES^H9faen7MXxsO`5Cu9k9i?KdPPZr zfw|#6EMZ9JoZOL(dPp?*dptmNoz51JBIZNf>B6Udux6j?F;hcpJRo1p|_U9tI3%6?C%=^g}mnxArAIt~;8=qbCCu!gC0<>zK(VcXltM z$aUFc#M`~kIEq-+RelU4UEoG?cvqpg6|mW!WHfFDlkMB*(wF{Oc;*91tual6xqk4Ye?Xf*FoS>YerZRN~d&cQOk z39Q5Ne-uddRwbum) z8`Vw%N#2&Iz$fEo(2Q~f@>Q#x_=cy0C+!9OZFZ?jtWv`v!Qf7q!ofjGrLw81bn!V z)eO;CSTh*?Y|9e!4BNkOT+5Eb8aMahQyN-h+tH#k?a?_t=F}K*(QUAjsT2ZZ`HTI5K#s5I=%xvRyFN@FHYp)(N@wmG9KC4G1WlsZv42ZLp;6Pzn zQCF!OK2Bf}__&C3d3rnzqbi*o0tjMF3QZK1VF}kaoc}sca!qpXzauuyKI{x$>tP3d z+v5GJ7^Z%WT4$B_ouK!Oowz9)W}@x7PYve(BlDxApg`y^oHj>y+I{y%RTPgi9*cx* z#Xqw>t)Ok7k6JI}mT7AX05qzg@=F$ouK82_`nMwV<+-G2O?c^}Wf4-XzMKgaN1eGL zUBu}prT9aM=U#(ddTl$;i`V#PIpPK1K1R8qnBY2)-}AxAXVIb^5EPc6!ZJxLBq%pP z+;-v*p3|QT?9ddZ$Qsk#AyXN~U&{Op$;Z#bF=9gUE?26cc}hL~>oW&`6!iei*{N?t zp@~xIO$!nZz5Z%+xD`dZkyLrV|0Mm5ArWG$A!*2^%o>`f6(@*GXE=Al+lERJ;`3w4 zHL$aKX`W#6giLlaEG4NRxxQIa*8iofuuvPBhnO8Ja(wi&mc+p?9lM2j+1c%PE?zN= zRdg_%Yq?Zwx*y7HRq{< zj$A7%-dKQZ-CH<3PI$I%95LHjWXsvZ8K{l32S{t5y6bT<2ap zR@1hhe;anxhnjP&m(aQ((u27cs%{IOJ@q7P&QObKY{Ne&*NHQEal`6S43>6JKO$qr zoW_n7fr=-)iZ5$VqrYWV3>r$=&fPJs*goC{N3*jjAcRR?MwXB@On(qoX%s!*l1jMl zIi%C7j?DN=e13uCpHYaHmD1=R<>R0vB?4gu?$sJT1lO)9rxCg6SvuZ({3|s&i02z> zSmT()&F;^4M0>=BYw^$|gb+SE&H~}$sfIKHlfkQ7J^DjAi0`zb)E%%!l_33;62iC2 zdKH&*E8K#Q7r}}vvdx}l>V#DKTc3l+efyg2;(b~jgG9-j69`=kBt0{o+H1{VAjHqAVnbUUzbCr(S6_FzsO^$M55lxA)p~&k zETD~hMITWQ@INs4ULXSf)qm;Jf;rO=@-gtyo{1M7%86p92}~?9?D!J)PS3(%j8OJ# z+KK_F)x`GEj%*!tH1RQ-7e){|mA#%(xFrz+9m6?(=NNFgcF*9T9mzoTxb%aU#uRq; zTlPx#v#h^BD*$Uh@?$*Cyx64Mch6ToQcw9_Jp;PXtrksZ3Ti)>C3(J_fpGG*D+{pG zrkwdW*$U4^pW_W1Yfa!>Q=-CuR;4}sc{%q$d^^BEY`9&I93?5sDBv#%r=!tADS%6q zj~cmW=m}#1f~F7H*NLbEQI@oBbfs4qqfIyOc8Tm9X%inK-@?vyjBR&euT$3E?r}H7 z`izMareD6s`70A|h=ri)xL3a0#1+iWL2+*^FbNUfJOy!mj{)y4LDgYnX+9Ui!Xk5{ftdD;4-sj%P&qiOh6-%-oJG7*ntp|KO~%SeWY^9?1W=+%Hq z8saR)s;ZsiaT$ATFxZCyhW*s_l?UqrW<)9WU$s#92y|Iu#UX9x3_sFvyTYSrr}AK7 z74>`f7IG=t8-LX}bBA=96_eYAZ9sNsA~mfM&bL}_l(=S(M_cz%S+u(802=NQ4QbXa ziiivRe@2uC9Y*}5n+|Fj4r*`Tbc0G9B&b?5{$}dNx2l9YR^V;y!sheTYzya8yyS7x z)`q3WKh?+3`7JXW=p`bR)l(RSZyc7>m|U|O!|HWla91(@PT#!9)w zqmooRh!Gg4>mA0=DNRTXvm}kG@FU4h-SOw*A}+Pl+$}DWZ4|G~P)DW&>>oLMiXc1BZ;SJCiF*cPZQnA5~B=nfqQGe7tCu_0IwebmvNT@csy=3!kQ|3(uC(I(ws%?x43B2)hu`|U)C zdiKE5)4?mVgGHsOgSU`B`jMyTg>Q?c_=*&d-6?|3BW{F8M>%+4 zYK9Eu+$jAf0BUVK?G3I%GEWy^{#_m_^XJz270$K84vrqUEQHrTEb~t&7r3)IG%j(Uj4A!9OMPQ-8U*|95Ufg-uy*-V zA&pOR-jincY=j?Gvw;!`DBQCl^hvnq?7=^EMI=gjLyFM)b~}b@BV>U+aoUEgqrCJPSB*^9?8)8 zH$FoJ?;||9K`YrB-s=Cy$Dtb!$at6gXft9Kuc!o7Q*l(B+%d9YRo4;{aht52nW38t zpegIVnKI~|GA0Pxg|!xK*GH~UmAjoy20?c}#h{L*Eupz>XkLOAA7MmW{!4mbo&tiB z|B`5je%ef$JG^=Q+m_$gYDwvZuLvSo^=MKQ0P7b0t-BnF3G;2&-_z8ZvR$%q$f}i> zM0;AE|5|#9!;AH5TpCYrO=TSirsxgUD&zElW?M#r->re=|fNItN-S8=P8*(DJGJO zwD|CF8)oPra1EAY2;lgFMIVkIa){nZGrEvej&t$;*j|c5Rybx{k5p5di9SglVgLH0 zggkr2i)b+kuPaif-HJEF@Uvq{MTPqFuHk~>Pa5sFep(vs4VkRmH@R$mwyI;qajMv( zmm?LT-}&$#@2LleO8e_NQ(MjXb1@;`#p+|b?3=8&d=m8Ipd|h3je-jd^vxgGt3S%o zI!FnNj7=e1v~t|JUT3vH`W|cDQdi<0wABz2F&xfIlcJS*F}>((-!!idh(#fJOE9=!;!PKacMYy zI2YFDOn#ZW^o@+{upiIp&%0s?P$%(dj=*ZavRvU)_#nnNSHHRQ4QvpDr*9X5eKw(CFp#B2Y05v zjTI5YwTGIMKxk@1tp8@)$Pa=V?>$PN&9XBctyh9*(nPu4_Q&0RfYpN2+YN`=2SZ=q z-(&ZIR5B?^09!%RaJO^1PZjj){}2PU^Z(U8fN^CHUfZrOzIdXCDt-C4GDXz&XDpcS zBKovKX=oxE4jMm|M8%? z0fJjt<^P)Te?I;X1^$Nu{~uC7B%|9WKPQlLA5X3B3i>Uu;9WK2zg0N@Q|(;KeEd`; zFrzrp*jP*Y@0VrF@DW1&#orUmPMvQAl(iqz)>f>abX5csJVu$mgz#oYoEuRB!fl zTA`S8gH%HpRia;^MUu|QsHd?DlI9y-y_1f-Qi>yuMlQ{j(`Uh8exWEK1vDioCV z4Ls{GhYIvMgpJwOTjUg!)KIu!fB7SOMSq}{ZQI_Z&b)TlF@3MKG^O(m^pdD{CWB@F z(%N;)Hakx~B~BWHW0p-30ANJ>`(FT(W9J!zb*_%{gM@QdnQ4C&8_HiZ)q}^Ksr-XG zeh6tci_=vr7?}{koM@4C(Xr4@axKc*g&QyuWHHQC-I)VK;EhI1D?**VO4?#4gt~c% zH&+58x1}V*Ki4W%^yQ>C6vavhjGXbLZgFrs$)!(afwDE@N z`}V%(i*z}lsFXlm24`P_dU~9UO?qNe@`H?b7NrjE%4*(F&m)up8VvrPjjC(;E#_|` z-~mV-)2RP4?Z0udWzAt{W@pH@ghcSAv=>S&%2OW6w1lUVBjr6gu7p)wpNzcRXBK_g zH)LKBRgaTZjmO*-e~$X^0l}h&FoACY9j6MXureo>Ow}3x%cdsYC-?OVjmdX+QTNwA zo(IzlgS~DY5{|eNDR)|hQ7f+(f;NSfHX0Z<-UdGRugR_*X~}(lE`Is#IO9+D91;QO zo_6jetZD2q{8F?cLS>4|b{^VO-O-(nIHq{OOZDA_@t5uBp-yTuNR2dL0;YSG_7IJQ%tDU{v4V}_DYZ;5zTNH$Pm*hftw=d3#e)7sO#({qK;fjhx-HCDwz+hCPZVuani z=X9}I%$Ko(2r@_KKi`b@7m=5uaVN}+-p1JnouC_eXWDOz8EWhA8odiA0xj=@g#DK) z#RJFXH@_UF**(3=Qxks#Tw4o&4d^R7m|lQL4u^QIq7eUQ)McsDK#$4qKHq-sC$H`^GCE!y>JE@&~l z^kZXwEl!tvy}!B(&?vF=6cSRIP#HT6OL%b&N5v4NHGh1*wUdXcZdKVTaw-4>O0gX1pY!awiWedG!e;@&%zaE)REV0<* z8Lr7hmh9&et4A1{DSY2Xmp-n9U*kx}f?XWD!m|ByU-ldH)bH6`ZrWMTsw+sp99*kl z*ybqvbXTRa-S#YpclSK5+*@e)-QH@cCc#gUw6RqqPm^!At8kxI<+iBQPF}UxK5q(_ z{(1~vpY^RJ+i%pOL>$(^E6-$bM>6YIyJCH|#-lFc-ks#cvPr|c|&b&vP7-!o>B%VTnh{7(ET71ruG>0S$`{Qb+ zkQ`g=XM>kxOJ>3F2oo$mCNTJSWYearqKFD1p&@KTlqthU!n4veVkZ$0R?&5Ld^bJJBgSsEd9z$#2}<40|15 z5%PJ_Wf*`1Uk*){_2WJ??d=zX6^4k0tQfoqBh<*kbv0 zz`Mlz{#R9B9u8I9zONJ!6AEEumvzX_2$_WJky3WXQW#5QY}qpQeIMH(k)6qsK|+|Z zM%l8CX%w<#-{w2&_x|4Ny{^w6GuJsY^PJB)=X0Lte(w98cHt`1H|r^pbK4F2oPa^G z+EHgwAPRU<+>36%8PAjhEvQ#NQfH$Ce{X@nXVYp)g1_FL_?%1jr_p?j+mC*@H*1+h z%4G1o(j)JlBHQArryX=$kwa&}ti+o7{OE1y;1#IjKG$${K~)a? zV&lTw0G!m&mi_MASI`x!)Rc(F?2|yk=`%Aqv)G;pdEZC;(hp{H@y`-{2FeX1xk=3_nLh?vRWDz$QhZzk|^93lA;4&|AE7aNGdps~UA(&^%r)hmYOPP1}3=B-N;Fx=2^ zN_P$VIT^sF7wL&X$xeQtfVbK6T$egn$mq6j1HPG9VEu_ROnEiEt!Zs{b9r?bZTDFw z-kt1s!J1cTVcc9X$ld=+;+R+dt3-CtRhP|UDH?DLQ7vZ56*NQ$znTS(|e$y_C1wL z6V|QN3D0Nt1M~>-?&e=~AyNNhL+9r8cUQFJlaNZH_mye%EXR;@u(am{#Cd;5vf0O} zOuEljG-uWV5skqL{AujlnCb7|R#kL)BuOejh%WZ4lFhe(fEUZkwWAnYl3EU#J6? zlR|gM!PTz;SLTtN`yq<@@g6lRp9-1*j!{?ucUfnWmvx%>*?FkFqcwlETe$ARr znaY{TKO#(8Tu{*LXIqtM6;WXYs%>Sja%tdM*g9s;@uGd!TL3~ph?(3%Vj#D&x73X5sBk;spC_9Xdpef-U}A>F`kVC zCXk7pz$~96kcm`xc81#r#av9dUWUNKw@Qb2wXiE$ocFdKzah*yEd<<1J?I(<${wmf z$A~Z1k~#bcdV#bvD4xu!A!kBXoh0s|TdS1*B@)?Jk}?|mix>yR*?d6Hu6gus?LMKl}ekXLG1N6pUvJx`86yfM069o9x z`tz}TB|Dg2t|-shJL4{I!W)AHJ+^#b6H{M0#TG-8ba|8Oq1#b(aI-#nD3qliFQx9| z7fn0VSivz<@f?y~%~~%dy)`?~>owr%8Wz(&Y3S^Lh<;fVKeU471KiS$Rf>5shOcXE zf0`<4yfjq0x?&y)KgOZWyfx0sQO^raVqI_D4`Q|5Hg6uHt#gN?^qK&sp6Fj{O08)2xI%qn{SjE3)(sL{B|PJn$=SC?0Gj|cx)hWTX( zm%rpLqf3qSd)qrtSMR=|aSS!Lw=g*rPNlrmfSqiwsOtssqq~v)r0dV1794aB+YEl_ zzKPjaXafZWe)X7Hkz+WwLF3`vBe$OU_Nxac+M1Y^o3DIumG<}owkhy3Semr$YO?ze zB#6GcF|*LievbX2vT`~5UO{SEXNN@VAMpzJ^dXYM=V0Di0{atVOEZ$4 zG@ULoDWiXJE7L1EN^e~eXLMH=SEdq@6ccmZQw-c{*J{WKiTKlGr>i1gBOOiz;$rS>uvjPhN#$+CG4iww?5wBo(24|DuI z*7hq!zr`lupu>|bs$ZCAmwOH44aRYY`jRHbb$?MXHLdd+?F?JCw`L??n;Ycr5FVgA zJ9cvuN3=lYQtxYx;u2(M3N{5d4<%=hzSrbP&f#6#Z>+D*Wjgq-if8J4@KFwQ6NJ97 zLPErH{ma|J?yq6a zzodun0-ZwvGPL(ompn_P8Z2ORDLp;j)#_Ojf#YlI)7IUq-I%sO)7plXaB1g|`w&B) z_49=KXtxAiOT?Lw>g-D6kwg&{bmd8HCo+iK_rGVn@19&xkn{$ehGV9)DYcGaPxN<$ zu@H6vtT1H-?!iJ6gH-Z*6XWwJo^mDry| z8PV|Wlm4zfKzxys@y5$i*23tuez}bDJnEgD+a2pDFI0?La~GN3cNWO7?fEv)Mw~7_Ar{`?rgv*RkmOT_bqrzmg(RlcGwpS zxlPcrXz(>~eR1(5|3Th@7Pv~zqd4u>Z6J50i`SDa^XmxR61`dT-mO=lotBZhis-Ll z6x9-8K=syh-VI)*<5-cS&Vx^Rl=HHO0n5(}NB5E)Go=1ya@M@_{@Q7M2{^5mK2xdN z<oQEgX|l_IZ0St~rHQRHP5qF-9$m}N70KWb z7`%W<|5AX~ce!6{sjUJwY>gE~30yD-PlD=s#htAijV-V4Ub^;C!usX7S_9PdZzHOJ z;@li3Kp*-xYUM$;%+o%!VhWAkx&>_BY0;$AIjWZ{DdU^zsOLLP!_y!q-Gwi-tP2Ax z>L6yXUv%niE9<)UCrx(o(_^csvNE?QQAHDuFeR&!F)xcB1SMQDQ)se?`p2Cq*+&|S zjWl+)*OXX)`Iod#?xB3~O2Z>Anj|sa&(kYQhG9sa3F_?-+ zM)RH!o}e<4Op?6(tVD{|b4=XrmOJeUayej(kK$Ng`KL*)N6nq@3sJL%5WkV7i-tCN zg=vPTY>?axx}Weq9FbWlsa@$8=uu%1_@N$${iUyMtU!8+2tS^xmGZVYyHRHI^bKP{{ol{byOT=3H z&0kzDLPs=T>UXpF`+fbv^*niKyA)mgJ)L-!L#EkMXfB5~X<)tH4Wqy}Y~IjIQr#PF zf&fpDH>R6k7qt!KKg{!qHKyy*tXCLX}%pY#XSEVk$Th6eToiDN!! zU;YC2;ftH67k2qRI6ZIy}9kN zL7Q{hKP+BIoqMHZai2Fr4ChG}j}4)qEkhS+ZGr%*2Ly z;p{=9wsqRzk9PgY^hkWp-xSIXTk|}nJ5ZZT&GoS$?)dY(wU4Ghy1k{tA!}*$!D2JD zbIE>vfQNxma@fF;joCfvt#{uRGFAD$p(|*T_8G?_jOzN~&QvVa^=5eQu{pF;%M9&X;r8K<+v^#s2#VvX^EqWm(=Ovtn!eEoackD}$ik*PxFG|11Qr51b}7r*`s_)B0^IhpxQ(YYnbWAkC_j zZ1(UM;~cjJ?;qq4xL^8AA?;wNB*vx6%kM4PMgg4h+ETtU_1Z_TF8yLjD>vjBFtuc- zhD7_g>(*|vZ)o2Ga`C|~q#C@gOXD+=tfuGLQVpL|bP3ZHZcMTJzE?8wAo*X|H0R@- zUyf#6gc%0V?m1-!tP%PcGp=IXHaZC^IWzNO_nrVT!q4CIUnDwvOqSFJnr9P4%tJkI zj(xML1H#(+#`(!*0PN!iogkj=Cx(qgl^r|&am`Zw#lEb>!6jPV%9$pP%s(fP#j&1jYMq zmAsua9U(RLZ$544?C|x4q`gcT1Ag}O?et#NnEoO(WY#QZ2F!}916Ak1h55dBJ^y9x z*e%Vn^8@P+Bo{UHx6jh=EG6LTvBOs!)}O-r$ZJCkJ#4 zzX)&Ne)sx8vpnBQ$#Sy#x2?;6eq>qsI9myoAT4Kw+{0_Wej0fOs?~~_k&BaOY=PW- z{ewpHp*$kJS=iAWXhWpGpPo4}-B51yn_0o^CXVdmW;Dx!9ds$O9C-xNTVSeJZU$bA zo^k`dV)Vs&?!YN=93!_; zEq6wjg!r>pCyTA34mFg18#{mVA2jm@5d! zmW~=OtUD(nGcA0m>L~v*0uZ)U|G5KUfdUR63Y+{prPkUXOFBD{b6>^R-Y~Bqx+;wP zgR;cZOl$3@CQGfBE=Lf`d3(4Md25ka&(__GhC!QE6r2}0DaeF7j@{7$7({O;rWkew z3|}no9En`vQCNzbs~es~AKaZ8YSSlhbLO;nhSYUxB{0wP-JCXV~tw$H;yZ1$x z9RM{7&-|rjRze4j&W)M3Uu z+6#LE9?~pK@byi3Nxdz$+%arX0EDViVqXNY;0~Kv<+kD?KfxaH5rM zWu15OHPuIdH5JE&R6a$q-WhsrNf}6i4dG_>AMn=oj}(0EI=vp@;2%d~vGBJ-yHkNl6$q2n+BXpQNX~)Nxbp))Ubjk zLN@qX0NL9~;k<{8?Vp3}uM6|RcS^IEtz3OC-b#jN0~5NWf&jYG(dg6RI`am(34NU# zTv6+H$WDSLk2vFH7Vn?I2InZx&2GH4FeHU-Cq*cy6NBjDmpv7ASYL=QbD@szGzUZ% zbvpYXM}6^E{Y)43SUe;Js|G3ySECEEvmS1eWG@Go>mb64j&gEEMaQkNQaJ6X zGg|wy@R)v$z5`6QUF}*O3uNehIl1djWozb?T@l^|LshBVfXuWxN3@Kx^sKNNJ~}!M z5Fc~J{UBHg1VFn>$sj@CKajxfdKdNsIeI69`8Y#a;7M^cOoeV4%*2=?eCCXDwzbm zkz)*idh?VCJEUzi&4a6#uBDODQ=nY^5_9&4q8RULb7>OL9mLJrT3Ntm1!1^^ z9F3fh)1x}9rB=CA*07#y7*Xnm|x6&vbIrB9nTedxa2myz`chen|!OBu)gnx z>T;;-)zL~}#oqYQ?%b4PGtegP1+VNBjO@<6nb6Z=kM622i1SmiBp1+pl`?0wm?VSb zj7hAn@X!ehFwN79*PZoOWvZ?8y)VV?pO8yviL4&m3-cpA6*UicZ)G|IT}GQ49ov}J zJX$*@C{(ILraqgYftm%4pb?PqSH;8y|A-o`>EB@_xNZlb7+FORKm9wIh@6Fjp)8by z`k%$d0b;ZHU57h1D$xFKYk*GiMUdx61n>_EdE{6ce?f~=}ARxk)MlbMB0V?W;C zKc)N?Kj8GJD6q}aDoi3iIe}W7kK8bnT}U0;CMMHo0AMVLk(%afGO6yAYbu804UXMz zSbvVwCjGpA^KNfb=ggwcVNO7VEakRCY1GrC?Ltn#A#rnpU5VvMSH^QQkQJdh<9!sz zWZ>y}@0||6o-dI_Igx(QfVt>H{L{l9v60ng7OWEiB*fTYF(c;Vyel~+6 z`fl3vy!hJXHR)mlW?828!F>T7qxzulej5#J!;h%}9%f=+b|IRhd#41+86P1Q|H?OI zBywbDM_^(Cz`hbdAcl+es9D$zmhW9~sZSYVT&ddv(k&h6;JAJ)E0FuusM=8s0KLj2 znun|U6yNjN(8p)=db*zfIiHtdQ#sVCkY<=pSI3`lVng{hq?`PJn8Lg~M;t8@18YIS zH^`QP!BvIi+1}wZ^a*bc5(lwvTAL8HT-h7{B+g!RrM3Y#tN6o(_KKT)ihc2$Lfb6M z0*14x2iIncH!ruBR=lPq-8admUT-Nts-MvzG=*N!Z|4t-qO9)N;V!uM#`D6qBh*KIu}{=n#u-`~IVn8*PfWD+_~suSrxYB z^=ZN-{B$TBfEIQU@?!Y1F1a)*C=nUjGtHte+%vT4V0eS?=>j))2o(c?AD4v{Ead~k z_Zghgw^sYegpexGtl{NwDZo<9yvoVcoRO+&B93dghV=5PF7xX&xxD$m?_fJ0iwPmFvcrHs!s;r4lTZ%T&^BrU z{U^KOwx`MoC^Zi|C5$)9OR%0u?qK$P+LS$nC*parR7Tiy0rd*ba%Pa zDB3ZKBHa>;#fLYV4d~UPTFF}cBxM%4_3>4xHqF@`_MdU*QY=^3Gk^9I%52#pl=Drt zFK6ww|7`vDqWMCJTGV)fa}hCTXZ}Sqzsk7*_bx|J!hd%&j0|>A*z)k0S+hdbUyr^K z_oC8Y{-e=9Wae)jO1?c_D20Sj3}4KZJr|l=ODOohpCeb_ftD{gK_!1<7^c&F-h?av zo9zGnz~ig)f2P3}`&uK$A9yecAyjp9R)CiEDsZl;2S()m-}RexXtl_ZX1rI72*4zY0K3w7~OL> zkGLAF`lFFHwVoxcVDyD37vc!C?Y-LRk51iU0qjR62x2ZMUnd$gBxvKKp|ZZ4LdqRqEEE F{|CI7K7#-N diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 2bfc94d13aa146..d5374e2819bd55 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -425,7 +425,7 @@ "nifi": {"requests", "packaging", "requests-gssapi"}, "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"} | sqlglot_lib, "powerbi-report-server": powerbi_report_server, - "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.1"}, + "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.2"}, "unity-catalog": databricks | sql_common | sqllineage_lib, # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index 7534f1295c5283..a1b98126feef1a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -133,6 +133,7 @@ def create(cls, config_dict: Dict, ctx: PipelineContext) -> "VerticaSource": return cls(config, ctx) def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: + yield from super().get_workunits_internal() sql_config = self.config if logger.isEnabledFor(logging.DEBUG): # If debug logging is enabled, we also want to echo each SQL query issued. @@ -170,11 +171,7 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit ), ) - if sql_config.include_tables: - yield from self.loop_tables(inspector, schema, sql_config) - - if sql_config.include_views: - yield from self.loop_views(inspector, schema, sql_config) + if sql_config.include_projections: yield from self.loop_projections(inspector, schema, sql_config) if sql_config.include_models: @@ -190,6 +187,15 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit profile_requests, profiler, platform=self.platform ) + def get_identifier( + self, *, schema: str, entity: str, inspector: VerticaInspector, **kwargs: Any + ) -> str: + regular = f"{schema}.{entity}" + if self.config.database: + return f"{self.config.database}.{regular}" + current_database = self.get_db_name(inspector) + return f"{current_database}.{regular}" + def get_database_properties( self, inspector: VerticaInspector, database: str ) -> Optional[Dict[str, str]]: @@ -299,7 +305,7 @@ def loop_views( entityUrn=dataset_snapshot.urn, aspect=lineage_info, ).as_workunit() - + except Exception as e: logger.warning( f"Unable to get lineage of view {schema}.{view} due to an exception.\n {traceback.format_exc()}" diff --git a/metadata-ingestion/tests/integration/vertica/docker-compose.yml b/metadata-ingestion/tests/integration/vertica/docker-compose.yml index 1ba7990c826b2e..24cc4ae715d36b 100644 --- a/metadata-ingestion/tests/integration/vertica/docker-compose.yml +++ b/metadata-ingestion/tests/integration/vertica/docker-compose.yml @@ -4,7 +4,6 @@ services: platform: linux/amd64 environment: APP_DB_USER: "dbadmin" - APP_DB_PASSWORD: "abc123" container_name: vertica-ce image: vertica/vertica-ce:23.4.0-0 ports: diff --git a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml index a182e54bd53c71..2b3e63c40071e2 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml +++ b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml @@ -4,13 +4,14 @@ source: host_port: 127.0.0.1:5433 database: Vmart username: dbadmin - password: abc123 + password: include_tables: true include_views: true include_projections: true include_models: true include_view_lineage: true include_projection_lineage: true + include_view_column_lineage: true sink: From a252ee22b74a4483c1c06c18630df3e28ccab149 Mon Sep 17 00:00:00 2001 From: rajasekhar-vuppala Date: Mon, 3 Jun 2024 08:29:29 -0400 Subject: [PATCH 5/8] updating golden files --- .../vertica/vertica_mces_with_db_golden.json | 5556 ++++++++--------- 1 file changed, 2723 insertions(+), 2833 deletions(-) diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json index ef535158165da5..cd1cd0d7e28a48 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -9,11 +9,7 @@ "customProperties": { "platform": "vertica", "env": "PROD", - "database": "vmart", - "cluster_type": "Enterprise", - "cluster_size": "101 GB", - "subcluster": " ", - "communal_storage_path": "" + "database": "vmart" }, "name": "vmart" } @@ -101,10 +97,7 @@ "platform": "vertica", "env": "PROD", "database": "vmart", - "schema": "public", - "projection_count": "12", - "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", - "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | TransformFunctions -- User-defined Python library | " + "schema": "public" }, "name": "public" } @@ -204,7 +197,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -215,6 +208,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -229,7 +223,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -246,7 +240,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -256,7 +250,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.308022+00:00", + "create_time": "2024-06-03 12:24:31.057395+00:00", "table_size": "0 KB" }, "name": "clicks", @@ -266,7 +260,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.clicks", + "schemaName": "Vmart.public.clicks", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -337,7 +331,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -355,7 +349,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -380,7 +374,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -391,6 +385,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -405,7 +400,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -422,7 +417,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -432,7 +427,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.846965+00:00", + "create_time": "2024-06-03 12:23:45.887434+00:00", "table_size": "2119 KB" }, "name": "customer_dimension", @@ -442,7 +437,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.customer_dimension", + "schemaName": "Vmart.public.customer_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -747,7 +742,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -765,7 +760,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -790,7 +785,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -801,6 +796,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -815,7 +811,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -832,7 +828,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -842,8 +838,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.857152+00:00", - "table_size": "138 KB" + "create_time": "2024-06-03 12:23:45.897002+00:00", + "table_size": "145 KB" }, "name": "date_dimension", "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", @@ -852,7 +848,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.date_dimension", + "schemaName": "Vmart.public.date_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1157,7 +1153,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1175,7 +1171,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1200,7 +1196,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1211,6 +1207,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1225,7 +1222,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1242,7 +1239,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1252,7 +1249,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.863745+00:00", + "create_time": "2024-06-03 12:23:45.903227+00:00", "table_size": "327 KB" }, "name": "employee_dimension", @@ -1262,7 +1259,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.employee_dimension", + "schemaName": "Vmart.public.employee_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1528,7 +1525,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1546,7 +1543,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1571,7 +1568,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1582,6 +1579,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1596,7 +1594,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1613,7 +1611,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1623,8 +1621,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.873181+00:00", - "table_size": "2564 KB" + "create_time": "2024-06-03 12:23:45.912348+00:00", + "table_size": "2567 KB" }, "name": "inventory_fact", "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", @@ -1633,7 +1631,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.inventory_fact", + "schemaName": "Vmart.public.inventory_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1743,7 +1741,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1761,7 +1759,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1786,7 +1784,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1797,6 +1795,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1811,7 +1810,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1828,7 +1827,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1838,7 +1837,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.408507+00:00", + "create_time": "2024-06-03 12:24:31.194163+00:00", "table_size": "0 KB" }, "name": "phrases", @@ -1848,7 +1847,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.phrases", + "schemaName": "Vmart.public.phrases", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1893,7 +1892,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1911,7 +1910,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1936,7 +1935,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1947,6 +1946,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1961,7 +1961,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1978,7 +1978,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1988,7 +1988,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.850505+00:00", + "create_time": "2024-06-03 12:23:45.890782+00:00", "table_size": "19 KB" }, "name": "product_dimension", @@ -1998,7 +1998,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.product_dimension", + "schemaName": "Vmart.public.product_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2303,7 +2303,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2321,7 +2321,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2346,7 +2346,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -2357,6 +2357,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2371,7 +2372,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -2388,7 +2389,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -2398,7 +2399,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.853878+00:00", + "create_time": "2024-06-03 12:23:45.893891+00:00", "table_size": "3 KB" }, "name": "promotion_dimension", @@ -2408,7 +2409,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.promotion_dimension", + "schemaName": "Vmart.public.promotion_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2596,7 +2597,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2614,7 +2615,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2639,7 +2640,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -2650,6 +2651,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2664,7 +2666,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -2681,7 +2683,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -2691,7 +2693,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.296044+00:00", + "create_time": "2024-06-03 12:24:31.046829+00:00", "table_size": "0 KB" }, "name": "readings", @@ -2701,7 +2703,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.readings", + "schemaName": "Vmart.public.readings", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2772,7 +2774,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2790,7 +2792,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2815,7 +2817,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -2826,6 +2828,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2840,7 +2843,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -2857,7 +2860,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -2867,7 +2870,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.867119+00:00", + "create_time": "2024-06-03 12:23:45.906471+00:00", "table_size": "1 KB" }, "name": "shipping_dimension", @@ -2877,7 +2880,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.shipping_dimension", + "schemaName": "Vmart.public.shipping_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2961,7 +2964,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2979,7 +2982,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3004,7 +3007,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3015,6 +3018,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3029,7 +3033,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3046,7 +3050,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3056,7 +3060,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.860541+00:00", + "create_time": "2024-06-03 12:23:45.900185+00:00", "table_size": "1 KB" }, "name": "vendor_dimension", @@ -3066,7 +3070,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.vendor_dimension", + "schemaName": "Vmart.public.vendor_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3202,7 +3206,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3220,7 +3224,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3245,7 +3249,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3256,6 +3260,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3270,7 +3275,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3287,7 +3292,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3297,7 +3302,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:04.970568+00:00", + "create_time": "2024-06-03 12:24:30.233405+00:00", "table_size": "0 KB" }, "name": "vmart_load_success", @@ -3307,7 +3312,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.vmart_load_success", + "schemaName": "Vmart.public.vmart_load_success", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3352,7 +3357,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3370,7 +3375,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3395,7 +3400,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3406,6 +3411,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3420,7 +3426,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3437,7 +3443,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3447,7 +3453,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.870169+00:00", + "create_time": "2024-06-03 12:23:45.909432+00:00", "table_size": "2 KB" }, "name": "warehouse_dimension", @@ -3457,7 +3463,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.warehouse_dimension", + "schemaName": "Vmart.public.warehouse_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3567,7 +3573,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3585,7 +3591,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3610,7 +3616,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3621,6 +3627,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3635,7 +3642,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3652,7 +3659,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3662,7 +3669,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.319029+00:00", + "create_time": "2024-06-03 12:24:31.075640+00:00", "table_size": "0 KB", "view_definition": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state", "is_view": "True" @@ -3674,7 +3681,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.sampleview", + "schemaName": "Vmart.public.sampleview", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3693,7 +3700,7 @@ }, "fields": [ { - "fieldPath": "SUM", + "fieldPath": "sum", "nullable": true, "description": "", "type": { @@ -3732,7 +3739,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3750,7 +3757,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { @@ -3768,27 +3775,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", - "type": "TRANSFORMED" + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", - "type": "TRANSFORMED" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -3800,8 +3799,96 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "store" + }, + "name": "store" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3810,10 +3897,6 @@ { "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - }, - { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -3826,7 +3909,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3837,6 +3920,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3851,12 +3935,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -3868,7 +3952,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3878,23 +3962,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(date_dimension.date_key)", - "Projection_size": "138 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.918904+00:00", + "table_size": "2 KB" }, - "name": "date_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "store_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.date_dimension_super", + "schemaName": "Vmart.store.store_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3913,7 +3991,7 @@ }, "fields": [ { - "fieldPath": "date_key", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { @@ -3923,36 +4001,10 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "full_date_description", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=18)", - "recursive": false, - "isPartOfKey": false + "isPartOfKey": true }, { - "fieldPath": "day_of_week", + "fieldPath": "store_name", "nullable": true, "description": "", "type": { @@ -3960,12 +4012,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "day_number_in_calendar_month", + "fieldPath": "store_number", "nullable": true, "description": "", "type": { @@ -3978,85 +4030,85 @@ "isPartOfKey": false }, { - "fieldPath": "day_number_in_calendar_year", + "fieldPath": "store_address", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=256)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "day_number_in_fiscal_month", + "fieldPath": "store_city", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "day_number_in_fiscal_year", + "fieldPath": "store_state", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=2)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_day_in_week_indicator", + "fieldPath": "store_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_day_in_month_indicator", + "fieldPath": "floor_plan_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_week_number_in_year", + "fieldPath": "photo_processing_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_month_name", + "fieldPath": "financial_service_type", "nullable": true, "description": "", "type": { @@ -4064,12 +4116,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_month_number_in_year", + "fieldPath": "selling_square_footage", "nullable": true, "description": "", "type": { @@ -4082,20 +4134,7 @@ "isPartOfKey": false }, { - "fieldPath": "calendar_year_month", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=7)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_quarter", + "fieldPath": "total_square_footage", "nullable": true, "description": "", "type": { @@ -4108,33 +4147,33 @@ "isPartOfKey": false }, { - "fieldPath": "calendar_year_quarter", + "fieldPath": "first_open_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_half_year", + "fieldPath": "last_remodel_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_year", + "fieldPath": "number_of_employees", "nullable": true, "description": "", "type": { @@ -4147,41 +4186,41 @@ "isPartOfKey": false }, { - "fieldPath": "holiday_indicator", + "fieldPath": "annual_shrinkage", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=10)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "weekday_indicator", + "fieldPath": "foot_traffic", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "selling_season", + "fieldPath": "monthly_rent_cost", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -4199,13 +4238,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" + "Table" ] } }, @@ -4217,44 +4256,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ + "path": [ { "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -4267,7 +4281,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -4278,6 +4292,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4292,12 +4307,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -4309,7 +4324,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -4319,23 +4334,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)", - "Projection_size": "19 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.929154+00:00", + "table_size": "8646 KB" }, - "name": "product_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "store_orders_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.product_dimension_super", + "schemaName": "Vmart.store.store_orders_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -4380,137 +4389,111 @@ "isPartOfKey": false }, { - "fieldPath": "product_description", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=128)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sku_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "category_description", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "department_description", + "fieldPath": "vendor_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "package_type_description", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "package_size", + "fieldPath": "order_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "fat_content", + "fieldPath": "date_ordered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "diet_type", + "fieldPath": "date_shipped", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "weight", + "fieldPath": "expected_delivery_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "weight_units_of_measure", + "fieldPath": "date_delivered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "shelf_width", + "fieldPath": "quantity_ordered", "nullable": true, "description": "", "type": { @@ -4523,7 +4506,7 @@ "isPartOfKey": false }, { - "fieldPath": "shelf_height", + "fieldPath": "quantity_delivered", "nullable": true, "description": "", "type": { @@ -4536,20 +4519,20 @@ "isPartOfKey": false }, { - "fieldPath": "shelf_depth", + "fieldPath": "shipper_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "product_price", + "fieldPath": "unit_price", "nullable": true, "description": "", "type": { @@ -4562,7 +4545,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_cost", + "fieldPath": "shipping_cost", "nullable": true, "description": "", "type": { @@ -4575,7 +4558,7 @@ "isPartOfKey": false }, { - "fieldPath": "lowest_competitor_price", + "fieldPath": "total_order_cost", "nullable": true, "description": "", "type": { @@ -4588,7 +4571,7 @@ "isPartOfKey": false }, { - "fieldPath": "highest_competitor_price", + "fieldPath": "quantity_in_stock", "nullable": true, "description": "", "type": { @@ -4601,7 +4584,7 @@ "isPartOfKey": false }, { - "fieldPath": "average_competitor_price", + "fieldPath": "reorder_level", "nullable": true, "description": "", "type": { @@ -4614,7 +4597,7 @@ "isPartOfKey": false }, { - "fieldPath": "discontinued_flag", + "fieldPath": "overstock_ceiling", "nullable": true, "description": "", "type": { @@ -4640,38 +4623,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", - "type": "TRANSFORMED" - } + "Table" ] } }, @@ -4683,7 +4641,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -4694,8 +4652,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -4708,7 +4666,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -4719,6 +4677,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4733,12 +4692,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -4750,7 +4709,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -4760,23 +4719,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(promotion_dimension.promotion_key)", - "Projection_size": "3 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.922050+00:00", + "table_size": "225096 KB" }, - "name": "promotion_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "store_sales_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.promotion_dimension_super", + "schemaName": "Vmart.store.store_sales_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -4795,7 +4748,7 @@ }, "fields": [ { - "fieldPath": "promotion_key", + "fieldPath": "date_key", "nullable": true, "description": "", "type": { @@ -4808,111 +4761,111 @@ "isPartOfKey": false }, { - "fieldPath": "promotion_name", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "price_reduction_type", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_media_type", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ad_type", + "fieldPath": "promotion_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "display_type", + "fieldPath": "customer_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "coupon_type", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ad_media_name", + "fieldPath": "pos_transaction_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "display_provider", + "fieldPath": "sales_quantity", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_cost", + "fieldPath": "sales_dollar_amount", "nullable": true, "description": "", "type": { @@ -4925,77 +4878,117 @@ "isPartOfKey": false }, { - "fieldPath": "promotion_begin_date", + "fieldPath": "cost_dollar_amount", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_end_date", + "fieldPath": "gross_profit_dollar_amount", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" + }, + { + "fieldPath": "transaction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_time", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIME()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tender_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_datetime", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -5007,7 +5000,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -5018,8 +5011,117 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "online_sales" + }, + "name": "online_sales" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" } ] } @@ -5032,7 +5134,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -5043,6 +5145,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5057,12 +5160,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } }, "systemMetadata": { @@ -5074,7 +5177,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -5084,23 +5187,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(vendor_dimension.vendor_key)", - "Projection_size": "1 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.938730+00:00", + "table_size": "6 KB" }, - "name": "vendor_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "call_center_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.vendor_dimension_super", + "schemaName": "Vmart.online_sales.call_center_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -5119,7 +5216,7 @@ }, "fields": [ { - "fieldPath": "vendor_key", + "fieldPath": "call_center_key", "nullable": true, "description": "", "type": { @@ -5129,36 +5226,36 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": false + "isPartOfKey": true }, { - "fieldPath": "vendor_name", + "fieldPath": "cc_closed_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_address", + "fieldPath": "cc_open_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_city", + "fieldPath": "cc_name", "nullable": true, "description": "", "type": { @@ -5166,12 +5263,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(length=50)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_state", + "fieldPath": "cc_class", "nullable": true, "description": "", "type": { @@ -5179,70 +5276,104 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "VARCHAR(length=50)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_region", + "fieldPath": "cc_employees", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "deal_size", + "fieldPath": "cc_hours", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=20)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_deal_update", + "fieldPath": "cc_manager", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=40)", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" + }, + { + "fieldPath": "cc_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } ] } }, @@ -5254,20 +5385,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -5279,7 +5403,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -5290,8 +5414,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } ] } @@ -5304,7 +5428,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -5315,6 +5439,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5329,12 +5454,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } }, "systemMetadata": { @@ -5346,7 +5471,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -5356,23 +5481,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(customer_dimension.customer_key)", - "Projection_size": "2119 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.935745+00:00", + "table_size": "9 KB" }, - "name": "customer_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "online_page_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.customer_dimension_super", + "schemaName": "Vmart.online_sales.online_page_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -5391,7 +5510,7 @@ }, "fields": [ { - "fieldPath": "customer_key", + "fieldPath": "online_page_key", "nullable": true, "description": "", "type": { @@ -5401,62 +5520,36 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_name", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false + "isPartOfKey": true }, { - "fieldPath": "customer_gender", + "fieldPath": "start_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "title", + "fieldPath": "end_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "household_id", + "fieldPath": "page_number", "nullable": true, "description": "", "type": { @@ -5469,59 +5562,7 @@ "isPartOfKey": false }, { - "fieldPath": "customer_address", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_city", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_state", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=2)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_region", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "marital_status", + "fieldPath": "page_description", "nullable": true, "description": "", "type": { @@ -5529,51 +5570,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_age", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number_of_children", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "annual_income", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=100)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "occupation", + "fieldPath": "page_type", "nullable": true, "description": "", "type": { @@ -5581,85 +5583,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "largest_bill_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_membership_card", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_since", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "deal_stage", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "deal_size", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_deal_update", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=100)", "recursive": false, "isPartOfKey": false } @@ -5677,38 +5601,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", - "type": "TRANSFORMED" - } + "Table" ] } }, @@ -5720,7 +5619,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -5731,8 +5630,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } ] } @@ -5745,7 +5644,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -5756,6 +5655,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5770,12 +5670,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } }, "systemMetadata": { @@ -5787,7 +5687,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -5797,23 +5697,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(employee_dimension.employee_key)", - "Projection_size": "327 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.941712+00:00", + "table_size": "182385 KB" }, - "name": "employee_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "online_sales_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.employee_dimension_super", + "schemaName": "Vmart.online_sales.online_sales_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -5832,7 +5726,7 @@ }, "fields": [ { - "fieldPath": "employee_key", + "fieldPath": "sale_date_key", "nullable": true, "description": "", "type": { @@ -5845,72 +5739,72 @@ "isPartOfKey": false }, { - "fieldPath": "employee_gender", + "fieldPath": "ship_date_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "courtesy_title", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_first_name", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_middle_initial", + "fieldPath": "customer_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_last_name", + "fieldPath": "call_center_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_age", + "fieldPath": "online_page_key", "nullable": true, "description": "", "type": { @@ -5923,85 +5817,85 @@ "isPartOfKey": false }, { - "fieldPath": "hire_date", + "fieldPath": "shipping_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_street_address", + "fieldPath": "warehouse_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_city", + "fieldPath": "promotion_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_state", + "fieldPath": "pos_transaction_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_region", + "fieldPath": "sales_quantity", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "job_title", + "fieldPath": "sales_dollar_amount", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "reports_to", + "fieldPath": "ship_dollar_amount", "nullable": true, "description": "", "type": { @@ -6009,12 +5903,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "salaried_flag", + "fieldPath": "net_dollar_amount", "nullable": true, "description": "", "type": { @@ -6022,12 +5916,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "annual_salary", + "fieldPath": "cost_dollar_amount", "nullable": true, "description": "", "type": { @@ -6035,12 +5929,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "hourly_rate", + "fieldPath": "gross_profit_dollar_amount", "nullable": true, "description": "", "type": { @@ -6053,15 +5947,41 @@ "isPartOfKey": false }, { - "fieldPath": "vacation_days", + "fieldPath": "transaction_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_saledate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_shipdate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false } @@ -6079,13 +5999,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" + "Table" ] } }, @@ -6097,19 +6017,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", - "type": "TRANSFORMED" + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } ] } @@ -6121,22 +6041,22 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "containerProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - }, - { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" - } - ] + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "cluster_type": "Enterprise", + "cluster_size": "52 GB", + "subcluster": " ", + "communal_storage_path": "" + }, + "name": "vmart" } }, "systemMetadata": { @@ -6146,22 +6066,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "status", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:dbadmin", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "removed": false } }, "systemMetadata": { @@ -6171,13 +6082,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "platform": "urn:li:dataPlatform:vertica" } }, "systemMetadata": { @@ -6187,133 +6098,14 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(warehouse_dimension.warehouse_key)", - "Projection_size": "2 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" - }, - "name": "warehouse_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.warehouse_dimension_super", - "platform": "urn:li:dataPlatform:vertica", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "warehouse_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_name", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=20)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_address", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_city", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=60)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_state", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=2)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_region", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - } - ] - } - } + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" ] } }, @@ -6324,15 +6116,22 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Projections" - ] + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "public", + "projection_count": "12", + "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", + "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | TransformFunctions -- User-defined Python library | " + }, + "name": "public" } }, "systemMetadata": { @@ -6342,22 +6141,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "status", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", - "type": "TRANSFORMED" - } - ] + "removed": false } }, "systemMetadata": { @@ -6367,21 +6157,30 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - }, - { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" - } + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" ] } }, @@ -6391,9 +6190,25 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -6404,6 +6219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6418,7 +6234,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -6435,7 +6251,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -6448,20 +6264,20 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(shipping_dimension.shipping_key)", - "Projection_size": "1 KB", + "Segmentation_key": "hash(date_dimension.date_key)", + "Projection_size": "145 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "shipping_dimension_super", + "name": "date_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.shipping_dimension_super", + "schemaName": "Vmart.public.date_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -6480,7 +6296,7 @@ }, "fields": [ { - "fieldPath": "shipping_key", + "fieldPath": "date_key", "nullable": true, "description": "", "type": { @@ -6493,20 +6309,20 @@ "isPartOfKey": false }, { - "fieldPath": "ship_type", + "fieldPath": "date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=30)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ship_mode", + "fieldPath": "full_date_description", "nullable": true, "description": "", "type": { @@ -6514,12 +6330,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=10)", + "nativeDataType": "VARCHAR(length=18)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ship_carrier", + "fieldPath": "day_of_week", "nullable": true, "description": "", "type": { @@ -6527,31 +6343,234 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "VARCHAR(length=9)", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" + }, + { + "fieldPath": "day_number_in_calendar_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_week_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_month_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_week_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=9)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_half_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "holiday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weekday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "selling_season", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } ] } }, @@ -6563,20 +6582,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Projections" ] } }, @@ -6588,7 +6600,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -6613,7 +6625,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -6624,6 +6636,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6638,7 +6651,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -6655,7 +6668,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -6668,20 +6681,20 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)", - "Projection_size": "2564 KB", + "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)", + "Projection_size": "19 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "inventory_fact_super", + "name": "product_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.inventory_fact_super", + "schemaName": "Vmart.public.product_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -6700,7 +6713,7 @@ }, "fields": [ { - "fieldPath": "date_key", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { @@ -6713,7 +6726,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_key", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { @@ -6726,33 +6739,85 @@ "isPartOfKey": false }, { - "fieldPath": "product_version", + "fieldPath": "product_description", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=128)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "warehouse_key", + "fieldPath": "sku_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "qty_in_stock", + "fieldPath": "category_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_type_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fat_content", "nullable": true, "description": "", "type": { @@ -6765,15 +6830,158 @@ "isPartOfKey": false }, { - "fieldPath": "inventory_date", + "fieldPath": "diet_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight_units_of_measure", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_width", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_height", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_depth", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lowest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "highest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "average_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discontinued_flag", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -6791,7 +6999,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -6809,32 +7017,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -6859,7 +7042,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -6870,6 +7053,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6884,7 +7068,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -6901,7 +7085,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -6912,22 +7096,22 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { "ROS_Count": "1", - "Projection_Type": "is_aggregate_projection, has_expressions", + "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(readings.meter_id)", - "Projection_size": "0 KB", + "Segmentation_key": "hash(promotion_dimension.promotion_key)", + "Projection_size": "3 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "readings_topk", + "name": "promotion_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.readings_topk", + "schemaName": "Vmart.public.promotion_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -6946,7 +7130,7 @@ }, "fields": [ { - "fieldPath": "meter_id", + "fieldPath": "promotion_key", "nullable": true, "description": "", "type": { @@ -6959,52 +7143,151 @@ "isPartOfKey": false }, { - "fieldPath": "recent_date", + "fieldPath": "promotion_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "VARCHAR(length=128)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "recent_value", + "fieldPath": "price_reduction_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" + }, + { + "fieldPath": "promotion_media_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "coupon_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_media_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_provider", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_begin_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_end_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } ] } }, @@ -7016,20 +7299,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Projections" ] } }, @@ -7041,7 +7317,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7066,7 +7342,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7077,6 +7353,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7091,7 +7368,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -7108,7 +7385,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7119,22 +7396,22 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { "ROS_Count": "1", - "Projection_Type": "is_aggregate_projection, has_expressions", + "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(clicks.page_id, (clicks.click_time)::date)", - "Projection_size": "0 KB", + "Segmentation_key": "hash(vendor_dimension.vendor_key)", + "Projection_size": "1 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "clicks_agg", + "name": "vendor_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.clicks_agg", + "schemaName": "Vmart.public.vendor_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7153,7 +7430,85 @@ }, "fields": [ { - "fieldPath": "page_id", + "fieldPath": "vendor_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", "nullable": true, "description": "", "type": { @@ -7164,6 +7519,19 @@ "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false } ] } @@ -7179,7 +7547,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -7197,32 +7565,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7247,7 +7590,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7258,6 +7601,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7272,7 +7616,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -7289,7 +7633,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7302,20 +7646,20 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(phrases.phrase)", - "Projection_size": "0 KB", + "Segmentation_key": "hash(customer_dimension.customer_key)", + "Projection_size": "2119 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "phrases_super", + "name": "customer_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.phrases_super", + "schemaName": "Vmart.public.customer_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7334,7 +7678,20 @@ }, "fields": [ { - "fieldPath": "phrase", + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_type", "nullable": true, "description": "", "type": { @@ -7342,7 +7699,254 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_gender", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "household_id", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "marital_status", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_age", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number_of_children", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_income", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "occupation", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "largest_bill_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_membership_card", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_since", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_stage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false } @@ -7360,7 +7964,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -7378,32 +7982,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7426,121 +8005,9 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "vertica", - "env": "PROD", - "database": "vmart", - "schema": "store", - "projection_count": "3", - "udx_list": "", - "udx_language": "" - }, - "name": "store" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:vertica" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Schema" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7551,6 +8018,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7565,12 +8033,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -7582,7 +8050,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7592,17 +8060,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.879951+00:00", - "table_size": "2 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(employee_dimension.employee_key)", + "Projection_size": "327 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "store_dimension", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "employee_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_dimension", + "schemaName": "Vmart.public.employee_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7621,7 +8095,7 @@ }, "fields": [ { - "fieldPath": "store_key", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { @@ -7631,10 +8105,10 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": true + "isPartOfKey": false }, { - "fieldPath": "store_name", + "fieldPath": "employee_gender", "nullable": true, "description": "", "type": { @@ -7642,25 +8116,25 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(length=8)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_number", + "fieldPath": "courtesy_title", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=8)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_address", + "fieldPath": "employee_first_name", "nullable": true, "description": "", "type": { @@ -7668,12 +8142,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_city", + "fieldPath": "employee_middle_initial", "nullable": true, "description": "", "type": { @@ -7681,12 +8155,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(length=8)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_state", + "fieldPath": "employee_last_name", "nullable": true, "description": "", "type": { @@ -7694,38 +8168,38 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_region", + "fieldPath": "employee_age", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "floor_plan_type", + "fieldPath": "hire_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "photo_processing_type", + "fieldPath": "employee_street_address", "nullable": true, "description": "", "type": { @@ -7733,12 +8207,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(length=256)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "financial_service_type", + "fieldPath": "employee_city", "nullable": true, "description": "", "type": { @@ -7746,64 +8220,64 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "selling_square_footage", + "fieldPath": "employee_state", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=2)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "total_square_footage", + "fieldPath": "employee_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "first_open_date", + "fieldPath": "job_title", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_remodel_date", + "fieldPath": "reports_to", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "number_of_employees", + "fieldPath": "salaried_flag", "nullable": true, "description": "", "type": { @@ -7816,7 +8290,7 @@ "isPartOfKey": false }, { - "fieldPath": "annual_shrinkage", + "fieldPath": "annual_salary", "nullable": true, "description": "", "type": { @@ -7829,7 +8303,7 @@ "isPartOfKey": false }, { - "fieldPath": "foot_traffic", + "fieldPath": "hourly_rate", "nullable": true, "description": "", "type": { @@ -7837,12 +8311,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "monthly_rent_cost", + "fieldPath": "vacation_days", "nullable": true, "description": "", "type": { @@ -7868,13 +8342,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -7886,7 +8360,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7897,8 +8371,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -7911,7 +8385,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7922,6 +8396,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7936,12 +8411,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -7953,7 +8428,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7963,17 +8438,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.890717+00:00", - "table_size": "8646 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(warehouse_dimension.warehouse_key)", + "Projection_size": "2 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "store_orders_fact", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "warehouse_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_orders_fact", + "schemaName": "Vmart.public.warehouse_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7992,72 +8473,7 @@ }, "fields": [ { - "fieldPath": "product_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_version", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "vendor_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order_number", + "fieldPath": "warehouse_key", "nullable": true, "description": "", "type": { @@ -8070,72 +8486,216 @@ "isPartOfKey": false }, { - "fieldPath": "date_ordered", + "fieldPath": "warehouse_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=20)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "date_shipped", + "fieldPath": "warehouse_address", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=256)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "expected_delivery_date", + "fieldPath": "warehouse_city", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=60)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "date_delivered", + "fieldPath": "warehouse_state", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "CHAR(length=2)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "quantity_ordered", + "fieldPath": "warehouse_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false - }, + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(shipping_dimension.shipping_key)", + "Projection_size": "1 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" + }, + "name": "shipping_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Vmart.public.shipping_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "fieldPath": "quantity_delivered", + "fieldPath": "shipping_key", "nullable": true, "description": "", "type": { @@ -8148,7 +8708,7 @@ "isPartOfKey": false }, { - "fieldPath": "shipper_name", + "fieldPath": "ship_type", "nullable": true, "description": "", "type": { @@ -8156,85 +8716,33 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "unit_price", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "shipping_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "total_order_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_in_stock", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=30)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "reorder_level", + "fieldPath": "ship_mode", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=10)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "overstock_ceiling", + "fieldPath": "ship_carrier", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=20)", "recursive": false, "isPartOfKey": false } @@ -8252,13 +8760,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -8270,7 +8778,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -8281,8 +8789,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -8295,7 +8803,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -8306,6 +8814,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8320,12 +8829,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -8337,7 +8846,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -8347,17 +8856,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.883186+00:00", - "table_size": "225060 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)", + "Projection_size": "2567 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "store_sales_fact", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "inventory_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_sales_fact", + "schemaName": "Vmart.public.inventory_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -8415,184 +8930,41 @@ "isPartOfKey": false }, { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "promotion_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "pos_transaction_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_quantity", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "gross_profit_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_time", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "TIME()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "tender_type", + "fieldPath": "warehouse_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_sales_date", + "fieldPath": "qty_in_stock", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_sales_datetime", + "fieldPath": "inventory_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false } @@ -8610,13 +8982,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -8628,7 +9000,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -8639,8 +9011,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -8653,7 +9025,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -8664,6 +9036,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8678,12 +9051,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -8695,7 +9068,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -8706,22 +9079,22 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { "ROS_Count": "1", - "Projection_Type": "is_super_projection", + "Projection_Type": "is_aggregate_projection, has_expressions", "Is_Segmented": "True", - "Segmentation_key": "hash(store_dimension.store_key)", - "Projection_size": "2 KB", + "Segmentation_key": "hash(readings.meter_id)", + "Projection_size": "0 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "store_dimension_super", + "name": "readings_topk", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_dimension_super", + "schemaName": "Vmart.public.readings_topk", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -8740,202 +9113,7 @@ }, "fields": [ { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_name", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_address", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_city", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_state", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=2)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_region", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "floor_plan_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "photo_processing_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "financial_service_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "selling_square_footage", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "total_square_footage", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "first_open_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_remodel_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number_of_employees", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "annual_shrinkage", + "fieldPath": "meter_id", "nullable": true, "description": "", "type": { @@ -8948,20 +9126,20 @@ "isPartOfKey": false }, { - "fieldPath": "foot_traffic", + "fieldPath": "recent_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "monthly_rent_cost", + "fieldPath": "recent_value", "nullable": true, "description": "", "type": { @@ -8969,7 +9147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false } @@ -8987,38 +9165,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { - "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Projections" ] } }, @@ -9030,7 +9183,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -9041,8 +9194,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -9055,7 +9208,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -9066,6 +9219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9080,12 +9234,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -9097,268 +9251,60 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "ROS_Count": "2", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(store_sales_fact.date_key, store_sales_fact.product_key, store_sales_fact.product_version, store_sales_fact.store_key, store_sales_fact.promotion_key, store_sales_fact.customer_key, store_sales_fact.employee_key, store_sales_fact.pos_transaction_number)", - "Projection_size": "225060 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" - }, - "name": "store_sales_fact_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_sales_fact_super", - "platform": "urn:li:dataPlatform:vertica", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "date_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_version", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "promotion_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "pos_transaction_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_quantity", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "gross_profit_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_time", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "TIME()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "tender_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=8)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_sales_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_aggregate_projection, has_expressions", + "Is_Segmented": "True", + "Segmentation_key": "hash(clicks.page_id, (clicks.click_time)::date)", + "Projection_size": "0 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" + }, + "name": "clicks_agg", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Vmart.public.clicks_agg", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "fieldPath": "store_sales_datetime", + "fieldPath": "page_id", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -9376,7 +9322,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -9394,32 +9340,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -9430,8 +9351,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -9444,7 +9365,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -9455,6 +9376,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9469,12 +9391,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -9486,7 +9408,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -9499,195 +9421,39 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)", - "Projection_size": "8646 KB", + "Segmentation_key": "hash(phrases.phrase)", + "Projection_size": "0 KB", "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" - }, - "name": "store_orders_fact_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_orders_fact_super", - "platform": "urn:li:dataPlatform:vertica", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "product_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_version", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "vendor_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date_ordered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date_shipped", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_delivery_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date_delivered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_ordered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_delivered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, + "Number_Of_Partitions": "0", + "Projection_Cached": "False" + }, + "name": "phrases_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Vmart.public.phrases_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "fieldPath": "shipper_name", + "fieldPath": "phrase", "nullable": true, "description": "", "type": { @@ -9695,85 +9461,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "unit_price", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "shipping_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "total_order_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_in_stock", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reorder_level", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "overstock_ceiling", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=128)", "recursive": false, "isPartOfKey": false } @@ -9791,7 +9479,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -9809,32 +9497,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -9845,8 +9508,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -9859,7 +9522,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -9868,12 +9531,12 @@ "platform": "vertica", "env": "PROD", "database": "vmart", - "schema": "online_sales", + "schema": "store", "projection_count": "3", "udx_list": "", "udx_language": "" }, - "name": "online_sales" + "name": "store" } }, "systemMetadata": { @@ -9884,7 +9547,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -9900,7 +9563,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -9916,7 +9579,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -9934,7 +9597,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -9948,30 +9611,9 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -9982,6 +9624,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9996,12 +9639,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -10013,7 +9656,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10023,17 +9666,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.900841+00:00", - "table_size": "6 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(store_dimension.store_key)", + "Projection_size": "2 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "call_center_dimension", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "store_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.call_center_dimension", + "schemaName": "Vmart.store.store_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -10052,46 +9701,124 @@ }, "fields": [ { - "fieldPath": "call_center_key", + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, - "isPartOfKey": true + "isPartOfKey": false }, { - "fieldPath": "cc_closed_date", + "fieldPath": "floor_plan_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_open_date", + "fieldPath": "photo_processing_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_name", + "fieldPath": "financial_service_type", "nullable": true, "description": "", "type": { @@ -10099,25 +9826,25 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_class", + "fieldPath": "selling_square_footage", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_employees", + "fieldPath": "total_square_footage", "nullable": true, "description": "", "type": { @@ -10130,80 +9857,80 @@ "isPartOfKey": false }, { - "fieldPath": "cc_hours", + "fieldPath": "first_open_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_manager", + "fieldPath": "last_remodel_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_address", + "fieldPath": "number_of_employees", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_city", + "fieldPath": "annual_shrinkage", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_state", + "fieldPath": "foot_traffic", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_region", + "fieldPath": "monthly_rent_cost", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -10221,13 +9948,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -10239,7 +9966,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -10250,8 +9977,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -10264,7 +9991,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -10275,6 +10002,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10289,12 +10017,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -10306,7 +10034,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10316,17 +10044,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.897788+00:00", - "table_size": "9 KB" + "ROS_Count": "2", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(store_sales_fact.date_key, store_sales_fact.product_key, store_sales_fact.product_version, store_sales_fact.store_key, store_sales_fact.promotion_key, store_sales_fact.customer_key, store_sales_fact.employee_key, store_sales_fact.pos_transaction_number)", + "Projection_size": "225096 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "online_page_dimension", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "store_sales_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_page_dimension", + "schemaName": "Vmart.store.store_sales_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -10345,7 +10079,7 @@ }, "fields": [ { - "fieldPath": "online_page_key", + "fieldPath": "date_key", "nullable": true, "description": "", "type": { @@ -10355,36 +10089,36 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": true + "isPartOfKey": false }, { - "fieldPath": "start_date", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "end_date", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "page_number", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { @@ -10397,7 +10131,111 @@ "isPartOfKey": false }, { - "fieldPath": "page_description", + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pos_transaction_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_quantity", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gross_profit_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_type", "nullable": true, "description": "", "type": { @@ -10405,20 +10243,59 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(length=16)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "page_type", + "fieldPath": "transaction_time", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIME()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tender_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_datetime", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", "recursive": false, "isPartOfKey": false } @@ -10436,13 +10313,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -10454,7 +10331,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -10465,8 +10342,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -10479,7 +10356,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -10490,6 +10367,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10504,12 +10382,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -10521,7 +10399,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10531,17 +10409,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.903963+00:00", - "table_size": "182356 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)", + "Projection_size": "8646 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "online_sales_fact", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "store_orders_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_sales_fact", + "schemaName": "Vmart.store.store_orders_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -10560,7 +10444,7 @@ }, "fields": [ { - "fieldPath": "sale_date_key", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { @@ -10573,7 +10457,7 @@ "isPartOfKey": false }, { - "fieldPath": "ship_date_key", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { @@ -10586,7 +10470,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_key", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { @@ -10599,7 +10483,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_version", + "fieldPath": "vendor_key", "nullable": true, "description": "", "type": { @@ -10612,7 +10496,7 @@ "isPartOfKey": false }, { - "fieldPath": "customer_key", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { @@ -10625,7 +10509,7 @@ "isPartOfKey": false }, { - "fieldPath": "call_center_key", + "fieldPath": "order_number", "nullable": true, "description": "", "type": { @@ -10638,59 +10522,59 @@ "isPartOfKey": false }, { - "fieldPath": "online_page_key", + "fieldPath": "date_ordered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "shipping_key", + "fieldPath": "date_shipped", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "warehouse_key", + "fieldPath": "expected_delivery_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_key", + "fieldPath": "date_delivered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "pos_transaction_number", + "fieldPath": "quantity_ordered", "nullable": true, "description": "", "type": { @@ -10703,7 +10587,7 @@ "isPartOfKey": false }, { - "fieldPath": "sales_quantity", + "fieldPath": "quantity_delivered", "nullable": true, "description": "", "type": { @@ -10716,20 +10600,20 @@ "isPartOfKey": false }, { - "fieldPath": "sales_dollar_amount", + "fieldPath": "shipper_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ship_dollar_amount", + "fieldPath": "unit_price", "nullable": true, "description": "", "type": { @@ -10737,12 +10621,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "net_dollar_amount", + "fieldPath": "shipping_cost", "nullable": true, "description": "", "type": { @@ -10750,12 +10634,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cost_dollar_amount", + "fieldPath": "total_order_cost", "nullable": true, "description": "", "type": { @@ -10763,12 +10647,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "gross_profit_dollar_amount", + "fieldPath": "quantity_in_stock", "nullable": true, "description": "", "type": { @@ -10776,46 +10660,33 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "online_sales_saledate", + "fieldPath": "reorder_level", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "online_sales_shipdate", + "fieldPath": "overstock_ceiling", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -10833,13 +10704,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -10851,7 +10722,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -10862,8 +10733,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -10874,9 +10745,100 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "online_sales", + "projection_count": "3", + "udx_list": "", + "udx_language": "" + }, + "name": "online_sales" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -10887,6 +10849,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10901,7 +10864,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -10918,7 +10881,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10944,7 +10907,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_page_dimension_super", + "schemaName": "Vmart.online_sales.online_page_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -11054,7 +11017,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -11072,32 +11035,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -11122,7 +11060,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -11133,6 +11071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -11147,7 +11086,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -11164,7 +11103,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -11190,7 +11129,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.call_center_dimension_super", + "schemaName": "Vmart.online_sales.call_center_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -11378,7 +11317,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -11396,32 +11335,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -11446,7 +11360,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -11457,6 +11371,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -11471,7 +11386,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -11488,7 +11403,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -11502,7 +11417,7 @@ "Projection_Type": "is_super_projection", "Is_Segmented": "True", "Segmentation_key": "hash(online_sales_fact.sale_date_key, online_sales_fact.ship_date_key, online_sales_fact.product_key, online_sales_fact.product_version, online_sales_fact.customer_key, online_sales_fact.call_center_key, online_sales_fact.online_page_key, online_sales_fact.shipping_key)", - "Projection_size": "182356 KB", + "Projection_size": "182385 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" @@ -11514,7 +11429,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_sales_fact_super", + "schemaName": "Vmart.online_sales.online_sales_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -11806,7 +11721,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -11824,32 +11739,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { From 9470237c522908c60deb68626d94a8fa2904b08b Mon Sep 17 00:00:00 2001 From: rajasekhar-vuppala Date: Mon, 3 Jun 2024 08:43:06 -0400 Subject: [PATCH 6/8] fixed linting issues --- metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index a1b98126feef1a..e8043a49da5f73 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -171,7 +171,6 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit ), ) - if sql_config.include_projections: yield from self.loop_projections(inspector, schema, sql_config) if sql_config.include_models: @@ -305,7 +304,7 @@ def loop_views( entityUrn=dataset_snapshot.urn, aspect=lineage_info, ).as_workunit() - + except Exception as e: logger.warning( f"Unable to get lineage of view {schema}.{view} due to an exception.\n {traceback.format_exc()}" From ffe39dfa32723f6926923b7f2fa5dbb39212bb0b Mon Sep 17 00:00:00 2001 From: rajasekhar-vuppala Date: Mon, 3 Jun 2024 08:48:25 -0400 Subject: [PATCH 7/8] deleted recipe file --- reciep.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 reciep.yml diff --git a/reciep.yml b/reciep.yml deleted file mode 100644 index 605380cc9a8948..00000000000000 --- a/reciep.yml +++ /dev/null @@ -1,26 +0,0 @@ -source: - type: vertica - config: - host_port: '10.20.73.210:5433' - database: Vertica_db - username: "dbadmin" - password: "vdb" - schema_pattern: - deny: - - - '^staging.*' - include_tables: true - include_views: true - include_projections: false - include_models: false - include_view_lineage: false - include_projection_lineage: false - profiling: - enabled: false - stateful_ingestion: - enabled: false -sink: - type: datahub-rest - config: - server: 'http://localhost:8080' - From ec95b21e04cc7d9e9b28d1a035564a31a2dce3e0 Mon Sep 17 00:00:00 2001 From: rajasekhar-vuppala Date: Fri, 7 Jun 2024 05:47:06 -0400 Subject: [PATCH 8/8] changes based on the comments --- .../src/datahub/ingestion/source/sql/vertica.py | 10 ---------- .../tests/integration/vertica/docker-compose.yml | 1 + .../tests/integration/vertica/vertica_to_file.yml | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index e8043a49da5f73..5b1a815e178b18 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -135,16 +135,6 @@ def create(cls, config_dict: Dict, ctx: PipelineContext) -> "VerticaSource": def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: yield from super().get_workunits_internal() sql_config = self.config - if logger.isEnabledFor(logging.DEBUG): - # If debug logging is enabled, we also want to echo each SQL query issued. - sql_config.options.setdefault("echo", True) - - # Extra default SQLAlchemy option for better connection pooling and threading. - # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow - if sql_config.is_profiling_enabled(): - sql_config.options.setdefault( - "max_overflow", sql_config.profiling.max_workers - ) for inspector in self.get_inspectors(): profiler = None diff --git a/metadata-ingestion/tests/integration/vertica/docker-compose.yml b/metadata-ingestion/tests/integration/vertica/docker-compose.yml index 24cc4ae715d36b..1ba7990c826b2e 100644 --- a/metadata-ingestion/tests/integration/vertica/docker-compose.yml +++ b/metadata-ingestion/tests/integration/vertica/docker-compose.yml @@ -4,6 +4,7 @@ services: platform: linux/amd64 environment: APP_DB_USER: "dbadmin" + APP_DB_PASSWORD: "abc123" container_name: vertica-ce image: vertica/vertica-ce:23.4.0-0 ports: diff --git a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml index 2b3e63c40071e2..6f628011279edd 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml +++ b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml @@ -4,7 +4,7 @@ source: host_port: 127.0.0.1:5433 database: Vmart username: dbadmin - password: + password: abc123 include_tables: true include_views: true include_projections: true