From da58bc0791dd4cad6f243f5830d9e9e742b343f2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 18:36:49 +0000 Subject: [PATCH 01/95] README: what it does, how to install it and where it came from (#26) The README now opens with what the code does in the words developers search for, then the install command, try-it links and, where there are any, the repositories it was created from. It ends with a Find out more section, and the logo is a file in the repository. Everything else the README said is kept. --- .github/images/logo.png | Bin 0 -> 17167 bytes README.md | 55 ++++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 .github/images/logo.png diff --git a/.github/images/logo.png b/.github/images/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..4be7430064391b6e021bb33b74ecb4a2d4fcbaad GIT binary patch literal 17167 zcmeHv_g@piw{}1}f+$T<1XM(VpmeE0iYUGJj)3&uOHe@+6hw-&(0dELMTnw+bRqN> zkX}L{bdr#F{oZ^3i2Hf(FPq)TIWuQx=FFUVW}f|^qov9~$3X`Iff!yqSJnf8$bnmO z5DgXZa~L#s4gxWOUMN2`2+G=-r_M4MpYPsx3J7d64#BD@P*5oE2nMKp67P-`=P+MT z{#4uZ#&W0KUiDLV>Z={@H=~6o3#l0QfPi(mbs_GWXm?Q=(UTKmSbSiBBinDsz+qqQ zvz#Zo4OyDStZAv9X4TI98kw#$w?dDCWaC(h zW$BoU>#;UMp`Me^GEyIcKzEE2o%J;{U(UQlMLf>4Dd<#)qXMBeOEUVMi!F1@wYWf$ zubkd~M-{i*b}D=@dmcvtLhWA>|JrXjepUj!`bn=S#t@Y){?w*T@nIp^cRvC?z2Co> z^DYQfGmg|m%u>9@vfSaJ1r;whb9P`(PnbcV(oYPazzpua4{sip{?iDcB#9Pd01X{$ zNLoN1Ex9ktJz+X>)P6~3Ktk{nCBF+0NGus6XxBfI%$`3kVx3Uw`rU9Cvp?<*c zdj$g_>m4-+b;ji?#P#60I0azx+fz+^J1rT>S$Zg*{1oPDhMS3t@hp^CS5o3kxNKk_dlyBeA8{s zp#L%zA8we}q$!;`i=HV44KcwWAW))*BxOaYm&iDeh&u(KrK4Of)p?5$#tlKhm!~OWZc{!AY)-bu3G7zRP=wOn8<*8_B+63 zQ4uHrWbq!jMABacu3kL<@)S}AF8?R|zpggAXd?v#EgF8e9%E3dpt+cTE&9K!aXG5& zdav<_{4@L$+XDgpCAkxt#2Db|RYMEeB?H{9^4a<@`yG0q2miU}Ef-R{$R$aa82wM+ z%am1n*40QHR|!qx+y5k_9r~@C;np%0SGa-aj9>MB@uCIsUa~lj^zai%#{Ign9uhwypkqgU2!_<@p_>$NM=-L3MJmbnL?NY+O5*l!u zs0v7W+Jh(v@OSP=AM~PV##cDqRPhv;pg(-X+@ zg|>dDuIhiMe~tCuHt_1N67NX$>%O7*k_Rw)t7!2YDWGFvL5wcE$D8Ay%4GV_Dob$4 zp>B$HrqQx1zkN>zKZuQ_!tu`vts*QE{7?Tn6 zCp78WwW#0!2CcE*w~sQRtA=_G4xiwms+giEH0OQg;_GCf8fJ6?1bosNS8j}-?HoEQ znfH!iYQ0XTAf7TUg?nU?ui#ylac3z~cTMJ_r!e>EX$M{_*w}*qj3|!$v8;Rk^9p|x zXV1m6x|&nWtp_I+VQaVY)aE?GS(CmL>Ktgo^8QXPq+n-z%O47ezTnO+eg!$R4<6O4 zkb^kwTCh$wq?FQL`^zjAw6{zCM~oN#t-KSDh5JvT)mm|MU01k-l$&a3Ko=_N4QB)D z(cmg{u$C|aTWlkcBd6VwQ(O@?b2()=qaya`CM(^f)k)wWOj7&DXr5ANQE_3pX6A&; z{7i#U)flw-Vr#sB`DWvMJr%Zx)nLg7G+WR?W21SO)>tPZL$re3RDXpTVh|E0I)EPpYUNiKV(pksLIy?6T?Fg4_ zapfxoQAFMslK2@AwhFfXHg;q5L^GpUL^Ox%!NY6i6{V$r&W~jA@y9M$Du@Z>$eIQlTibhd1`uWL{1iIh{`2&QL9JVQUx}7)!1Z5PalPyT z4W6LuCm*k+P)38pPH&_{9C8`1tWKVHGV_ac z_wTBfOHtC%8%lfUt1SfSvsM`K7i-=un%~#%O1Lc;r4{rMl$K{QHhA&JsB)g0bT|zr zp3*z@TYX<=ep69b=QCUSJxLo<ZDT%JDTCUR9`y!GPqPLKzG;Z*E zFw+osJPQt9w=}G_Mq|jB0Lxm>-Ek{Nv9Wu36*v7!W>DqEdEpk*9`~=^%($m$ihR6GS z<5ueD%js%osGOB0rxF!!Hihd4yTZ?DwB<#KxD|bE*Qve43^C_AQA1GFO#~9MK8D(w zKkM<^$gP~T_S9RqzF1~iwem~!(Sj71t$V7nn6*oem~N=9qj8<~R~G@#Z=1@VkJ=0T z(e?ESJ%uNshv6d?x7%1Clw?#(#LN7=5Uw&S6W&Sf?2NfB5_sWTflNy)9YYdSUX;MV zQ3*BI$!Jz;>2vRU&m+9jv?yv&k^oEaYWVTS1u;I4u1XQgBa26`Lo&J(@JA2 z4}*29t_{=8|Q15~fOAmO+D?&!@spOXsS-1Hn>L+9+w5hstGoa9MacGCXW zI<-yatc)s6AB{acOI}{ZhkRkWK}S#TKQ0a($!K&HJN|Y+5%mT086$xR5$25OZt;$V zoN|jBY*Lf-mfBCipL492w}mk?{L2@37Ri_InL0WPSiC+UTv3y}`!$zVUOe{-Ctf_a zQ@@P%KatBo+T>jXZ&$@8lms!o>1UhhZ9mk;4X5eCPiDI!(i0%XQ}W?sOeqz{0;J+h z!wZylV+Y^G_e&`eJrI~r2aV@aEzKX*bt87f^*<2LJSNixDCtrP%h!#IP%IAx_bafk zZ;H$@xsy{qxiPSZ5n*OL_ltT=h4#*KcNA6AP{IlDx3sVx}9Nb_|hnd5n#5 z+g*8LiN8G<2rDRU|N0x04<0>Y|1;;72%GaAt_WPuImIU>pZ$$v*d?f=q1LVJ7ACIi z*esS;Yv-GUyYz9OyODt#yNW9zcvztS2(M$%1eX1e1^L<#WPQ%pu=eGP_Jy3o7{~+6 zTIRS;bYO!1>l??$f{j9MoTSQ0z6bKpV<>ro%w$1RxTaEbLaAKzZ^z+l*Gaz%$TQx` z-MrT`muURZ+zNiemmSfj2S*q&Tz&jn9J^IvC}OKvh5fKO_aYtMUOwe9_V9zU;B63C z!tL7D8U}6dJugk4*sn!RX>RWOKUje5#fpB&>i8rJ4F!dmnQ1yssMUa#o^5O!^P(iD zxB+Pme@By?3dYv;uhz#LkKJi9e&mpjU7w2QjQ;(LMIq|>lUH0kIJvz)KQm+kvm?8~ zEtumF`0>fEs2TsHWy!lb81$@grkQJY{*vjV_R_S8PZb>R_x}7vfWSu8Mvs4CTVu@H z&(l6U8AEt@!c4RAqw$jnb zg=G6y-|rx%h8)^>^KlgKKuCt0FR;{#%xnx*qU>& zh)C4$iN9prHfwcn?g&Y^NJ9mh)cJzHBn?bfm?ZALsa+RH>+2_G2>ZTU2oB7bC+_&3 zX*}7B9sv^wgs^$<$@!4;-k#O{Ad0aVIko0RE>bBv(>*b#hf5SG`emGn_xP|wTfU6Q zP+O}8J-oX9z*@P34a;-ML7q}cBXffzBi0%HOqibmZ(IF=k6)LT`lHb~gRwxhIlIbT zxTMzIcULCo0~hkjAX=lL*p33|LPX}XcO&3cIP&aS0%X51@*I4+&Pg{`|IIUlmaf(k zQO-jUbt>QMz;B0X_wlm#%h*IRahioud`AXFzu}p==Axk%`URJX4byM_^K>uh1f$cDUYMW;k zYsqp-ho0k4Z=osqKx$XIc`YdXuepn7}pCFY%`MOlsR4(CbJZ64&`T)HMA;mq~uEcus-tx<3 z3eRt1RB$`l*tsssg)nzt2z^=xl};E!-kA3y`iESkJB^harYgD*&9#OcpRohh3Jp~x z6edC8-4BQx1(qE>GHz*1;bvkWiasJ8%UAGq*$=d_&jMH>89AH`Q-SD_kgX3rN*xb| zg}&E0yqT;ty=&uiVL349nG$61GiC$B;&!&uiwMi9I!V|-)!hZ9uxhn$+PrUakEHoz zs}!?^k+d@IVfrd5$j?90b98Gf8e!lu*V2FT3kXQGE8dZY4u;FoGj9~US0=fh`U~C! zx#=Plir(-v2}XneJV{Sa4>>rR9BXuawSDi~#raJzy`JS|Wl8KpXoWDie!H2fLL^3M zHea`2i|jOvj)f}k8n^kvA7$78&G+z=xq_&uKbpLP*0qW`;TNrun63uTW}xURT-DvB za~4QTlZ&n+8K3KkykK^+XRgsQQ4`re-)$wvaESzq_`Sxh8e1_`)OEcIKH_&V%K5C5 zbWRo7HbkYc;xY}=(RF@C_GYhNDE=x)3^tz{_~$kClNk2szWlDZ#I?}~Mk#(N<_8Op^I0UfjAH8xcALD`6`ot>TE_MoKHHo0P6O^pue z+-q{SZt?a$Fr?w+vY*`!oA~_mjT$;c{k@Q`6eUwq^1a;Fd+FE`>Rp8K0#nMwmxY%K_GYwum^`c8aoVUyhv|Dppe1)CzW1%6sjknK z5u18aqE&F3Fd=kd88}BuHN{o($u=W-Kwioj;u4|-H51~K&9|_9E9&ZOLEN{u)ETy| zaur$=1&y!v>R7A{5XzL*yjZO;kPpo!sHbM;#CJ~yqC@tyZ2EgxwwNK7!Lyc(;d&g~&55Re9#fOP}VxJw(Dg^lFH1(CLKL4MT$ukW-tk z0Bm*aYQJk(d)T=juUP)ITt2B6-l03XSf8IpKaQZr6SJPe92}Hp4JF_lgu;(SQL@~X z_JmdImckb>lF%6uH*8+|B~aerLhyti-%_6(z7-8V+kli0^froHMQ>ay*T*{!W((Sc z@5egwRcuNHD`tn|&EHA1{_%%(Fli{}t^O$u zkN7d+Kv9o#509k!Eo~VAmBQmWU+2lS8o??4wQtH!3ZqLVWY+05jG~N%J#SHR#ek<#K zkC`+5@ToM&^mx+=Oq`5SeCaZ+5_!Ph@7@Q6!L$0Sec2d2=4~T?+g(!VdA8H)QCe3g zvbw%bKb376p5u*`18l;LsvqA}{gmi)G|o_dxy5j)#k+h3FLtMKheqMW^1F&VFu`ZZ z(1Z3qr0X}P;J<(WirFaqJh6~8NT+>INf}KZz$W?XvmWLaL2GBfqw7&eVfTCIaW#Hu zhXsR!>g`yraD(@tf1PNJyxvkaXbZ67x>)@8-eC+RW%EO=o@KX zSU~x;?vG{1VIv2_>D)R<{fly9x>Gj4=iPWVp71sai5Iez7MPl-wiL(}65ibG3p+Li z#$x2p14ivPkojMd1nmNyXj;U)1kE z#hR{UOLv-_hRc~gCpTbP>$K$)vD@%*ld@*Ck^9R$t+A^24+JI2KYxX%jj`CO{zFeT z&S=Ra-^Jd1qO+MG|2NB^&G6!rr1<%e?0sI9O-FFv0v0=^`~DX2uHP=Dp!TX!YSDr|u|U6*^#u|3-S3i71}>DvKHCM?#xMno zY*`&<1f4D^zND>CiLAQKRq}Bbo>XF(%AL}2*1^;v;ck~fwKRBxc1oJl#^V8w6yRQP z<<0>khrzaM09if^tniW+ZVB4oSCd|S(n#E31@Hvh3;)4`;e(*|*j|mm;v0Y4{B_-$ zhh)U?78dYsxtWxg(Io zNWy@9mJtzxVxgmzE!u zA0EJ61XH#@oPa%kb9Y^tx1qW>=iB@}R^L~=jd!2uWSusLy zlh3+Oa+A++?OoHu3WmEX@Q#9HNcO|1um0PU6|%oWvoU3FKdh~TUNKsUCghxL3H03z zeI0u{Sj7 z(4-{86Gp%tJfds*^-I~Iu$~GuC{V0K?a)rBQ?m-y>^;gtoD64`DGbH0&WdISSu9+= z%iVdv|Dl(RiZN+1S<1289@Qj_{@5Ysb41QLi}%52mh-?iAJ08N7V;bXWg zy{botQUWK`joOB@V{5HTwUS~l*0b-Xq|Ow!_Md!>I9>AOO4BX-7{leWcA>P2_N_nq z9Vl=#UuVI3wk1iPd5x+H{=OuLIm?)%rWU5_ybK)m0xy)UV zn4X4OoyaZ$A(0ar?fYK)u@o7|*8XSrrx}(v`oqU)h+m)y4TFpyvG` z{C;2$4GK!H4Qw0y*SUaRm}q_h<3`G6gO`q1zhJG0v-)th|? z;&*8otUsl^oLq3@?el;lAGH8N#tgSZ4m5C$ig@zZc570?v{VuF>donrQgXl`FZlXN z>6>1NQwNj$f^eYHD$kRnoyE6|Y^b)RrD;dtchVJ_YVk?f@+Uw<NqV4d z=aUS!;2o}uL~F@S)Y-Ij9n`h;23hWnUje(NX^a>R7sk=PJtnr>ikP(_SV@_F37F*_ z_C}lG#H(+3q*WfG!@|g^$Fu!7=IQy*n&ViOCL@K?0klb_a|K=U&L`cr8>x^&?SorX zPO2tCvn6r6l0Pc%FC#5gKuj4b7Q@xJD*B6w#|p82Iql6k+DnU=L4+Q%1=MjOs%&e(lMUZQs>BoA`H&9~%Bab) zu-we5sguS1aDN@_;w>`BZ;IHVP1?rl-cwuC?D*S@3%Ve31>X-vD=!<&T3u2++E&>_ znVAV?*Ud?!eEM#>rR`{bSB|kW^Cq3a4b_rz?d^LdrNdxJAC9W5F7%+L&wobl0Owg=V-7tLM8+y> zu8d7sH2@PCW33cVrRN7$Atclc2m4n{Z2QfQgZ@>mda5RMBW-57nc`nEG88*reibi0 z4Mh_~HTuPPctm^UjSt|i<_}AzjGYxwkI;=qqJF;rqdP}L5m{XiqpEu<^@>^(Yw%#}67@Jb|c94k_v9_JM z!a#gp#vtD4jkkTebQ2V|$|JK0moo4eFshk)E5j6G4LzvxJ>dceph>~&S%OhWvb7;% zJ$;OH+^l?hb1(n7eRFDSN|ntN(!K7TpPH{Uk;l;k2~A6$YjkCHLEY`2XC?b70!j`$ zO%Joy*6R*$exP^C{?MrASCa>C)I=<+z|mLag=U`Lud6W?p(g;yTXVGOPHR2;&>T(r zDo%OsZ<*-phJkO>zZFTo4L)XU4R`rAQxq~jq@g?2!--%^ zXUKMJ1zgv|qgt`cgNXmRSwf%a4)@s9nqBDb^l_@4CNLP9qa!js;bt~Y&H2-|TRN1{ z@x^yNGtI`+XPK!m=WmaXTUk)OGO+ORgX4@;rscW_J2U)OAVv3Xo2%bG^?{ zo13|l+3`G!yZn)&Lpv_*Z(D$#2_1#O;T#2}zQ+0=)>SVAyyM~-yMH+k&&31hnxYS+$P)i{VJ1(9@&1Xk(JHqtv#`1zOjWFA z;?xJ>>@47uN0I_Shy&|cDh%rUm5T+vch*kKvoULVl?d!*skI;C1i)ahUMY-#Pa11$ zM5uouA1wR+v2gBi`iq%0ay?DnC8^Wa-1aIc)+*E(Si$b#!n@$wW5vrudR-a*bkrTa zCPAs4c0tsp|fC}Piv;r7b! zGUzpr4(rsCezdvH_O@qEBccrvh9w=FJp`S86*3-bHGVQnErSvKzpOSv0d zrDGQCdyoQMmK$p9B>;Qc=A~Km)w*+wWNPwfPr9(8ir$u)f$=>j2O6x%i|ju63q>+t z$r}LrF5QbTx6bpBl4=DD3XHe;Z#M+{#sdrDiZ1okC-}4V0a%6C;aAscqpv~(N^Oaj zxsLrbvj-GSZRe35eixcGcS;PX<%BhZi^idnv+(U-z5$aw;TwcNHXOD^Azi!G82tb} zw&~(=Vd|R#sn}xg2wc%AvI>j;_A}sX&vr^lo%UZ$evLHtV z=HQfEQyzq|&s8NY_3aMJ|K0oPzs1i-ziXe+#d*+qx$5`4_iM9ejc0E-w!e;3N^iB^ z5YK-8s)FU>BtLI!q;H-Q>cWfz>Fg~g;fSa8Z>k6-n|;E9#hL=XeDk{A^KB2Bj01y1 zPaY39rt%epgol&9tZJ!R*Lt)AP(_*Dh@PR_tJd>{7d;}7Tkh|$ttH*_{w0g@0zT^g zrzqqrd)=e>t>~PIjlv&^?emlIZUa`vugntHsvr4iv4(kRcuk7Qo)f{uGh)U%A%Sb^ z89Yn)H~-|vjOd4;DOOSuGpXt?`&*>DM`%i>giKY~IXw2Bx8nw630SaijMUnz%6XyX z)LzggDCseoX%lMNx`)u`uRNNbeaa~}PuM8Ci;oG^qzR4M{jzgy5N)#KcZ`~3c{MT9 z=~BQkF{$ozc-VW%PnopF8Sw~MUld`7Q0l01Pg-n7V(vcQA&M5E4{JATiZDFst@HFyXTFTBp!&-97~}XA+%G>C?EWHpqcwjn(fhLW(pDJ z<*W+J4M+2})E@uZbp|&PSm<%;r4Ktg;Z4q&vHh$MwKdGDPMkc&?%rzmTPp32Uc|*l zdJx<`?Be@0#;GDg0qhupomGKK)ga7x>p$1=ytx>!gGKD3J@g#{;&rY7dt>+>L)>GH zTdZy{=-K;GAeq$(C`5?CzL9gz-1kGdjj_UUsDqYukVYx8eI8hUN(JM^A1! z3m}4U3Ct1W@NKBTxV!cv0!rS}uPefG^YkWvsdb~miw{0R#7zA?>Z7EfUq95QoC=xF zL_|am2lTaFhHaH7P1dpFx}2zG|JqF&R{_JDpLPu1F(^VhMB_v( zbf2!cJ5pJ%gx0FQ|U@;qk<+W`Mp89`SUZq)l7bRnG)#S zcZs&S%ST)`iWC{DKp}*h`4pc`sedO?Y$rvyTGhRYq4bGW?z2r9dou!Ce>&PDXDme8 z>QOtg1gC@gqt*OdM5iB)PPLYW&p#vx2d?zaR9VJ4_imvjOn*w3lv&SqYuiTDK2~K% z%4i3#ZTpNa2AmTYo1_D{*BvO7{_tt(J(4r7#ms`SsrY+2+NGv{_f^yKNLLkK%GeI< zw=JD-r=)aQ(rxurZ$ZLeg_fAQ;`$GXqq5=64~IK zpAR^^G+R;%-#kS)N?>n145043(tmK+cjJms zpqR^g-Iz%!deGH$V87RYgwo#s%mI4s4@&?7&95{g5!@3C|gqK zLJ9S5tq*^MY*38uICmeZH(whC`mSrmOL(2eww-kF?8vOEF={%nVU|&oI_DZ!8(KX; z82u^ch)MD8OLe~XOZc*JSM3K_cyoD}QP6~wCvDRL6aZy>g@3D@57@?zol!?Vzls}75;nb|hFpcm&f0&*ks@N|`jRs8D|>pRoLISW ze%J|3ZOBnn$jKHj=)JO_(Ierr!h-R0>qi0+V_vPAgx`5C2iQrL+0?vm&rwKg$b8`N zlkh|M$tHnJxq(eEX9Y89US-7r&u3rI8S#Zn#T+X)PMPrLc;}Cjyc?NJr2}^>nYFJM z(D!7+0py=dp^!fA`~&|wth$gn-eBqHQ;aIp|6l>b<)@!cSU&B{M|7XcupoOBczj+u zI7s!J#9!|y8AYKeqdHes&w!QtGt0;2_4+DTXx`QE=iW+6DLXX3(EiXv)Z+Dw$Q>Q- z(93;Dr{-|~m%!>Fb4u88SuAG24*C3ZMn?HhMmc5G+Yv2{GCMkCc_Q8u&*o1z?FOfH zDnG1WEu_Kwm{o3rIOWS`-li z;gvFVUSlN9)elch9SpNFsX?j+CLNg9w@VBuXR~Dz4|cA;v<;2Yb2|}oA)Y7U1~V$^ zv^xy;qdr^}y~Bh=KPa;DOVwYDArhDR88@+VF3IuII2xx%P`H)cszWGPaUh%4t89}H z=y>IF+rh?y3+&bcI>~o}U8;v^mi*cKhfCq_@>@mgeb<_btb+4m(dWP^s)!Pt^}2wL zPN~m)Maz*D*ZrhJ&T z|JIc5CF;w6a-Uujf0av{=%{vILP*OB^G<;$0%jbdZzS*CxR3m{dNog(0a4Ej)SS>gfo2BRe;LzFq zDaNU_QWBoCKLc-DO6qFc43zU9dLOTNdO&em<#A|JBy{Q8+1H1GYM1fsX+3eNR1R(2 z)y9Ec3^SSwWZ{F~A|3@(9|f{KdIrWf(kD}6KVJ={Np{(AtFNzLbzMVB4$ZC3>YwUB z>DdK+s>AVzSIxRJQIp$#C~t{oiC!zGhTN)F(S*3N3sDp22n<}5vkkKs7nb7U5k7lw zbLK6=b;k-8wa2nb`=uvL_+!~?FP28K+-Iz;bSy0VxosrA zo|itCdGoo*xMf)YTySdLaU-*+HP*;d^xEOb6wge7eL0i-96%~?Qv%UEY7Xl#IEiSq z=vp0w+(YXz?xd8_Kl2~OQa9^=Y3>3K7qC%)z8BGemYnQuTW_C56x}A-rqsm|W0k2- zhDK=-%_A?0QP?j(Iybxk7Jx-8h-tx!)Xzp{P_PIuojrGiA z{|+>_u#j;-0R~y;MrYntRyFnas68QP7dnI)$r|Y~TDTf~?)YM;sr^K-2tX+~XPV1@ zHV=AeD!&9U2O$;Et8asjK^KUk*7IQh*N&C^fAmoNaKQ(tcb%Ng&5YQr;t75$Clf*} zu?;NMv3al;CJ zEtj8{SxNcS_TnI&E68er(J1Q)0H?}?#Ut&f8*PS1QHPwH? zsR{P;kBczV?D!Lf$X`oN6b$@&;=mgUbB#F+Dk(@88?G{yWeqGb2;zsl6-6=Vb3u5nuf18IHfM4)B7G5a5Qs{sF&U#1A&^-_WDES zk5)wnC4LgcYwnU%A5(hoowjw~vaPiczhSz!yR@IZE=X(x|4-!DfW54RHa+YO>o_`#_ z%U45{D+GYN)#4Y3ls4SkXvLTHDoGw^VISDi5^q(w#~NE($ZEWmE~IG$j+A>9!N0Cd zAP$Fp^hnwI3* zX0mJ?0EUXX7I3(rZ{Lu14Wfq0H?-IH-TGm?Qt@ZZkn-Hms%as+^#^}QgQ&UN=DAYa zb|BG=_ULEeBam_Ihy7CF4#~jmCQICI7-n4DvH;=q1VMg`O)qOIv&wUJ`%!duGfq5Q z^L72rpGIbtRRnKU->=Z)_wj9B27Dzxwo237BWc5NY#)K{xO#jQWZn@tcAC>rTWT>_ z1fvCAnkg#v%zSy|oK!e3Q&?V-E^>V;yY*H7Lhtar&Q7v$6AWHukOnMRB-2S(+>Sb{ zd`}TRI3QN?e96Ztm)pqUnty|6YQ2FfDv;@~q5(y!>(c6&C0Q4vtkSEn1DxSC)8((j z&G=+q=a#f?H_(sI6m`o~eF~nF;FZ6l$`l)((#Ov)IfymTvPd*HjGfBXuaPY=xT(qK zI;>TLEpshVu{7iD+WgZ|<@b^_A#Hoo6h!7W?@S|4F@oA6lY>qN1ytNqVNTFR;~?OIQ2czDElj9x5U=M4?~R{X1{57jb# zS=H7w8mCqv?HPJxxHH8&rO$wIZ8cx8&%_L18l_aVo1CuPof+0Bn<`6LlQmvaJs=&Z zUhJuu zP_NtTvo57j1!@@jp8M#=-#&GWzL1lgJq!Ar{^e}5FHtMfN0g0>{JR!_!THY&?R;)x zxYXpcI1%g?$^vJg<-O=;@I*w*b=8CGea#i>`?+%{J};U0-z<~wWT4Z>j$a5*JOe7I(ZwyVbV8Xns!=KxB=gw#>_nSW^2c~HHx;NY&?Z?!tX>a7e5 z|3+8VkTk}H-GCazZL{*dp@*V5By_aYkQ5_H6c&*O&z}k1xzo9>#ku&9`8R4WQ$1=-@CQ=_Pm`*dt9t+xZBzRB&)u5f`cs8QX^;+STJFJ z%)>x$Irmw~A|E@-*v+=J?orsu_}j6+oIwzrE9kTABV5bLTk4O?_S?eHLTx^z>YjW8rW*}CTX5~-zWy8XBsGS54P3+xdF8 z`iA7V585WgmuXc+hKJ7Fd1|Vacu$OR7DV>d?xxMglnmPQ>)O_*&24eifb*l_st&I~ zArp0U-Sa;}CiSc!M4HVXDY+jO%wxS4f%3=)6S9e^OzQ)_#*HmzC(YQFAg`XTH^(mQ zEHkd@q~_5MsV?*CS@~n^p4sC-{;7HDCav!t?)qZ6;dKW=2F;h(4l}Qk0W%7@LO(x% zBm4sv=0|1ME%02%D=PkFj;WHW0Y;^erSLFZ8!}jcA2Hk~u z5Zpsnk{+!yQU{d4*yp6Ql5%IarAR_Crh%eS(ip*(pP7DK6aD5qO$#aNde@A_6(CSv zp5lSd7S1FvHu-~^7=OwnV94an8TWgeW6s@zcf!6EIV$+p`BS=eAq_;w7cG`6in-jp zxe-Uz1?j~#zPMxWd)aOq62Oe{p2?h^_DE+%h(q}_ms{k26Vor4{2f3ks(4HeT>p=% z`2PWMQv#Gs`;XvgQjk{-I=p?t$plb^jhxNQMJa2hH}?kLy7`_N0gjmw z?kWSt+u&s`*a2d7{&buh;2*(wSyB-75yUcmrRftWjgM|{)$(8J z;qJr`dM0=T9S0aXvXO^@O!0uyUhbu|h6w`GhGvIEIDt_d>N0a2_>X@sRyNTb8W3@Z ziDqiOI86e0FYZyI+YF5;In&fdFF=AeU+_ZylWPfIA422C4z)z-m@3NkUVj9#XKF1@ zm*}HNQa=Pf9+3R+djS-Yu`uhsvEv)}DW-JjZjJX)*&}g)P)Y-8R-3n-P)!*VAk`w=VJ7cBXvb%z!;0*? z2+#NGOQ$~?Isl889P^yB)q}g0SFIz$k-04Ve+Yi=5KL3>MHsX7Zhh?;naCfSRB% z&vv%gSSK}R9X!AE(J0%H@Y1!f>8nL(M4;jkBC`z#8nbuP zU`W=nt3YvAU*8>A^(^MR1AHbSWnMQhXY&wZ^gc=q+`kZG)4&n|G6vkMh7DQkd293FFS3t9(wz)B_w;VEH z;2IQ#8!rV&xsDvQ6VfuQiJ$U2?`OH+2?p&9P`r^ZH&cG%)!!IXuEtnW0lw7$IIEyj zQn~b57|KV}+;L`3&^>;0kDXtG2m$ -This repository holds the 51Degrees Java packages. Each package is still -published to Maven Central under its own coordinates, and the examples show -how a project of your own uses them. +**Java packages for the Pipeline API, Device Detection, IP Intelligence and Location** + +Our User-Agent parser for Java supports Device Detection, browser detection, mobile detection, and crawler detection on-premise or in the cloud. IP Intelligence adds IP geolocation, including IP to country data. + +It reads User-Agent and User-Agent Client Hints, HTTP request headers that carry browser and device information. For cloud bot detection, Human Confidence reports confidence that a person is viewing the page. + +## Install + +Each product is a separate Maven Central package. For Device Detection, cloud or on-premise: + +```xml + + + com.51degrees + device-detection + 4.5.x + +``` + +The product guides below list the other packages. To build from source, see [Getting the code](#getting-the-code). + +Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). + +Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, `location-java`, `device-detection-java-examples`, and `ip-intelligence-java-examples`, this repository continues their work. | Product | Read first | Documentation | | --- | --- | --- | @@ -67,9 +88,9 @@ mvn install -DskipTests -pl device-detection -am mvn surefire:test ``` -The cloud tests need a resource key, passed as `-DTestResourceKey=`. The +The cloud tests need a Resource Key, passed as `-DTestResourceKey=`. The on-premise IP Intelligence tests need the Lite or Enterprise data file, which -is not in the repository; see `ip-intelligence/README.md` for how to get it. +is not in the repository. See `ip-intelligence/README.md` for how to get it. Tests find data files by searching from the root of this repository, so run Maven from here rather than from inside a package folder. @@ -106,6 +127,16 @@ with the plain version, such as `4.5.127`. ## History +| Earlier repository | What it held | +| --- | --- | +| [pipeline-java](https://github.com/51Degrees/pipeline-java) | The Pipeline API | +| [device-detection-java](https://github.com/51Degrees/device-detection-java) | Device Detection | +| [ip-intelligence-java](https://github.com/51Degrees/ip-intelligence-java) | IP Intelligence | +| [location-java](https://github.com/51Degrees/location-java) | Location | +| [device-detection-java-examples](https://github.com/51Degrees/device-detection-java-examples) | Device Detection examples | +| [ip-intelligence-java-examples](https://github.com/51Degrees/ip-intelligence-java-examples) | IP Intelligence examples | + + The folders `pipeline`, `device-detection`, `ip-intelligence`, `location` and the two example folders were brought in from the separate repositories [pipeline-java](https://github.com/51Degrees/pipeline-java), @@ -120,6 +151,14 @@ SHA. The history of a file before the import is under its old path, so read it from the imported commit, for example `git log aff7c544 -- pom.xml` for what is now `pipeline/pom.xml`. -## Licence +## License [European Union Public Licence v1.2](LICENSE) + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- [Bots and humans](https://51degrees.com/bots-and-humans?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- [Documentation](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- https://github.com/51Degrees/51degrees-cxx From 99217cd6dfd9b27ad88f0dbc63e88838936ea1d2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 18:36:58 +0000 Subject: [PATCH 02/95] Check that only rc merges into the default branch (#29) --- .github/workflows/only-rc-into-main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/only-rc-into-main.yml diff --git a/.github/workflows/only-rc-into-main.yml b/.github/workflows/only-rc-into-main.yml new file mode 100644 index 000000000..69e19ca3a --- /dev/null +++ b/.github/workflows/only-rc-into-main.yml @@ -0,0 +1,11 @@ +name: Only rc into main + +# A pull request into the default branch must come from rc. See +# 51Degrees/common-ci .github/workflows/only-rc-into-main.yml. +on: + pull_request: + +jobs: + rc: + name: Only rc into main + uses: 51Degrees/common-ci/.github/workflows/only-rc-into-main.yml@main From 9ffbcf87e4830e7bea685dda4449b60e84d80ae2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 19:53:17 +0000 Subject: [PATCH 03/95] Point links to the archived Java repositories at their folders here (#31) pipeline-java, device-detection-java, ip-intelligence-java, location-java, device-detection-java-examples and ip-intelligence-java-examples are archived, and the note at the top of each README names the folder of this repository that now holds its code. Four READMEs and 33 example source files still linked those repositories, so each of the 39 links now points at the same file, folder or README heading on main here. Only the URLs change. The History section of the root README, two Java string constants, the creator context demo page and its test, and the four parent poms that the open package text pull request (#27) changes keep their links. --- device-detection/README.md | 4 ++-- examples/device-detection/README.md | 2 +- .../devicedetection/examples/console/GettingStartedCloud.java | 2 +- .../examples/console/GettingStartedOnPrem.java | 2 +- .../devicedetection/examples/console/MatchMetrics.java | 2 +- .../devicedetection/examples/console/MetadataCloud.java | 2 +- .../devicedetection/examples/console/MetadataOnPrem.java | 2 +- .../devicedetection/examples/console/NativeModelCloud.java | 2 +- .../devicedetection/examples/console/OfflineProcessing.java | 2 +- .../examples/console/PerformanceBenchmark.java | 2 +- .../fiftyone/devicedetection/examples/console/TacCloud.java | 2 +- .../devicedetection/examples/console/UpdateDataFile.java | 2 +- .../devicedetection/examples/web/GettingStartedWebCloud.java | 2 +- .../devicedetection/examples/web/GettingStartedWebOnPrem.java | 2 +- .../ipintelligence/examples/console/CompareOnPrem.java | 2 +- .../examples/console/GetAllPropertiesCloud.java | 2 +- .../ipintelligence/examples/console/GettingStartedCloud.java | 2 +- .../examples/console/GettingStartedCloudMixed.java | 2 +- .../ipintelligence/examples/console/GettingStartedMixed.java | 2 +- .../ipintelligence/examples/console/GettingStartedOnPrem.java | 2 +- .../ipintelligence/examples/console/MetadataCloud.java | 2 +- .../ipintelligence/examples/console/MetadataOnPrem.java | 2 +- .../ipintelligence/examples/console/MetricsOnPrem.java | 2 +- .../ipintelligence/examples/console/OfflineProcessing.java | 2 +- .../ipintelligence/examples/console/PerformanceBenchmark.java | 2 +- .../ipintelligence/examples/console/SuspiciousOnPrem.java | 2 +- .../ipintelligence/examples/console/UpdateDataFile.java | 2 +- .../ipintelligence/examples/web/GettingStartedApi.java | 2 +- .../examples/web/GettingStartedWebCloudMixed.java | 2 +- .../ipintelligence/examples/web/GettingStartedWebCloud.java | 2 +- .../ipintelligence/examples/web/GettingStartedWebMixed.java | 2 +- .../ipintelligence/examples/web/GettingStartedWebOnPrem.java | 2 +- ip-intelligence/README.md | 2 +- .../java/fiftyone/geolocation/examples/CombiningServices.java | 2 +- .../fiftyone/geolocation/examples/cloud/GettingStarted.java | 2 +- pipeline/README.md | 4 ++-- .../java/pipeline/developerexamples/usagesharing/Main.java | 2 +- 37 files changed, 39 insertions(+), 39 deletions(-) diff --git a/device-detection/README.md b/device-detection/README.md index 575ed61c9..c393a561e 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -212,14 +212,14 @@ or shaded. The names come from an `Automatic-Module-Name` manifest entry, so the jars remain ordinary non-modular jars, build identically on every JDK from 8 upwards, and nothing changes for consumers who stay on the classpath. See the -[pipeline-java README](https://github.com/51Degrees/pipeline-java#native-library-access) +[pipeline-java README](https://github.com/51Degrees/51degrees-java/blob/main/pipeline/README.md#native-library-access) for the pipeline module names and for how to configure element builders when a jar is on the module path. ## Examples Examples can be found in -[device-detection-java-examples](https://github.com/51Degrees/device-detection-java-examples) +[device-detection-java-examples](https://github.com/51Degrees/51degrees-java/tree/main/examples/device-detection) repository. ## Tests diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 8d38975ed..5d2edd339 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -150,4 +150,4 @@ A fat jar cannot narrow this any further, because everything inside it is part o unnamed module. To grant native access to 51Degrees code alone you need the individual jars, with `pipeline.engines.fiftyone` and `device-detection.hash.engine.on-premise` on the module path - see the -[device-detection-java README](https://github.com/51Degrees/device-detection-java#native-library-access). +[device-detection-java README](https://github.com/51Degrees/51degrees-java/blob/main/device-detection/README.md#native-library-access). diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java index 573ea86b4..c97a1868a 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java @@ -158,7 +158,7 @@ private static void analyzeEvidence(Map evidence, * * @include{doc} example-getting-started-cloud.txt * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/main/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java). * * @include{doc} example-require-resourcekey.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 113be2a40..9d8819565 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -184,6 +184,6 @@ private static void analyzeEvidence(Map evidence, * @example console/GettingStartedOnPrem.java * @include{doc} example-getting-started-onpremise.txt *

- * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java). * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index 26a305721..a54bfe71c 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -235,7 +235,7 @@ static void run(String dataFile, List> evidenceList, * [performance options](/documentation/4.5/_device_detection__features__performance_options.html) * page. * # Location - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java). * */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java index aaebc3151..8ca360fab 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java @@ -147,7 +147,7 @@ private static void outputProperties(DeviceDetectionCloudEngine engine, PrintWri * keys that, when added to the evidence collection in flow data, could have some impact on the * result returned by device detection. * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java). * * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 677b322e4..a58e739ab 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -231,7 +231,7 @@ private static String truncateToNl(String text) { * keys that, when added to the evidence collection in flow data, could have some impact on the * result returned by device detection. * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/main/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java). * * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index 0a5f472b0..e920f8c9d 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -133,5 +133,5 @@ static void analyseNativeModel(String nativemodel, Pipeline pipeline, PrintWrite * pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). Once this is done, a resource key with the * properties required by this example can be created at [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java). */ \ No newline at end of file diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index d044ae365..031c63368 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -262,7 +262,7 @@ private static Map getPopulatedProperties(DeviceData data, Strin * @example console/OfflineProcessing.java * @include{doc} example-offline-processing-hash.txt *

- * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java). * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index 72172d4e2..aeaf3180c 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -403,7 +403,7 @@ public PerformanceConfiguration(Constants.PerformanceProfiles profile, * and [hash dataset options](/documentation/4.5/_device_detection__hash.html#DeviceDetection_Hash_DataSetProduction_Performance) * for more information about adjusting performance. * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java). * * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index 0db97d885..91213032b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -138,7 +138,7 @@ static void analyseTac(String tac, Pipeline pipeline, PrintWriter output) throws * [pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). Once this is done, a resource key with the * properties required by this example can be created [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java). * * @include{doc} example-require-resourcekey.txt */ \ No newline at end of file diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index 54c34bf39..6dae1a226 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -162,7 +162,7 @@ .setUpdateRandomisationMax(10*60) * ``` * # Location - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java/blob/master/device-detection-java-examples/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java). * * @include{doc} example-require-licensekey.txt */ diff --git a/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java b/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java index 83c9c0c31..53046d3b9 100644 --- a/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java +++ b/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java @@ -133,7 +133,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { * * @include{doc} example-getting-started-web.txt * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java). * * @include{doc} example-require-resourcekey.txt * diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index c9a780e9b..22b31c746 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -145,7 +145,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { * * @include{doc} example-getting-started-web.txt * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java). * * @include{doc} example-require-datafile.txt * diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index c411b9a9d..06538b059 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -54,7 +54,7 @@ * The output CSV file contains the input truth and the result fields for easy * evaluation. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=contact-us). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java index a21d88e83..3a55152aa 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java @@ -31,7 +31,7 @@ * 2. How to process an IP address and retrieve all available properties * 3. How to enumerate through all the properties returned by the service * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java index ed23f80b5..fb63e8546 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java @@ -31,7 +31,7 @@ * 2. How to pass input data (evidence) to the Pipeline * 3. How to retrieve the results * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java index a5cbe3c67..e5bf11069 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java @@ -31,7 +31,7 @@ * 2. How to pass input data (User-Agent and IP address) to the Pipeline * 3. How to retrieve device and IP intelligence results from a single pipeline * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 8014fc2bf..9bab40ab5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -33,7 +33,7 @@ * 2. How to pass combined evidence (User-Agent and IP address) to the Pipeline * 3. How to retrieve results from both engines * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java). * * This example requires: * - A Device Detection data file (.hash format) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 1637f76c8..8c6e4d0b9 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -31,7 +31,7 @@ * 2. How to pass input data (evidence) to the Pipeline * 3. How to retrieve the results * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=header). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java index 15a1af9a4..00da31c62 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java @@ -39,7 +39,7 @@ * cloud. If you are only using a single product (for example, IP intelligence) then not all * of these keys will be relevant. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 4b3224857..7a96b562f 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -50,7 +50,7 @@ * The example will output the available properties along with details about their data types and descriptions. * This helps you understand what IP Intelligence data you can access for your use case. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 0c81a3b34..aaf6ddac4 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -38,7 +38,7 @@ * verifying the published metrics associated with 51Degrees' * IP intelligence service. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=contact-us). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 354ca5387..983d25997 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -42,7 +42,7 @@ * * Evidence files can be obtained from the [ip-intelligence-data repository](https://github.com/51Degrees/ip-intelligence-data). * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=header). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index a4180e6f2..2be3b4599 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -32,7 +32,7 @@ * This can help you optimize your IP Intelligence configuration for your specific use case * and understand the performance characteristics of different settings. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index 8e6ee75b5..dd46f9cc5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -33,7 +33,7 @@ * 2. How to combine diversity values with other network related values to * assess the likelihood of an IP address being something suspicious. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index 88ee407cd..144931e2c 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -151,7 +151,7 @@ .setUpdateRandomisationMax(10*60) * ``` * # Location - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java). * * To test this example, you need to: * 1. Host an IP Intelligence data file (.ipi) at a custom URL accessible to this application diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 588976708..2de381969 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -38,7 +38,7 @@ * A cloud example, such as GettingStartedCloud, can be pointed at this server as a * custom endpoint. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java index 279133750..4090fc110 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java @@ -32,7 +32,7 @@ * 2. How evidence from the web request is automatically passed to both engines * 3. How to retrieve results from both engines in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java index eb2d74e97..dbcb50e7f 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java @@ -31,7 +31,7 @@ * 2. How evidence from the web request is automatically passed to the Pipeline * 3. How to retrieve the results in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 6be4130a2..9969e3262 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -32,7 +32,7 @@ * 2. How evidence from the web request is automatically passed to both engines * 3. How to retrieve results from both engines in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java). * * This example requires: * - A Device Detection data file (.hash format) diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index a5595d706..d73bafdf8 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -31,7 +31,7 @@ * 2. How evidence from the web request is automatically passed to the Pipeline * 3. How to retrieve the results in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/master/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=header). diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index a3d470e8e..2b087fe20 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -104,7 +104,7 @@ This is not recommended unless absolutely necessary and should be used with caut ## Examples Examples can be found in -[ip-intelligence-java-examples](https://github.com/51Degrees/ip-intelligence-java-examples) +[ip-intelligence-java-examples](https://github.com/51Degrees/51degrees-java/tree/main/examples/ip-intelligence) repository. ## Tests diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java index 999ba8892..748499f05 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java @@ -53,7 +53,7 @@ * Getting started example of using 51Degrees geo-location alongside 51Degrees * device detection. * - * This example is available in full on [GitHub](https://github.com/51Degrees/location-java/blob/master/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java). * * Required Maven Dependencies: * - [com.51degrees.geo-location](https://central.sonatype.com/artifact/com.51degrees/geo-location/) diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java index be73df1ef..553517c6a 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java @@ -42,7 +42,7 @@ * * Getting started example of using cloud geo-location. * - * This example is available in full on [GitHub](https://github.com/51Degrees/location-java/blob/master/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java). * * Required Maven Dependencies: * - [com.51degrees.geo-location](https://central.sonatype.com/artifact/com.51degrees/geo-location/) diff --git a/pipeline/README.md b/pipeline/README.md index 718333c20..a860e40c0 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -94,7 +94,7 @@ consumers who stay on the classpath. Device detection users need to grant access to a second module as well. The JNI `native` declarations live in `device-detection.hash.engine.on-premise`, and JEP 472 checks binding them against the module that declares them - see the -[device-detection-java README](https://github.com/51Degrees/device-detection-java#native-library-access). +[device-detection-java README](https://github.com/51Degrees/51degrees-java/blob/main/device-detection/README.md#native-library-access). ### Configuring builders from the module path @@ -122,7 +122,7 @@ Most packages contain tests which use junit and mockito. These tests can be run There are several examples available in the `pipeline.developer-examples` folder that demonstrate how to make use of the Pipeline API in isolation. These are described in the table below. If you want examples that demonstrate how to use 51Degrees products such as device detection, -then these are available in the corresponding [repository](https://github.com/51Degrees/device-detection-java) +then these are available in the corresponding [repository](https://github.com/51Degrees/51degrees-java/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=examples). | Example | Description | diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java index 06c4f4ce4..407759b4d 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java @@ -58,7 +58,7 @@ * For details of what each setting does, see the * [Share Usage reference](/documentation/4.5/_pipeline_api__features__usage_sharing.html) * - * This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-java/blob/main/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java). * * Expected output: * ``` From 5ac0cd496356e65eb6a6c2ba4490dd72a8d252cc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:00:22 +0000 Subject: [PATCH 04/95] Drop the moved example folders and fix the native model example name (#33) The examples moved to examples/, which the device detection and IP Intelligence Doxyfiles already read, so the old folders go. Point the location IMAGE_PATH at the shared documentation images, and name the native model example console/NativeModelCloud.java in place of the doubled example command. --- device-detection/docs/Doxyfile | 5 +---- .../devicedetection/examples/console/NativeModelCloud.java | 2 +- ip-intelligence/docs/Doxyfile | 1 - location/docs/Doxyfile | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/device-detection/docs/Doxyfile b/device-detection/docs/Doxyfile index 2c34c2ac9..0cbff9b26 100644 --- a/device-detection/docs/Doxyfile +++ b/device-detection/docs/Doxyfile @@ -820,7 +820,6 @@ WARN_LOGFILE = INPUT = ../device-detection/src/main \ ../device-detection.cloud/src/main \ - ../device-detection.shell.examples/src/main \ ../device-detection.hash.engine.on-premise/src/main \ ../device-detection.shared/src/main \ ../../examples/device-detection \ @@ -897,9 +896,7 @@ EXCLUDE_SYMBOLS = # that contain example code fragments that are included (see the \include # command). -EXAMPLE_PATH = ../device-detection.shell.examples \ - ../device-detection.web.examples \ - ../../examples/device-detection \ +EXAMPLE_PATH = ../../examples/device-detection \ ../../../documentation/src/snippets # If the value of the EXAMPLE_PATH tag contains directories, you can use the diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index e920f8c9d..754f0596a 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -119,7 +119,7 @@ static void analyseNativeModel(String nativemodel, Pipeline pipeline, PrintWrite } } /*! - * @example @example console/NativeModel.java + * @example console/NativeModelCloud.java * * This example shows how to use the 51Degrees Cloud service to look up the details of a device * based on a given 'native model name'. Native model name is a string of characters that are diff --git a/ip-intelligence/docs/Doxyfile b/ip-intelligence/docs/Doxyfile index 6b9d53dae..836e6d2b9 100644 --- a/ip-intelligence/docs/Doxyfile +++ b/ip-intelligence/docs/Doxyfile @@ -820,7 +820,6 @@ WARN_LOGFILE = INPUT = ../ip-intelligence/src/main \ ../ip-intelligence.cloud/src/main \ - ../ip-intelligence.shell.examples/src/main \ ../ip-intelligence.engine.on-premise/src/main \ ../ip-intelligence.shared/src/main \ ../../examples/ip-intelligence \ diff --git a/location/docs/Doxyfile b/location/docs/Doxyfile index f6c4b5094..953e37e3b 100644 --- a/location/docs/Doxyfile +++ b/location/docs/Doxyfile @@ -917,7 +917,7 @@ EXAMPLE_RECURSIVE = YES # that contain images that are to be included in the documentation (see the # \image command). -IMAGE_PATH = images +IMAGE_PATH = ../../../documentation/docs/images # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program From edbf8fcffa3564e0eae1e2b517d15f862aca90e6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 22:30:05 +0000 Subject: [PATCH 05/95] Link this repository from the native library message and the demo page (#32) When the native library fails to load, both on-premise engines suggest rebuilding the binary from the source code and instructions in device-detection-java or ip-intelligence-java, which are archived. The instructions for building the native binary are now in the README of device-detection/ and ip-intelligence/ here, beside the engine source, so the message links those folders. The creator context demo page linked pipeline-java as the source of the demo and of the code it runs on, and now links pipeline/ here. The test that checks the page ends with somewhere to go next asserts the new address. --- .../hash/engine/onpremise/interop/Constants.java | 2 +- .../ipintelligence/engine/onpremise/interop/Constants.java | 2 +- .../src/main/resources/fodid/creator-context/page.html | 2 +- .../developerexamples/fodid/CreatorContextPageTests.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java index af5116e58..1fa22d3ef 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java @@ -29,7 +29,7 @@ public class Constants { "possible, you may be able to use a different library by " + "rebuilding the binary on the target system using the source " + "code and instructions available on " + - "https://github.com/51degrees/device-detection-java."; + "https://github.com/51Degrees/51degrees-java/tree/main/device-detection."; /** * Message returned when the native library could not be loaded. diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java index a51211766..7c09eb920 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java @@ -29,7 +29,7 @@ public class Constants { "possible, you may be able to use a different library by " + "rebuilding the binary on the target system using the source " + "code and instructions available on " + - "https://github.com/51degrees/ip-intelligence-java."; + "https://github.com/51Degrees/51degrees-java/tree/main/ip-intelligence."; /** * Message returned when the native library could not be loaded. diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html index 0690d416a..41821e1ed 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html @@ -147,7 +147,7 @@

Find out more

The source of this demo and of the code it runs on.

diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java index 9459f60b3..61d32d13c 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java @@ -303,7 +303,7 @@ public void The_Page_Ends_With_Find_Out_More() throws Exception { String page = pageText(); assertTrue(page.contains("Find out more"), "the page says where next"); assertTrue(page.contains("utm_campaign=51degrees-java"), "campaign tag"); - assertTrue(page.contains("https://github.com/51Degrees/pipeline-java"), + assertTrue(page.contains("https://github.com/51Degrees/51degrees-java/tree/main/pipeline"), "the repository is linked"); } } From 6b2a9dea5276757a7fff40efd1368c4b451d1218 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 23:33:58 +0000 Subject: [PATCH 06/95] Link the hardware profile engine to its specification page (#34) --- .../cloud/flowelements/HardwareProfileCloudEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java index bf288c8ee..ed7910e5a 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java @@ -35,7 +35,7 @@ * Engine that takes the JSON response from the {@link CloudRequestEngine} and * uses it to populate a {@link MultiIPIDataCloud} instance for easier * consumption. - * @see Specification + * @see Specification */ public class HardwareProfileCloudEngine extends PropertyKeyedCloudEngineBase { From ebb595b78b3c4ddd6fb9436348586b4235499ff4 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:31:27 +0100 Subject: [PATCH 07/95] Merged Pull Request '#36 update-packages/rc->rc: Update packages' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- examples/device-detection/pom.xml | 2 +- examples/ip-intelligence/pom.xml | 4 ++-- examples/ip-intelligence/web/getting-started.api/pom.xml | 2 +- location/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index 47e7b11a2..d7617e6e6 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -50,7 +50,7 @@ 4.13.2 true 1.30 - 4.5.126 + 4.5.128 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 9653a86aa..945f34223 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -50,8 +50,8 @@ 4.13.2 true 1.30 - 4.5.96 - 4.5.126 + 4.5.128 + 4.5.128 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/web/getting-started.api/pom.xml b/examples/ip-intelligence/web/getting-started.api/pom.xml index ce2f2b0b1..5cb1db7d1 100644 --- a/examples/ip-intelligence/web/getting-started.api/pom.xml +++ b/examples/ip-intelligence/web/getting-started.api/pom.xml @@ -37,7 +37,7 @@ 2.0.20 1.3.16 - 4.5.48 + 4.5.128 20231013 diff --git a/location/pom.xml b/location/pom.xml index ac0e576ed..e3e332f3f 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -60,7 +60,7 @@ 1.8 ${project.version} - ${project.version} + 4.5.128 4.13.2 1.7.36 From 282fc0b25b677faf73eb9453861bffec449a432e Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:18:03 +0100 Subject: [PATCH 08/95] Merged Pull Request '#35 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cxx b/cxx index e0a6561f5..1bec7b570 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit e0a6561f521476151fbd4ba4710b94edf3c0c0b5 +Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 From 0da326d6bc307cf070952f90a7e3768ce1e73d8a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:07:30 +0000 Subject: [PATCH 09/95] Use the data repository for the examples, tests and CI (#37) --- AGENTS.md | 11 +- README.md | 25 +++- ci/README.md | 8 +- ci/fetch-assets.ps1 | 16 ++- cxx | 2 +- device-detection/README.md | 19 +-- .../devicedetection/cloud/TestsBase.java | 4 +- .../hash/engine/onpremise/TestsBase.java | 4 +- .../onpremise/flowelements/EngineTests.java | 6 +- .../shared/testhelpers/FileUtils.java | 12 +- .../devicedetection/DeviceDetectionTests.java | 4 +- examples/device-detection/README.md | 11 +- .../console/GettingStartedOnPrem.java | 10 +- .../examples/console/MatchMetrics.java | 3 +- .../examples/console/MetadataOnPrem.java | 10 +- .../examples/console/OfflineProcessing.java | 17 +-- .../console/PerformanceBenchmark.java | 5 +- .../examples/console/ShareUsageExample.java | 8 +- .../examples/console/UpdateDataFile.java | 6 +- .../main/resources/gettingStartedOnPrem.xml | 2 +- .../console/GettingStartedOnPremTest.java | 2 +- .../examples/console/MetadataOnPremTest.java | 2 +- .../console/OfflineProcessingTest.java | 2 +- .../examples/shared/DataFileHelper.java | 111 +++++++++++++-- .../examples/web/GettingStartedWebOnPrem.java | 15 +- .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 2 +- .../web/GettingStartedWebOnPremTest.java | 2 +- .../web/OnPremDataFileOptionTest.java | 2 +- examples/ip-intelligence/.gitignore | 2 - examples/ip-intelligence/README.md | 44 +++--- .../examples/console/CompareOnPrem.java | 4 +- .../examples/console/GettingStartedMixed.java | 9 +- .../examples/console/OfflineProcessing.java | 6 +- .../main/resources/gettingStartedMixed.xml | 4 +- .../main/resources/gettingStartedOnPrem.xml | 2 +- .../examples/shared/DataFileHelper.java | 128 +++++++++++++++--- .../examples/web/GettingStartedApi.java | 2 +- .../examples/web/GettingStartedWebMixed.java | 9 +- .../main/webapp/WEB-INF/51Degrees-Mixed.xml | 4 +- .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 2 +- .../examples/web/HtmlContentHelper.java | 4 +- ip-intelligence/README.md | 7 +- .../ip-intelligence.translation/pom.xml | 12 +- 43 files changed, 412 insertions(+), 148 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d163ccbfe..5a1d6f620 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,9 +22,18 @@ Tests and examples find data files with `FileFinder`, which searches below `project.root`. The surefire configuration sets that to `maven.multiModuleProjectDirectory`, which Maven takes from the nearest `.mvn` folder above the working directory. The only `.mvn` folder is at the root, so -the search covers `cxx/device-detection-data` and `cxx/ip-intelligence-data` +the search covers `cxx/data/device-detection` and `cxx/data/ip-intelligence` from wherever Maven is run. Do not add a `.mvn` folder below the root. +Those folders are the data repository, a submodule of cxx, where each free +data file is a `.link` until `get-data-files.ps1` or `get-data-files.sh` +downloads it. `ci/fetch-assets.ps1` runs the script, and the examples' +`DataFileHelper.findDataFile` runs it for a file that is missing. Look for a +free file by its folder and name, such as +`device-detection/51Degrees-LiteV4.1.hash`, so the search does not stop at a +file of the same name elsewhere, such as a Git LFS pointer in an old +`cxx/device-detection-data` folder. + ## Native builds Each on-premise engine compiles its own copy of the common C code from `cxx`, diff --git a/README.md b/README.md index 4ba31cf22..479f1ff36 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, ` | `location` | The Location packages, `com.51degrees:geo-location*` | | `examples/device-detection` | Console and web examples for Device Detection | | `examples/ip-intelligence` | Console and web examples for IP Intelligence, including the mixed examples that run both products | -| `cxx` | Submodule of [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx), with the C and C++ code the on-premise engines are built from and the data file submodules | +| `cxx` | Submodule of [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx), with the C and C++ code the on-premise engines are built from and the [data](https://github.com/51Degrees/data) submodule in `cxx/data` | | `pom.xml` | The reactor that builds everything here. It is never published | | `ci` | The scripts the shared 51Degrees CI workflows call | @@ -55,10 +55,20 @@ git clone --recurse-submodules https://github.com/51Degrees/51degrees-java.git cd 51degrees-java ``` -The free Device Detection Lite data file in `cxx/device-detection-data` is -stored with [Git LFS](https://git-lfs.github.com/). Run `git lfs pull` in that -folder to fetch it. The IP Intelligence ASN file in `cxx/ip-intelligence-data` -is plain git. +The free data files come from the [data](https://github.com/51Degrees/data) +repository, which is the `cxx/data` submodule. Each one is listed there as a +`.link` file, and a script downloads them all into place: + +``` +cxx/data/get-data-files.sh +``` + +On Windows, run `cxx\data\get-data-files.ps1` in PowerShell instead. The +examples run the script themselves for a free file they need that is missing. + +In a clone made before the data repository was added, delete the +`cxx/device-detection-data` and `cxx/ip-intelligence-data` folders once cxx no +longer includes them, because Git leaves a removed submodule's folder on disk. ## Building @@ -89,8 +99,9 @@ mvn surefire:test ``` The cloud tests need a Resource Key, passed as `-DTestResourceKey=`. The -on-premise IP Intelligence tests need the Lite or Enterprise data file, which -is not in the repository. See `ip-intelligence/README.md` for how to get it. +on-premise tests need the free data files, which the script in +[Getting the code](#getting-the-code) downloads. For the Enterprise IP +Intelligence data file, see `ip-intelligence/README.md`. Tests find data files by searching from the root of this repository, so run Maven from here rather than from inside a package folder. diff --git a/ci/README.md b/ci/README.md index 1041809a8..a537b0271 100644 --- a/ci/README.md +++ b/ci/README.md @@ -21,8 +21,12 @@ repositories used to do for one package tree each. workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). - [fetch-assets.ps1](fetch-assets.ps1) links every data file it downloads into - `cxx/device-detection-data` or `cxx/ip-intelligence-data`, where the tests - and examples find them. + `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests + and examples find them. Those are the folders of the + [data](https://github.com/51Degrees/data) repository, which is a submodule + of cxx. The script then runs the data repository's `get-data-files.ps1`, + which keeps the linked files and downloads the free files that are not + assets, which is the Asn file. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 23f509ea1..4062061b1 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -7,9 +7,11 @@ param ( $ErrorActionPreference = "Stop" # Tests and examples find data files by searching below the repository root, -# so every asset is linked into the data folders of the cxx submodule. -$deviceDetectionData = "$PSScriptRoot/../cxx/device-detection-data" -$ipIntelligenceData = "$PSScriptRoot/../cxx/ip-intelligence-data" +# so every asset is linked into the folders of the data repository, which is +# the data submodule of cxx. +$data = "$PSScriptRoot/../cxx/data" +$deviceDetectionData = "$data/device-detection" +$ipIntelligenceData = "$data/ip-intelligence" $deviceDetectionAssets = 'TAC-HashV41.hash', '51Degrees-LiteV4.1.hash', '20000 Evidence Records.yml', '20000 User Agents.csv' ./steps/fetch-assets.ps1 -DeviceDetection:$DeviceDetection -DeviceDetectionUrl:$DeviceDetectionUrl -Assets $deviceDetectionAssets @@ -24,6 +26,14 @@ New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-Enterprise # link it as that. New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$ipIntelligenceData/51Degrees-LiteV41.ipi" +# The data repository's script downloads every linked file that is not +# already there. The files linked above are kept, so this gets the free files +# that are not assets, which is the Asn file. +& "$data/get-data-files.ps1" +if (-not $?) { + throw "The data repository's get-data-files.ps1 could not download every file" +} + Write-Host "Assets hashes:" Get-FileHash -Algorithm MD5 -Path assets/* diff --git a/cxx b/cxx index 1bec7b570..e866f6d3c 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 +Subproject commit e866f6d3cea936587790f87967eff7863778b30e diff --git a/device-detection/README.md b/device-detection/README.md index c393a561e..73722fcb5 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -28,23 +28,24 @@ how to use this. #### On-Premise -If you are using on-premise detection, a "Lite" version of the data required is packaged -in this repository. It contains only a limited set of "essential" device detection properties. +If you are using on-premise detection, a free "Lite" version of the data required is +available from the [data](https://github.com/51Degrees/data) repository, which is the +`cxx/data` folder of this repository. It contains only a limited set of "essential" +device detection properties. You may want to license our complete data file containing all properties. [Details of our licenses](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) are available on our website. -If you want to use the lite file, you will need to install [GitLFS](https://git-lfs.github.com/), then: +To get the Lite file, run the data repository's download script from the root of +this repository: ``` -git lfs install +cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash ``` -Then, navigate to 'device-detection.hash.engine.on-premise/src/main/cxx/device-detection-cxx/device-detection-data' and execute: - -``` -git lfs pull -``` +On Windows, run `cxx\data\get-data-files.ps1 51Degrees-LiteV4.1.hash` in PowerShell +instead. The file is saved in `cxx/data/device-detection`, where the tests and +examples find it. ## Installation diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java index ac8007cd1..44d233d6a 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; import fiftyone.pipeline.util.FileFinder; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; public class TestsBase { @@ -43,7 +43,7 @@ protected UserAgentGenerator getUserAgents() { protected void testInitialize() throws Exception { wrapper = new WrapperCloud(); userAgents = new UserAgentGenerator( - FileFinder.getFilePath(UA_FILE_NAME)); + FileFinder.getFilePath(UA_FILE_PATH)); } public void testCleanup() { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java index 0351da463..f3826ee0e 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java @@ -26,7 +26,7 @@ import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.Constants; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; public class TestsBase { @@ -47,7 +47,7 @@ protected void testInitialize(Constants.PerformanceProfiles profile) throws Exce FileUtils.getHashFile(), profile); userAgents = new UserAgentGenerator( - getFilePath(UA_FILE_NAME)); + getFilePath(UA_FILE_PATH)); } public void testCleanup() { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index b73fbd83c..0b58bc098 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -33,7 +33,7 @@ import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.Constants; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -66,7 +66,7 @@ private void testDataSourceTier(String fileName) throws Exception { String tier = engine.getDataSourceTier(); engine.close(); - if (fileName.equals(LITE_HASH_DATA_FILE_NAME)) { + if (fileName.equals(LITE_HASH_DATA_FILE_PATH)) { assertEquals("Lite", tier); } else { @@ -81,7 +81,7 @@ private void testDataSourceTier(String fileName) throws Exception { */ @Test public void Engine_Hash_GetDataSourceTier_Lite() throws Exception { - testDataSourceTier(LITE_HASH_DATA_FILE_NAME); + testDataSourceTier(LITE_HASH_DATA_FILE_PATH); } /** diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java index 8eff15a70..fe9e13a9e 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java @@ -47,6 +47,14 @@ public class FileUtils { private static File UA_FILE; public static final String EVIDENCE_FILE_NAME = "20000 Evidence Records.yml"; private static Optional EVIDENCE_FILE; + // Where each free file is in the data repository. A search for the path + // rather than the name does not stop at another file of the same name, + // such as a Git LFS pointer in an old device-detection-data folder. + public static final String LITE_HASH_DATA_FILE_PATH = + "device-detection/" + LITE_HASH_DATA_FILE_NAME; + public static final String UA_FILE_PATH = "device-detection/" + UA_FILE_NAME; + public static final String EVIDENCE_FILE_PATH = + "device-detection/" + EVIDENCE_FILE_NAME; /** * Helper to find the location of an Enterprise or Lite Hash file in the default search scope @@ -78,7 +86,7 @@ public static File getHashFile() { } catch (Exception ex) { try { // following throws exception if not found - HASH_DATA_FILE = Optional.of(getFilePath(LITE_HASH_DATA_FILE_NAME)); + HASH_DATA_FILE = Optional.of(getFilePath(LITE_HASH_DATA_FILE_PATH)); return HASH_DATA_FILE.get(); } catch (Exception exc) { HASH_DATA_FILE = Optional.empty(); @@ -93,7 +101,7 @@ public static File getEvidenceFile() { return EVIDENCE_FILE.orElse(null); } try { - EVIDENCE_FILE = Optional.of(getFilePath(EVIDENCE_FILE_NAME)); + EVIDENCE_FILE = Optional.of(getFilePath(EVIDENCE_FILE_PATH)); return EVIDENCE_FILE.get(); } catch (Exception e) { EVIDENCE_FILE = Optional.empty(); diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java index 7d1bac24d..a59a0a6d6 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java @@ -52,7 +52,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; import static fiftyone.pipeline.core.Constants.EVIDENCE_HTTPHEADER_PREFIX; import static fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; @@ -99,7 +99,7 @@ public class DeviceDetectionTests { @BeforeClass public static void initClass() throws IOException { userAgents = new UserAgentGenerator( - FileFinder.getFilePath(UA_FILE_NAME)); + FileFinder.getFilePath(UA_FILE_PATH)); } private static String reportErrors(Collection errors) { diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 5d2edd339..1f42ecd77 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -21,6 +21,14 @@ Among other things, the examples illustrate: - use of device detection pipeline for off-line processing tasks - configuring device detection trade-offs between speed and conserving memory +## Data files + +The on-premise examples use the free Lite data file and the evidence files in +`cxx/data/device-detection`, a folder of the +[data](https://github.com/51Degrees/data) repository. An example that cannot +find one of these files runs the data repository's download script for it, +and logs the command it runs. + ## Cloud resource keys You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) @@ -114,7 +122,8 @@ PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-ex The on-premise web example takes its data file from the `51DEGREES_DD_PATH` environment variable, then from `TestDataFile`, and otherwise uses the free Lite file -in `device-detection-data`. +in `cxx/data/device-detection`, which it downloads before the server starts if it is +missing. ### Native library access diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 9d8819565..332ef1dca 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -58,12 +58,13 @@ public class GettingStartedOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, by default, the 51degrees "Lite" file needs to be somewhere in the project - space, or you may specify another file as a command line parameter. + /* In this example, by default, the 51degrees "Lite" file is used from the data repository in + cxx/data, and is downloaded if it is missing, or you may specify another file as a command line + parameter. Note that the Lite data file is only used for illustration, and has limited accuracy and capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ - public static String LITE_V_4_1_HASH = "51Degrees-LiteV4.1.hash"; + public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static String ENTERPRISE_HASH = "TAC-HashV41.hash"; public static void main(String[] args) throws Exception { @@ -87,7 +88,8 @@ public static void run(String dataFile, OutputStream outputStream) throws Exception { logger.info("Running GettingStarted example"); try { - String dataFileLocation = getFilePath(dataFile).getAbsolutePath(); + String dataFileLocation = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); // the location of the test data file is interpolated in the pipeline // configuration file System.setProperty("TestDataFile", dataFileLocation); diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index a54bfe71c..4c0aa55a1 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -27,7 +27,6 @@ import fiftyone.devicedetection.examples.shared.EvidenceHelper; import fiftyone.devicedetection.hash.engine.onpremise.data.DeviceDataHash; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.core.data.ElementPropertyMetaData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -80,7 +79,7 @@ static void run(String dataFile, List> evidenceList, boolean showDescs, OutputStream out) throws Exception { PrintWriter writer = new PrintWriter(out, true); if (Objects.isNull(dataFile)) { - dataFile = FileUtils.getHashFileName(); + dataFile = DataFileHelper.getDataFileLocation(null); } logger.info("Constructing pipeline from file " + dataFile); diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index a58e739ab..1dd9b1d21 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -45,12 +45,13 @@ public class MetadataOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, by default, the 51degrees "Lite" file needs to be somewhere in the project - space, or you may specify another file as a command line parameter. + /* In this example, by default, the 51degrees "Lite" file is used from the data repository in + cxx/data, and is downloaded if it is missing, or you may specify another file as a command line + parameter. Note that the Lite data file is only used for illustration, and has limited accuracy and capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ - public static String LITE_V_4_1_HASH = "51Degrees-LiteV4.1.hash"; + public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -63,7 +64,8 @@ public static void run(String dataFile, OutputStream output) throws Exception { logger.info("Running MetadataOnPrem example"); String dataFileLocation; try { - dataFileLocation = getFilePath(dataFile).getAbsolutePath(); + dataFileLocation = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { DataFileHelper.cantFindDataFile(dataFile); throw e; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index 031c63368..8fc928a56 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -67,22 +67,22 @@ */ public class OfflineProcessing { static final Logger logger = LoggerFactory.getLogger(OfflineProcessing.class); - // This 51degrees "Lite" file (distributed with the source) needs to - // be somewhere in the project space + // This 51degrees "Lite" file is used from the data repository in + // cxx/data, and is downloaded if it is missing // // Note that the Lite data file is only used for illustration, and has // limited accuracy and capabilities. Find out about the Enterprise data // file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash public static final String LITE_V_4_1_HASH = - "device-detection-data/51Degrees-LiteV4.1.hash"; - // This 51degrees file of 20,000 examples (distributed with the source) - // needs to be somewhere in the project space + "device-detection/51Degrees-LiteV4.1.hash"; + // This 51degrees file of 20,000 examples is used from the same folder, + // and is downloaded if it is missing public static final String HEADER_EVIDENCE_YML = - "device-detection-data/20000 Evidence Records.yml"; + "device-detection/20000 Evidence Records.yml"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - File evidenceFile = getFilePath(HEADER_EVIDENCE_YML); + File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(LITE_V_4_1_HASH, Files.newInputStream(evidenceFile.toPath()), System.out); } @@ -100,7 +100,8 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws String detectionFile; try { - detectionFile = getFilePath(dataFile).getAbsolutePath(); + detectionFile = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { DataFileHelper.cantFindDataFile(dataFile); throw e; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index aeaf3180c..8c6f2b0fe 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -388,8 +388,9 @@ public PerformanceConfiguration(Constants.PerformanceProfiles profile, * @example PerformanceBenchmark.java * The example illustrates a "clock-time" benchmark for assessing detection speed. * - * Using a YAML formatted evidence file - "20000 Evidence Records.yml" - supplied with the - * distribution or can be obtained from the [data repository on Github](https://github.com/51Degrees/device-detection-data/blob/master/20000%20Evidence%20Records.yml). + * Using a YAML formatted evidence file - "20000 Evidence Records.yml" - which is downloaded by + * the scripts in the [data repository on GitHub](https://github.com/51Degrees/data), and by the + * example itself when the file is missing. * * It's important to understand the trade-offs between performance, memory usage and accuracy, that * the 51Degrees pipeline configuration makes available, and this example shows a range of diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java index 9e9ba2320..621ef0761 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; @@ -75,9 +76,10 @@ public class ShareUsageExample { static final Logger logger = LoggerFactory.getLogger(ShareUsageExample.class); - // Evidence file with 20,000 examples + // Evidence file with 20,000 examples, from the data repository in + // cxx/data. The example downloads the file if it is missing. public static final String HEADER_EVIDENCE_YML = - "device-detection-data/20000 Evidence Records.yml"; + "device-detection/20000 Evidence Records.yml"; // Configurable settings private static final int MINIMUM_ENTRIES_PER_MESSAGE = 10; // Send after 10 entries (default is 50) @@ -100,7 +102,7 @@ public class ShareUsageExample { public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - File evidenceFile = getFilePath(HEADER_EVIDENCE_YML); + File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(Files.newInputStream(evidenceFile.toPath())); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index 6dae1a226..d3babcd27 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -73,9 +73,9 @@ * ``` * ## Lite Data File * Lite data files (free-to-use, limited capabilities, no license key required) are created roughly - * once a month and cannot be updated using auto-update, they may be downloaded from - * [Github](https://github.com/51Degrees/device-detection-data) and are included with - * source distributions of this software. + * once a month and cannot be updated using auto-update, they may be downloaded with the + * scripts in the [data repository](https://github.com/51Degrees/data) on GitHub, which is + * the cxx/data folder of the source. * # Update on Start-Up * You can configure the pipeline builder to download an Enterprise data file on start-up. * ## Pre-Requisites diff --git a/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml b/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml index fed362bfd..ee634b786 100644 --- a/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml +++ b/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml @@ -5,7 +5,7 @@ false false - ${TestDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash} + ${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash} LowMemory DeviceDetectionHashEngine diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java index 1c272b52e..779a70fc2 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java @@ -31,7 +31,7 @@ public class GettingStartedOnPremTest { @Test public void gettingStartedOnPremTest() throws Exception { GettingStartedOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFileName(), EvidenceHelper.setUpEvidence(), System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java index 32d3b8882..a128b1393 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java @@ -29,7 +29,7 @@ public class MetadataOnPremTest { @Test public void gettingMetaDataOnPremTest() throws Exception { MetadataOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFileName(), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java index 80d767c58..4861f14d6 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java @@ -32,7 +32,7 @@ public class OfflineProcessingTest { @Test public void offlineProcessingTest() throws Exception { OfflineProcessing.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFileName(), new FileInputStream(Objects.requireNonNull(FileUtils.getEvidenceFile())), System.out); diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java index c5aab3928..337f6e165 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java @@ -32,10 +32,15 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.List; +import java.util.Locale; import java.util.Objects; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -43,6 +48,12 @@ public class DataFileHelper { static Logger logger = LoggerFactory.getLogger(DataFileHelper.class); + /** + * Sent with each download, so the Distributor can count the downloads + * made by these examples. + */ + private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; @@ -90,9 +101,10 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { "than {} days old. A more recent data file " + "may be needed to correctly detect the latest " + "devices, browsers, etc.", daysOld); - logger.info("The latest 'Lite' data file is available " + - "from the device-detection-data repository on GitHub " + - "https://github.com/51Degrees/device-detection-data. " + + logger.info("Run get-data-files.ps1 -Update or " + + "get-data-files.sh -u in cxx/data, which is the data " + + "repository https://github.com/51Degrees/data, to get " + + "the latest 'Lite' data file. " + "Find out about the Enterprise data file, which " + "includes automatic daily updates, on our pricing " + "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-age-warning"); @@ -102,9 +114,10 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { public static void cantFindDataFile(String dataFile) { logger.error("Could not find the data file '{}' which must be " + "somewhere in the project space to be found.", dataFile); - logger.info("The latest 'Lite' data file is available " + - "from the device-detection-data repository on GitHub " + - "https://github.com/51Degrees/device-detection-data. " + + logger.info("The free 'Lite' data file and the evidence files " + + "are downloaded by get-data-files.ps1 or get-data-files.sh " + + "in cxx/data, which is the data repository " + + "https://github.com/51Degrees/data. " + "Find out about the Enterprise data file, which includes " + "automatic daily updates, on our pricing " + "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-not-found"); @@ -120,12 +133,12 @@ public static void cantFindDataFile(String dataFile) { @SuppressWarnings("RedundantThrows") public static File getEvidenceFile(String evidenceFilename) throws Exception { if (Objects.isNull(evidenceFilename)) { - evidenceFilename = FileUtils.EVIDENCE_FILE_NAME; + evidenceFilename = FileUtils.EVIDENCE_FILE_PATH; } File evidenceFile; try { - evidenceFile = FileFinder.getFilePath(evidenceFilename); + evidenceFile = findDataFile(evidenceFilename); } catch (Exception e) { logger.error("Could not find evidence file {}", evidenceFilename); throw e; @@ -134,7 +147,8 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { } /** - * Tries to find the passed file, or if null a default file + * Tries to find the passed file, or if null the first data file found, + * or the free Lite file when there is none * @param dataFilename a filename to find * @return a full pathname * @throws Exception if the file was not found @@ -144,13 +158,90 @@ public static String getDataFileLocation(String dataFilename) throws Exception { if (Objects.isNull(dataFilename)) { dataFilename = FileUtils.getHashFileName(); } + if (Objects.isNull(dataFilename)) { + dataFilename = FileUtils.LITE_HASH_DATA_FILE_PATH; + } String dataFileLocation; try { - dataFileLocation = getFilePath(dataFilename).getAbsolutePath(); + dataFileLocation = findDataFile(dataFilename).getAbsolutePath(); } catch (Exception e) { DataFileHelper.cantFindDataFile(dataFilename); throw e; } return dataFileLocation; } + + /** + * Finds a file below the project root, as + * {@link FileFinder#getFilePath(String)} does. A free data file that is + * missing has a {@code .link} in the data repository, so the data + * repository's script is run to download it, and the file is looked for + * again. + * @param file the name of the file, or the end of its path + * @return the file + * @throws IllegalArgumentException if the file was not found, naming the + * command that downloads it when there is one + */ + public static File findDataFile(String file) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + File link; + try { + link = getFilePath(file + ".link"); + } catch (IllegalArgumentException noLink) { + throw notFound; + } + String command = runDownload(link); + try { + return getFilePath(file); + } catch (IllegalArgumentException stillNotFound) { + String message = "Could not download '" + file + + "'. Run this command to download it: " + command; + logger.error(message); + throw new IllegalArgumentException(message, stillNotFound); + } + } + } + + /** + * Runs the data repository's script for the file that a link is for, + * and waits for it to finish. The data repository is the folder above + * the one the link is in. + * @param link the {@code .link} file + * @return the command, as it would be typed + */ + private static String runDownload(File link) { + String name = link.getName().substring( + 0, link.getName().length() - ".link".length()); + File data = link.getAbsoluteFile().getParentFile().getParentFile(); + List command; + if (System.getProperty("os.name", "") + .toLowerCase(Locale.ROOT).startsWith("windows")) { + command = Arrays.asList("powershell", "-NoProfile", + "-ExecutionPolicy", "Bypass", "-File", + new File(data, "get-data-files.ps1").getPath(), name, + "-Source", DOWNLOAD_SOURCE); + } else { + command = Arrays.asList("sh", + new File(data, "get-data-files.sh").getPath(), + "-s", DOWNLOAD_SOURCE, name); + } + List typed = new ArrayList<>(); + for (String argument : command) { + typed.add(argument.contains(" ") ? + "\"" + argument + "\"" : argument); + } + String line = String.join(" ", typed); + logger.info("'{}' is missing, so it is being downloaded by running {}", + name, line); + try { + new ProcessBuilder(command).inheritIO().start().waitFor(); + } catch (IOException e) { + logger.error("Could not run {}. {}", line, e.getMessage()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return line; + } } diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index 22b31c746..8387720c6 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.web; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; @@ -34,6 +35,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.File; import java.io.PrintWriter; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; @@ -48,7 +50,8 @@ *

* The data file is taken from the 51DEGREES_DD_PATH environment variable, then * from TestDataFile (an environment variable or system property), and otherwise - * the free Lite file in device-detection-data is used. + * the free Lite file in cxx/data/device-detection is used, which is downloaded + * before the server starts if it is missing. *

* The server listens on port 8081 and stops when Enter is pressed. If the PORT * environment variable is set, the server listens on that port instead and runs @@ -63,6 +66,16 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); logger.info("Running Example {}", GettingStartedWebOnPrem.class); + // If the data file the configuration names is not at that path, it + // is searched for, and the free Lite file is downloaded if missing. + String dataFile = PipelineOptionsFactory.getOptionsFromFile( + getFilePath(resourceBase) + "/WEB-INF/51Degrees-OnPrem.xml") + .findAndSubstitute("DeviceDetectionHashEngine", "DataFile"); + if (!new File(dataFile).exists()) { + System.setProperty("TestDataFile", + DataFileHelper.getDataFileLocation(dataFile)); + } + String portEnv = System.getenv("PORT"); if (portEnv != null) { // Automated runs (e.g. the unified Selenium suite) inject a port and have no diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index f785a6607..bba6dd60e 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -32,7 +32,7 @@ variable, then from TestDataFile (an environment variable or system property), and otherwise the free Lite file is used. --> - ${51DEGREES_DD_PATH:-${TestDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash}} + ${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} LowMemory DeviceDetectionHashEngine diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java index 70636a724..d6a94896f 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java @@ -43,7 +43,7 @@ public class GettingStartedWebOnPremTest { @BeforeClass public static void startJetty() throws Exception { System.setProperty("TestDataFile", FileUtils.getHashFile() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFile().getAbsolutePath()); SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); } diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java index 7c898e848..108fbd615 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java @@ -43,7 +43,7 @@ public class OnPremDataFileOptionTest { private static final String DD_PATH = "51DEGREES_DD_PATH"; private static final String TEST_DATA_FILE = "TestDataFile"; private static final String LITE = - "cxx/device-detection-data/51Degrees-LiteV4.1.hash"; + "cxx/data/device-detection/51Degrees-LiteV4.1.hash"; private String savedDdPath; private String savedTestDataFile; diff --git a/examples/ip-intelligence/.gitignore b/examples/ip-intelligence/.gitignore index 092783861..23e4fdfef 100644 --- a/examples/ip-intelligence/.gitignore +++ b/examples/ip-intelligence/.gitignore @@ -48,5 +48,3 @@ SESSIONS.ser 51Degrees-IPIV4EnterpriseIpiV41.ipi.gz 51Degrees-IPIV4AsnIpiV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi.gz -# device detection data file used by the mixed examples -device-detection-data/ diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 022af4b4a..3ebef2ce6 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -10,16 +10,20 @@ These examples are not distributed as maven jars and need to be built by you. ## Required Files -See [ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data/) -repository for instructions on obtaining the necessary data files for on-premise IP intelligence. - -**IMPORTANT**: The enterprise IP Intelligence data file must be placed in the `ip-intelligence-data` -directory at the root of this repository. The expected filename is `51Degrees-EnterpriseIpiV41.ipi`. - -Two free data files are also usable for many of the examples. The ASN file -(`51Degrees-IPIV4AsnIpiV41.ipi`) is included in the `ip-intelligence-data` submodule, and the -Lite file (`51Degrees-LiteV41.ipi`) can be downloaded by running -`ip-intelligence-data/get-lite-file-from-azure.ps1` (or `.sh`). When the enterprise file is +The data files for on-premise IP intelligence are kept in `cxx/data/ip-intelligence`, a +folder of the [data](https://github.com/51Degrees/data) repository, which is a submodule of +cxx. + +**IMPORTANT**: The enterprise IP Intelligence data file must be placed in the +`cxx/data/ip-intelligence` directory of this repository. The expected filename is +`51Degrees-EnterpriseIpiV41.ipi`. + +Two free data files are also usable for many of the examples, the ASN file +(`51Degrees-IPIV4AsnIpiV41.ipi`) and the Lite file (`51Degrees-LiteV41.ipi`). Run +`cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download +them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. An example +that needs one of them and cannot find it runs the script for that file, and logs the +command it runs. When the enterprise file is absent, the tests for examples that work with any data file fall back to the ASN file and then the Lite file. The OfflineProcessing and PerformanceBenchmark examples need the RegisteredName property, which the ASN file does not contain, so their tests fall back to @@ -27,12 +31,16 @@ the Lite file only. Tests for examples needing enterprise-only properties (Metri and Suspicious) are skipped instead. ``` -ip-intelligence-java-examples/ -├── ip-intelligence-data/ -│ └── 51Degrees-EnterpriseIpiV41.ipi -├── console/ -├── web/ -└── shared/ +51degrees-java/ +├── cxx/ +│ └── data/ +│ └── ip-intelligence/ +│ └── 51Degrees-EnterpriseIpiV41.ipi +└── examples/ + └── ip-intelligence/ + ├── console/ + ├── web/ + └── shared/ ``` The examples locate the data file in the following order: @@ -40,9 +48,9 @@ The examples locate the data file in the following order: 1. The `51DEGREES_IPI_PATH` environment variable or system property, which can be set to an explicit path to the data file. 2. A search of the project folder hierarchy for the expected file name. -3. The expected location `ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi` +3. The expected location `cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi` (the configuration files for the getting started examples fall back to the - free Lite file `ip-intelligence-data/51Degrees-LiteV41.ipi`). + free Lite file `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi`). This project contains sub-modules - **console**, giving examples that are intended to be run from the command line/console and **web**, illustrating use diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index 06538b059..b8df87518 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -112,10 +112,10 @@ public class CompareOnPrem { /** * Relative path to the CSV file of truth records distributed with the - * ip-intelligence-data submodule. + * data repository, in cxx/data/ip-intelligence. */ public static final String GEOIP_COMPARISON_EVIDENCE_REL_PATH = - "ip-intelligence-data/geoip_comparison_evidence.csv"; + "ip-intelligence/geoip_comparison_evidence.csv"; /** * Default name of the output file written to the working directory. diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 9bab40ab5..07f33c130 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -83,7 +83,7 @@ public class GettingStartedMixed { private static final Logger logger = LoggerFactory.getLogger(GettingStartedMixed.class); - public static final String DD_DATA_FILE_DEFAULT = "device-detection-data/51Degrees-LiteV4.1.hash"; + public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; public static final String IPI_DATA_FILE_DEFAULT = DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; public static void main(String[] args) throws Exception { @@ -271,11 +271,12 @@ private static String resolveDataFile(String dataFile) throws IOException { if (Files.exists(relativePath)) { return relativePath.toAbsolutePath().toString(); } - // Try FileFinder (searches within project directory) + // Search within the project directory, downloading the free Lite + // file if it is missing try { - return getFilePath(dataFile).getAbsolutePath(); + return DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { - throw new IOException("Data file not found: " + dataFile); + throw new IOException("Data file not found: " + dataFile, e); } } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 983d25997..608989b73 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -40,7 +40,7 @@ * advanced operation this example can be used to experiment with tuning IP Intelligence for * performance using Performance Profile settings. * - * Evidence files can be obtained from the [ip-intelligence-data repository](https://github.com/51Degrees/ip-intelligence-data). + * Evidence files can be obtained from the [data repository](https://github.com/51Degrees/data). * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). * @@ -99,10 +99,10 @@ public class OfflineProcessing { // be somewhere in the project space // // For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=datadir - private static final String dataDir = "ip-intelligence-data"; + private static final String dataDir = "ip-intelligence"; // This 51degrees file of 20,000 examples (distributed with the source) // needs to be somewhere in the project space. Additional evidence files - // can be obtained from the [ip-intelligence-data repository](https://github.com/51Degrees/ip-intelligence-data) + // can be obtained from the [data repository](https://github.com/51Degrees/data) public static final String HEADER_EVIDENCE_YML = dataDir + "/evidence.yml"; diff --git a/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml b/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml index c6dba02ab..cfb3acbcf 100644 --- a/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml +++ b/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml @@ -5,7 +5,7 @@ false false - ${TestDeviceDetectionDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash} + ${TestDeviceDetectionDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash} LowMemory DeviceDetectionHashEngine @@ -14,7 +14,7 @@ false false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-LiteV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml b/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml index 7f1171389..f156a40b5 100644 --- a/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml +++ b/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml @@ -6,7 +6,7 @@ false false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-LiteV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index d448834d4..4cfa22723 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -39,7 +39,11 @@ import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.List; +import java.util.Locale; import java.util.Objects; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -48,31 +52,38 @@ public class DataFileHelper { static Logger logger = LoggerFactory.getLogger(DataFileHelper.class); /** - * Relative path to the enterprise IP Intelligence data file from the examples repository root. + * Sent with each download, so the Distributor can count the downloads + * made by these examples. + */ + private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + + /** + * Folder and name of the enterprise IP Intelligence data file, which + * belongs in cxx/data/ip-intelligence, a folder of the data repository. * This path is used by both console and web examples for testing. - * The data file is downloaded to this location by CI scripts. + * The data file is linked to this location by CI scripts. */ - public static final String ENTERPRISE_DATA_FILE_REL_PATH = "ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi"; + public static final String ENTERPRISE_DATA_FILE_REL_PATH = "ip-intelligence/51Degrees-EnterpriseIpiV41.ipi"; /** - * Relative path to the freely downloadable Lite IP Intelligence data file. - * Run ip-intelligence-data/get-lite-file-from-azure.ps1 (or .sh) to obtain it. + * Folder and name of the free Lite IP Intelligence data file, which the + * data repository's get-data-files script downloads to that folder. * It contains the RegisteredCountry, RegisteredName and RegisteredOwner properties. */ - public static final String LITE_DATA_FILE_REL_PATH = "ip-intelligence-data/51Degrees-LiteV41.ipi"; + public static final String LITE_DATA_FILE_REL_PATH = "ip-intelligence/51Degrees-LiteV41.ipi"; /** - * Relative path to the ASN IP Intelligence data file which is part of the - * ip-intelligence-data submodule. It contains the Asn and AsnName properties. + * Folder and name of the free ASN IP Intelligence data file, which the + * same script downloads. It contains the Asn and AsnName properties. */ - public static final String ASN_DATA_FILE_REL_PATH = "ip-intelligence-data/51Degrees-IPIV4AsnIpiV41.ipi"; + public static final String ASN_DATA_FILE_REL_PATH = "ip-intelligence/51Degrees-IPIV4AsnIpiV41.ipi"; /** * Find the best IP Intelligence data file available, preferring the - * enterprise file, then the ASN file from the ip-intelligence-data - * submodule, then the Lite file. Suitable for examples that work with - * whatever properties the data file contains, where any data file - * allows the example to run. + * enterprise file, then the free ASN file, then the free Lite file, + * and downloading a free file that is missing. Suitable for examples + * that work with whatever properties the data file contains, where any + * data file allows the example to run. * @return a resolved data file path, or null when none is available */ public static String findAvailableDataFile() { @@ -180,7 +191,7 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { File evidenceFile; try { - evidenceFile = FileFinder.getFilePath(evidenceFilename); + evidenceFile = findDataFile(evidenceFilename); } catch (Exception e) { logger.error("Could not find evidence file {}", evidenceFilename); throw e; @@ -244,13 +255,92 @@ public static String getDataFileLocation(String dataFilename) throws Exception { if (Files.exists(dataPath)) { return dataPath.toAbsolutePath().toString(); } - DataFileHelper.cantFindDataFile(dataFilename); - throw new IOException("Data file not found at relative path: " + dataFilename + - ". Searched in project directories and current working directory." + - " An explicit path can be supplied via the " + IPI_PATH_ENV_VAR + - " environment variable."); + // Download it, if it is a free data file + try { + return findDataFile(dataFilename).getAbsolutePath(); + } catch (IllegalArgumentException notFound) { + DataFileHelper.cantFindDataFile(dataFilename); + throw new IOException("Data file not found at relative path: " + dataFilename + + ". Searched in project directories and current working directory." + + " An explicit path can be supplied via the " + IPI_PATH_ENV_VAR + + " environment variable.", notFound); + } } } } } + + /** + * Finds a file below the project root, as + * {@link FileFinder#getFilePath(String)} does. A free data file that is + * missing has a {@code .link} in the data repository, so the data + * repository's script is run to download it, and the file is looked for + * again. + * @param file the name of the file, or the end of its path + * @return the file + * @throws IllegalArgumentException if the file was not found, naming the + * command that downloads it when there is one + */ + public static File findDataFile(String file) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + File link; + try { + link = getFilePath(file + ".link"); + } catch (IllegalArgumentException noLink) { + throw notFound; + } + String command = runDownload(link); + try { + return getFilePath(file); + } catch (IllegalArgumentException stillNotFound) { + String message = "Could not download '" + file + + "'. Run this command to download it: " + command; + logger.error(message); + throw new IllegalArgumentException(message, stillNotFound); + } + } + } + + /** + * Runs the data repository's script for the file that a link is for, + * and waits for it to finish. The data repository is the folder above + * the one the link is in. + * @param link the {@code .link} file + * @return the command, as it would be typed + */ + private static String runDownload(File link) { + String name = link.getName().substring( + 0, link.getName().length() - ".link".length()); + File data = link.getAbsoluteFile().getParentFile().getParentFile(); + List command; + if (System.getProperty("os.name", "") + .toLowerCase(Locale.ROOT).startsWith("windows")) { + command = Arrays.asList("powershell", "-NoProfile", + "-ExecutionPolicy", "Bypass", "-File", + new File(data, "get-data-files.ps1").getPath(), name, + "-Source", DOWNLOAD_SOURCE); + } else { + command = Arrays.asList("sh", + new File(data, "get-data-files.sh").getPath(), + "-s", DOWNLOAD_SOURCE, name); + } + List typed = new ArrayList<>(); + for (String argument : command) { + typed.add(argument.contains(" ") ? + "\"" + argument + "\"" : argument); + } + String line = String.join(" ", typed); + logger.info("'{}' is missing, so it is being downloaded by running {}", + name, line); + try { + new ProcessBuilder(command).inheritIO().start().waitFor(); + } catch (IOException e) { + logger.error("Could not run {}. {}", line, e.getMessage()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return line; + } } diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 2de381969..6e3fa2e8a 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -94,7 +94,7 @@ * and a JSON builder element which serialises the results. *

* The data file can be supplied as a command line argument. When no argument is given - * the enterprise data file is looked for in the ip-intelligence-data directory. + * the enterprise data file is looked for in the cxx/data/ip-intelligence directory. *

* Note that the .NET version of this example also includes a device detection engine * in the same pipeline. This Java version is IP Intelligence only. diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 9969e3262..5b7331767 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -87,7 +87,7 @@ public class GettingStartedWebMixed extends HttpServlet { private static final long serialVersionUID = 1734154705981153541L; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebMixed.class); - public static final String DD_DATA_FILE_DEFAULT = "device-detection-data/51Degrees-LiteV4.1.hash"; + public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -310,10 +310,13 @@ private static String resolveDataFile(String dataFile) throws IOException { if (Files.exists(relativePath)) { return relativePath.toAbsolutePath().toString(); } + // Search within the project directory, downloading the free Lite + // file if it is missing try { - return getFilePath(dataFile).getAbsolutePath(); + return fiftyone.ipintelligence.examples.shared.DataFileHelper + .findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { - throw new IOException("Data file not found: " + dataFile); + throw new IOException("Data file not found: " + dataFile, e); } } } diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml index 95f859502..127b4665d 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml @@ -27,7 +27,7 @@ false false - ${TestDeviceDetectionDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash} + ${TestDeviceDetectionDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash} LowMemory DeviceDetectionHashEngine @@ -36,7 +36,7 @@ false false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index d1bc62de7..c8e8d515b 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -29,7 +29,7 @@ false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java index 2425e1be5..0d70be0f1 100644 --- a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java +++ b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java @@ -78,8 +78,8 @@ public static String getDataFileAgeWarning(FlowData flowData) { return "

\n" + " The IP intelligence data file is " + daysOld + " days old. " + "A more recent data file may be needed for the most accurate IP " + - "intelligence results. The latest Lite data file is available from the " + - "ip-intelligence-data " + + "intelligence results. The latest Lite data file is downloaded by the " + + "scripts in the data " + "repository on GitHub. Find out about the full Enterprise data file, " + "which includes automatic daily updates, on our " + "pricing page.\n" + diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index 2b087fe20..3d194b87b 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -33,14 +33,13 @@ how to use this. In order to perform IP intelligence on-premise, you will need to use a data file. -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data/) repository instructs how to obtain a 'Lite' data file, otherwise [contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) to obtain an 'Enterprise' data file. +The free 'Lite' and Asn data files come from the [data](https://github.com/51Degrees/data) repository, which is the `cxx/data` folder of this repository. Run `cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. Otherwise [contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) to obtain an 'Enterprise' data file. For tests, the data file can be supplied explicitly by setting the `51DEGREES_IPI_PATH` environment variable or system property to the path of the .ipi file. When it is not set, the project space is searched for the -expected file name, with the free 'Lite' file expected in the -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data/) -location. +expected file name, with the free 'Lite' file expected in +`cxx/data/ip-intelligence`. ## Installation diff --git a/ip-intelligence/ip-intelligence.translation/pom.xml b/ip-intelligence/ip-intelligence.translation/pom.xml index 83cbc8865..a620ab341 100644 --- a/ip-intelligence/ip-intelligence.translation/pom.xml +++ b/ip-intelligence/ip-intelligence.translation/pom.xml @@ -60,9 +60,11 @@ + they are copied in at build time from the data repository in + cxx/data/ip-intelligence/Translations, as the .NET project + embeds the same files. A folder that is not there is skipped + without an error, so TranslationTests, which read the files + from the classpath, are what show they were copied. --> org.apache.maven.plugins maven-resources-plugin @@ -78,13 +80,13 @@ ${project.build.outputDirectory}/fiftyone/ipintelligence/translation - ${project.basedir}/../../cxx/ip-intelligence-data/Translations + ${project.basedir}/../../cxx/data/ip-intelligence/Translations countrycodes.en_GB.yml - ${project.basedir}/../../cxx/ip-intelligence-data/Translations/OSM + ${project.basedir}/../../cxx/data/ip-intelligence/Translations/OSM countries.*.yml From 82cc6a7e830c904b5bf7a6b4f88f94e7ad234166 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:07:55 +0000 Subject: [PATCH 10/95] Say in AGENTS.md how to get the data, build and run the examples (#41) --- AGENTS.md | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5a1d6f620..971eb5d05 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,226 @@ # Working in this repository -Notes on what is not obvious from the code. Keep them short and current. -Building, testing and the layout are described in the [README](README.md). +If you have been asked to get an example running, follow +[Run the examples](#run-the-examples). It gives every step in order, from a +fresh clone to the output. The sections after it are notes on what is not +obvious from the code, for changing it. Keep them short and current. The +[README](README.md) describes the build, the tests and the layout. + +## Run the examples + +Run each command from the repository root unless the step says otherwise. +Device Detection's on-premise `GettingStartedOnPrem` is the quickest example +to show, because its data file is about 35 MB. IP Intelligence's on-premise +`GettingStartedOnPrem` needs a download of about 230 MB that unpacks to about +460 MB, so run it second. + +The examples use the packages published to Maven Central unless the `dev` +profile is active (see +[One reactor, six package trees](#one-reactor-six-package-trees)). Those +packages carry the native engines, so building only an example folder, as +below, needs no CMake and no C++ toolchain. `mvn install` at the root builds +the native engines from `cxx` and does need them, as the README says. + +### Prerequisites + +- Git. +- A JDK, 8 or later. CI tests 8, 11, 17 and 21 (`ci/options.json`). +- Maven 3.8 or later, or the `mvnw` wrapper at the root. +- Network access to GitHub, to Maven Central for the packages the examples + use, and to `distributor.51degrees.com` for the data files. +- For the data files, `curl`, `gzip` and `md5sum` (`md5` on macOS) on Linux + and macOS, where the examples download with the shell script, and Windows + PowerShell 5.1 or PowerShell 7 on Windows, where they use the PowerShell + script. + +Clone the repository with its submodules. + +``` +git clone --recurse-submodules https://github.com/51Degrees/51degrees-java.git +cd 51degrees-java +``` + +In a clone made without `--recurse-submodules`, fetch the submodules. + +``` +git submodule update --init --recursive +``` + +### Data files + +The examples use the free Lite data files, which need no licence key. They are +not stored in this repository. The `cxx` submodule holds the +[data](https://github.com/51Degrees/data) repository as `cxx/data`, which has a +link to each file, and a script downloads them into `cxx/data/device-detection` +and `cxx/data/ip-intelligence`. + +On Linux and macOS, run the shell script. + +``` +cxx/data/get-data-files.sh +``` + +On Windows, run the PowerShell script. Windows PowerShell can refuse to run a +script file under the policy Windows sets by default, so run it the way the +examples do. + +``` +powershell -NoProfile -ExecutionPolicy Bypass -File cxx/data/get-data-files.ps1 +``` + +Where PowerShell 7 is installed, `pwsh cxx/data/get-data-files.ps1` does the +same on any system. Either script downloads every free file, and keeps a file +that is already there. To download one file, give its name, as in +`cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash`. An example whose free +file is missing runs the script for that file itself. + +#### Check the files + +- `cxx/data/device-detection/51Degrees-LiteV4.1.hash` is more than 10 MB. +- `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi` is more than 100 MB. + +### Build + +Build the console examples of a product, with the examples' shared module, in +that product's examples folder. Each build makes a jar that holds the example +and everything it needs. + +``` +cd examples/device-detection +mvn -pl console -am package -DskipTests +cd ../.. +cd examples/ip-intelligence +mvn -pl console -am package -DskipTests +cd ../.. +``` + +With the wrapper, run `../../mvnw` in place of `mvn`, or `..\..\mvnw.cmd` on +Windows. The jars are +`examples/device-detection/console/target/device-detection-java-examples.console--jar-with-dependencies.jar` +and +`examples/ip-intelligence/console/target/ip-intelligence-java-examples.console--jar-with-dependencies.jar`, +where `` is the `` in that examples folder's `pom.xml`, +`4.5.127-SNAPSHOT` on `rc` today. + +### Run + +Run the examples from the repository root. They find `logback.xml`, the data +file and their options file with `FileFinder`, which searches only below the +folder Java runs in (`pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/FileFinder.java`). +When a free data file is not there, the example finds its `.link` the same way +and runs the download script for it (`findDataFile` in each examples folder's +`shared` module, `DataFileHelper.java`), with Windows PowerShell on Windows and +`sh` on Linux and macOS. The same commands work on Windows, Linux and macOS. + +``` +java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedOnPrem +``` + +The IP Intelligence example looks for the paid Enterprise file, +`ip-intelligence/51Degrees-EnterpriseIpiV41.ipi`, so give it the Lite file as +its first argument. + +``` +java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.GettingStartedOnPrem cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +``` + +The example downloads the Lite file first when it is not there. In place of +the argument, the `51DEGREES_IPI_PATH` environment variable or system property +can give the absolute path of a data file that is already there. + +On Java 24 or later, add `--enable-native-access=ALL-UNNAMED` before the class +name, as +[examples/device-detection/README.md](examples/device-detection/README.md#native-library-access) +explains, or Java warns about the native library. + +### Correct output + +When an example downloads its data file, a Logback line first reports +`'' is missing, so it is being downloaded by running` and the command, +and the script's `Downloading` and `Saved` lines follow. + +The Device Detection example processes five sets of evidence. For each it +prints `Input values:` and the evidence, then `Results:` with +`Mobile Device`, `Platform Name`, `Platform Version`, `Browser Name` and +`Browser Version`. Logback lines come before and after. The last ones report +the data file and end the run, with these messages. + +``` +Used a 'Lite' data file, created , days ago, from location '' +This example is using the 'Lite ' data file. This is used for illustration, ... +All done +``` + +A warning that the data file is more than 28 days old may come too, which is +expected with the Lite file. The exit code is 0. + +The IP Intelligence example looks up `116.154.188.222`, `45.236.48.61` and an +IPv6 address. For each it prints `Input values:` with `query.client-ip`, then +`Results:` with a line for each property, from `RegisteredName` to +`TimeZoneOffset`. The Lite file has RegisteredCountry, RegisteredName and +RegisteredOwner, so a line for a property it does not have reads +`Unknown. The property '' is not available in this data file.` followed +by a link, and the first such line also logs a warning that some properties +need a paid subscription. It then logs the data file in the same way, and +`All done`. + +The paid data files have many more properties, as the +[pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=correct-output) +describes. + +### Cloud examples + +The cloud examples need a resource key, which a person creates in the +configurator, so ask for one if none is set. A key with the properties most of +the Device Detection examples use can be made +[here](https://configure.51degrees.com/jqz435Nc?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=cloud-examples). +The Device Detection cloud example takes the key as its first argument, or +from a `TestResourceKey` environment variable or system property. + +``` +java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedCloud +``` + +It prints the same `Input values:` and `Results:` blocks as the on-premise +example. The IP Intelligence cloud examples read their key from +`51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and +`examples/ip-intelligence/README.md` gives the keys to create for them. + +### Pitfalls + +- **A data file that is not there.** `cxx/data` holds a `.link` for each file + rather than the file. An example whose file is missing runs the download + script for it, so it needs the network and the tools under "Prerequisites". + Check the file sizes as above. +- **Missing submodules.** The example then logs `Could not find the data file + 'device-detection/51Degrees-LiteV4.1.hash' which must be somewhere in the + project space to be found.` Run `git submodule update --init --recursive`. A + zip download of the repository has no submodules, so clone with Git. +- **Running from another folder.** `FileFinder` looks only below the folder + Java runs in, so `java` started in `examples/device-detection`, as the + examples' README shows, cannot find the data file or its link in `cxx`. + Start it from the repository root. The same applies to `mvn exec:java`, + which runs in the folder Maven starts in. +- **The version in the jar name.** The examples' READMEs show jar names with + `4.4.20` and `4.4.19`, which are out of date. Use the name in `target`. +- **The IP Intelligence example without an argument.** It searches for the + Enterprise file, logs `Failed to find IP Intelligence data file` and stops, + even when the Lite file is in place. +- **The `dev` profile.** `-Pdev`, which CI uses, builds the examples against + the libraries here, which needs `mvn install` at the root first and so the + native build, with CMake 3.10 or later and a C++ toolchain. It is not needed + to run the examples. +- **Platforms the packages cover.** The published + `device-detection.hash.engine.on-premise` 4.5.128 jar has native libraries + for Windows x64 and x86, Linux x64 and aarch64, and macOS x64 and aarch64. + The `ip-intelligence.engine.on-premise` 4.5.128 jar has the same, without + Windows x86. On a platform a jar lacks, the example cannot load the engine. +- **What CI does not test.** On a pull request CI builds and tests the + examples with the `dev` profile (`ci/build-project.ps1` and + `ci/run-unit-tests.ps1`), and against the published packages only when it + publishes (`ci/run-integration-tests.ps1`). It never runs the console + example jars as above, and the example tests use the paid data files when + CI has them. ## One reactor, six package trees @@ -53,6 +272,14 @@ The Windows pre-build scripts take a toolset and SDK version from the pom. JavaScript templates and the OWID library are submodules inside `pipeline`, where the Pipeline API expects them. +## Releases + +Pull requests go to `rc`, not `main`, and stay drafts until they have been +reviewed, because the nightly merges any pull request into `rc` that is not a +draft and has no review requested. When a release is wanted, one pull request +merges `rc` into `main`, and the "Only rc into main" check fails a pull request +into `main` from any other branch. + ## Writing Plain words, no em dashes, sentence case headings. Nothing in this public From 086322c51744b8b6e51ca9bc6f76fe82fa69f829 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:08:22 +0000 Subject: [PATCH 11/95] Point package metadata at this repository and fix stale links (#42) --- ci/README.md | 2 +- device-detection/README.md | 8 ++--- .../device-detection.cloud/pom.xml | 2 +- .../pom.xml | 2 +- .../device-detection.shared/pom.xml | 2 +- device-detection/device-detection/pom.xml | 2 +- device-detection/pom.xml | 8 +++-- examples/device-detection/README.md | 29 ++++++++----------- .../examples/console/GettingStartedCloud.java | 2 +- .../console/GettingStartedOnPrem.java | 2 +- .../examples/console/MetadataOnPrem.java | 2 +- .../examples/console/MinimalExample.java | 2 +- .../examples/console/NativeModelCloud.java | 2 +- .../examples/console/TacCloud.java | 2 +- examples/ip-intelligence/README.md | 8 ++--- .../examples/console/CompareOnPrem.java | 2 +- .../examples/console/GettingStartedCloud.java | 2 +- .../console/GettingStartedOnPrem.java | 2 +- .../examples/console/MetricsOnPrem.java | 2 +- .../examples/console/SuspiciousOnPrem.java | 6 ++-- ip-intelligence/README.md | 8 ++--- ip-intelligence/ip-intelligence.cloud/pom.xml | 2 +- .../ip-intelligence.engine.on-premise/pom.xml | 2 +- .../ip-intelligence.shared/pom.xml | 2 +- .../ip-intelligence.translation/pom.xml | 2 +- ip-intelligence/ip-intelligence/pom.xml | 2 +- ip-intelligence/pom.xml | 8 +++-- location/README.md | 6 ++-- location/pom.xml | 6 ++-- pipeline/README.md | 4 +-- pipeline/pipeline.caching/pom.xml | 2 +- pipeline/pipeline.cloudrequestengine/pom.xml | 2 +- .../cloudrequestengine/Constants.java | 2 +- pipeline/pipeline.common/pom.xml | 2 +- pipeline/pipeline.core/pom.xml | 2 +- pipeline/pipeline.did/pom.xml | 6 +++- pipeline/pipeline.engines.fiftyone/pom.xml | 2 +- pipeline/pipeline.engines/pom.xml | 2 +- .../fiftyone/pipeline/engines/Constants.java | 4 +-- pipeline/pipeline.javascriptbuilder/pom.xml | 2 +- pipeline/pipeline.jsonbuilder/pom.xml | 2 +- pipeline/pipeline.translation/pom.xml | 2 +- .../pipeline.web.mvc/pom.xml | 2 +- .../pipeline.web.shared/pom.xml | 2 +- .../pipeline.web/pom.xml | 2 +- pipeline/pom.xml | 6 ++-- 46 files changed, 90 insertions(+), 83 deletions(-) diff --git a/ci/README.md b/ci/README.md index a537b0271..bf6b2a62b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -55,7 +55,7 @@ To run the on-premise tests: To run the cloud tests: -- `_51DEGREES_RESOURCE_KEY_BESPOKE`: [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) +- `_51DEGREES_RESOURCE_KEY_BESPOKE`: [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) passed to the tests as `TestResourceKey` and `SuperResourceKey`. - `_51DEGREES_LICENSE_KEY_51DID`: license key for the 51Did cloud tests, together with the `_51DEGREES_RESOURCE_KEY_BASIC` and diff --git a/device-detection/README.md b/device-detection/README.md index 73722fcb5..5441e364a 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -11,7 +11,7 @@ This repository contains Java implementation of the device detection [specificat ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -21,9 +21,9 @@ The Java API can either use our cloud service to get its data or it can use a lo #### Cloud -You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) +You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on +our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise @@ -225,7 +225,7 @@ repository. ## Tests -You will need [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) +You will need [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) (see above) to complete the tests and run examples which include exercising the cloud API. To verify the code: diff --git a/device-detection/device-detection.cloud/pom.xml b/device-detection/device-detection.cloud/pom.xml index 4e6c839b7..ddfbaf069 100644 --- a/device-detection/device-detection.cloud/pom.xml +++ b/device-detection/device-detection.cloud/pom.xml @@ -12,7 +12,7 @@ device-detection.cloud 51Degrees :: Device Detection :: Cloud Cloud-based device detection for Java that uses the 51Degrees cloud service via the Pipeline API. Parses HTTP headers to identify device hardware, operating systems, browsers, and crawlers. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection.cloud-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.hash.engine.on-premise/pom.xml b/device-detection/device-detection.hash.engine.on-premise/pom.xml index 87250bfe8..7d141d302 100644 --- a/device-detection/device-detection.hash.engine.on-premise/pom.xml +++ b/device-detection/device-detection.hash.engine.on-premise/pom.xml @@ -12,7 +12,7 @@ device-detection.hash.engine.on-premise 51Degrees :: Device Detection :: Hash :: On Premise High-performance on-premise hash-based device detection engine for Java, built for the 51Degrees Pipeline API. Parses HTTP headers locally to identify device hardware, operating systems, browsers, and crawlers, with no external service calls. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection.hash.engine.on-premise-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.shared/pom.xml b/device-detection/device-detection.shared/pom.xml index 202797eb3..7d2618c67 100644 --- a/device-detection/device-detection.shared/pom.xml +++ b/device-detection/device-detection.shared/pom.xml @@ -12,7 +12,7 @@ device-detection.shared 51Degrees :: Device Detection :: Shared Shared components and utilities for building 51Degrees Pipeline API device detection engines in Java. Supports parsing HTTP headers to identify device type, model, OS, browser, and crawler information—an alternative approach to UAParser, DeviceAtlas, and WURFL-based solutions. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection.shared-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection/pom.xml b/device-detection/device-detection/pom.xml index 7ffa7b7d3..96eefb171 100644 --- a/device-detection/device-detection/pom.xml +++ b/device-detection/device-detection/pom.xml @@ -13,7 +13,7 @@ device-detection 51Degrees :: Device Detection Parse HTTP headers to detect the device type, model, OS, browser, and crawler information. This is an alternative to popular UAParser, DeviceAtlas, and WURFL packages. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/pom.xml b/device-detection/pom.xml index 8aef20aea..ce56dd201 100644 --- a/device-detection/pom.xml +++ b/device-detection/pom.xml @@ -1,4 +1,4 @@ - 4.0.0 @@ -9,7 +9,11 @@ pom 51Degrees :: Pipeline :: Device Detection - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java + + 51Degrees + https://51degrees.com/device-detection?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.device-detection&utm_term=organization + Device detection engines for the 51Degrees Pipeline API diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 1f42ecd77..4c80193e5 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -31,9 +31,9 @@ and logs the command it runs. ## Cloud resource keys -You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) +You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on +our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on how to use this. A resource key configured with the properties needed @@ -49,23 +49,18 @@ resource keys and license keys at our [pricing page](https://51degrees.com/prici ## Running examples with changes to Pipeline packages -A common use case is to make a change to the Pipeline logic in -device-detection-java and then use these examples to observe the results of the -change. +A common use case is to make a change to the Device Detection or Pipeline +packages in this repository and then use these examples to observe the +results of the change. -By default, the examples are configured to use the packages from Maven central. -In order to produce and use local packages instead: +By default, the examples use the packages from Maven Central. To build them +against the packages in this repository instead, run Maven from the root of +the repository with the `dev` profile, as the +[root README](../../README.md#examples) describes: -- Clone and make your changes to device-detection-java -- Set the version of the device detection packages that we're going to create to 0.0.0: - `mvn versions:set-property -Dproperty="project.version" "-DnewVersion=0.0.0"` -- Create and install the packages locally (skipping tests is needed): - `mvn clean install [-DskipTests]` -- Modify the POM for the examples to reference these new local packages. This can - be done by editing the POM directly or by using the command line: - `mvn versions:set-property -Dproperty="device-detection.version" "-DnewVersion=0.0.0"` - -The same principle can be applied to incorporate changes in pipeline-java if needed. +``` +mvn install -DskipTests -Pdev +``` ## Examples diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java index c97a1868a..846e1fcb4 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java @@ -84,7 +84,7 @@ public static void run(String resourceKey, /* In this example, we use the DeviceDetectionPipelineBuilder and configure it in code. For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=run + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=run Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ try (Pipeline pipeline = new DeviceDetectionPipelineBuilder() diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 332ef1dca..44b7227fb 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -102,7 +102,7 @@ public static void run(String dataFile, options contained in the file "gettingStartedOnPrem.xml". For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=run + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=run Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ // the configuration file is in the resources directory diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 1dd9b1d21..5b6939705 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -81,7 +81,7 @@ public static void run(String dataFile, OutputStream output) throws Exception { // example. See the documentation for more detail on this and other // configuration options: // https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=run - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=run + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=run .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) // inhibit auto-update of the data file for this test .setAutoUpdate(false) diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java index 4b996daba..8c5fb5206 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java @@ -29,7 +29,7 @@ /** * This is the code for the minimal example displayed in the configurator. Please see - * + * * Getting Started Console Cloud for a fuller example. */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index 754f0596a..3c6c724ac 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -56,7 +56,7 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // This example creates the pipeline and engines in code. For a demonstration // of how to do this using a configuration file instead, see the TacCloud example. // For more information about builders in general see the documentation at - // https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=run + // https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=run resourceKey = getOrSetSuperResourceKey(resourceKey, NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME); try (PrintWriter output = new PrintWriter(os)) { diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index 91213032b..ee5ee1e0b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -72,7 +72,7 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // For a demonstration of how to do this in code instead, see the // NativeModelLookup example. // For more information about builders in general see the documentation at - // https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=run + // https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=run // the configuration file is in the resources directory File optionsFile = getFilePath("tacCloud.xml"); diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 3ebef2ce6..9344df4e0 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -129,14 +129,14 @@ to get a paid subscription with more properties. ## Running Examples and Tests -**IMPORTANT**: All examples and tests must be run from the repository root directory (`ip-intelligence-java-examples/`). +**IMPORTANT**: All examples and tests must be run from this folder, `examples/ip-intelligence`. ### Running Tests To run all tests: ```bash -# From the repository root directory +# From examples/ip-intelligence mvn test ``` @@ -155,14 +155,14 @@ mvn test -pl web/getting-started.onprem Console examples can be run directly using Maven exec plugin: ```bash -# From the repository root directory +# From examples/ip-intelligence mvn compile exec:java -pl console -Dexec.mainClass="fiftyone.ipintelligence.examples.console.GettingStartedOnPrem" ``` Web examples: ```bash -# From the repository root directory +# From examples/ip-intelligence mvn compile exec:java -pl web/getting-started.onprem -Dexec.mainClass="fiftyone.ipintelligence.examples.web.GettingStartedWebOnPrem" ``` diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index b8df87518..4b994e3b9 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -202,7 +202,7 @@ public static void run(String dataFile, // We use the max performance profile for optimal lookup // speed in this example. See the documentation for more detail // on this and other configuration options. - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) // inhibit sharing usage for this example .setShareUsage(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java index fb63e8546..684990a91 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java @@ -125,7 +125,7 @@ public static void run(String resourceKey, /* In this example, we use the IPIntelligencePipelineBuilder and configure it in code. For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java&utm_term=configuration-builders + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java&utm_term=configuration-builders Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ IPIntelligenceCloudPipelineBuilder builder = new IPIntelligencePipelineBuilder() diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 8c6e4d0b9..41a7712a5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -139,7 +139,7 @@ public static void run(String dataFile, options contained in the file "gettingStartedOnPrem.xml". For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=run + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=run Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ // the configuration file is in the resources directory diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index aaf6ddac4..6c66b486f 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -343,7 +343,7 @@ public static void run(String dataFile, // detection speed in this example. See the // documentation for more detail on this and other // configuration options. - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) // inhibit sharing usage for this example .setShareUsage(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index dd46f9cc5..3df68b732 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -120,7 +120,7 @@ public static void run(String dataFile, /* In this example, we use the IPIntelligencePipelineBuilder and configure it in code. For more information about builders in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=configuration-builders + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=configuration-builders Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ try (Pipeline pipeline = new IPIntelligencePipelineBuilder() @@ -128,8 +128,8 @@ public static void run(String dataFile, // We use the max performance profile for optimal lookup speed in this // example. See the documentation for more detail on this and other // configuration options. - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates - // https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates + // https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) // inhibit sharing usage for this example, usually this should be set to "true" .setShareUsage(false) diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index 3d194b87b..8f4534ed9 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -11,7 +11,7 @@ This repository contains Java implementation of the IP Intelligence [specificati ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -23,9 +23,9 @@ The Java API can either use our cloud service to get its data or it can use a lo The cloud service for IP Intelligence is currently in development. -You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) +You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon), see our [documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) on +our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) on how to use this. #### On-Premise @@ -108,7 +108,7 @@ repository. ## Tests -You will need [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) +You will need [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) (see above) to complete the tests and run examples which include exercising the cloud API. The tests look for a resource key in the following order: diff --git a/ip-intelligence/ip-intelligence.cloud/pom.xml b/ip-intelligence/ip-intelligence.cloud/pom.xml index 9a1d434f4..cfc6ffa32 100644 --- a/ip-intelligence/ip-intelligence.cloud/pom.xml +++ b/ip-intelligence/ip-intelligence.cloud/pom.xml @@ -12,7 +12,7 @@ ip-intelligence.cloud 51Degrees :: IP Intelligence :: Cloud Provides IP Intelligence for the 51Degrees Pipeline API using the 51Degrees Cloud service. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.cloud-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml index 00b37b646..0c48477de 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml +++ b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml @@ -12,7 +12,7 @@ ip-intelligence.engine.on-premise 51Degrees :: IP Intelligence :: On Premise Provides the On-premise IP Intelligence engine for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.engine.on-premise-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.shared/pom.xml b/ip-intelligence/ip-intelligence.shared/pom.xml index 71ab0f830..3f2271d0a 100644 --- a/ip-intelligence/ip-intelligence.shared/pom.xml +++ b/ip-intelligence/ip-intelligence.shared/pom.xml @@ -12,7 +12,7 @@ ip-intelligence.shared 51Degrees :: IP Intelligence :: Shared Shared code used by 51Degrees IP Intelligence modules for the Pipeline API. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.shared-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.translation/pom.xml b/ip-intelligence/ip-intelligence.translation/pom.xml index a620ab341..98c629255 100644 --- a/ip-intelligence/ip-intelligence.translation/pom.xml +++ b/ip-intelligence/ip-intelligence.translation/pom.xml @@ -14,7 +14,7 @@ Translation engines that turn the weighted ISO country codes from 51Degrees IP Intelligence into localized, ordered country name and code lists. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.translation-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence/pom.xml b/ip-intelligence/ip-intelligence/pom.xml index b22ba399b..2baf5e0fd 100644 --- a/ip-intelligence/ip-intelligence/pom.xml +++ b/ip-intelligence/ip-intelligence/pom.xml @@ -13,7 +13,7 @@ ip-intelligence 51Degrees :: IP Intelligence Provides IP Intelligence for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/pom.xml b/ip-intelligence/pom.xml index 4327c14b2..16a23b1a3 100644 --- a/ip-intelligence/pom.xml +++ b/ip-intelligence/pom.xml @@ -1,4 +1,4 @@ - 4.0.0 @@ -9,7 +9,11 @@ pom 51Degrees :: Pipeline :: IP Intelligence - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java + + 51Degrees + https://51degrees.com/ip-intelligence?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.ip-intelligence&utm_term=organization + IP Intelligence engines for the 51Degrees Pipeline API diff --git a/location/README.md b/location/README.md index b48074eaa..031f81f12 100644 --- a/location/README.md +++ b/location/README.md @@ -10,16 +10,16 @@ This repository contains the geo-location engines for the Java implementation of ## Dependencies -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ### Data -You will require a [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) +You will require a [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) to use the Cloud API. You can create resource keys using our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data), see our -[documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) on how to use this. +[documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) on how to use this. ## Installation diff --git a/location/pom.xml b/location/pom.xml index e3e332f3f..1939c6520 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -20,7 +20,7 @@ ~ such notice(s) shall fulfill the requirements of that article. --> - 4.0.0 @@ -31,12 +31,12 @@ pom 51Degrees :: Pipeline :: Geo Location - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java Geo location engines for the 51Degrees Pipeline API 51Degrees - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url-2 + https://51degrees.com/geolocation?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.geo-location&utm_term=organization diff --git a/pipeline/README.md b/pipeline/README.md index a860e40c0..8370cc09b 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -16,7 +16,7 @@ Reference documentation for the Java implementation can be found on the ## Pre-requisites -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pre-requisites) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pre-requisites) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -123,7 +123,7 @@ There are several examples available in the `pipeline.developer-examples` folder how to make use of the Pipeline API in isolation. These are described in the table below. If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [repository](https://github.com/51Degrees/51degrees-java/tree/main/device-detection) -and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=examples). +and on our [website](https://51degrees.com/documentation/_device_detection__examples__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=examples). | Example | Description | |----------------------------------------------------|-------------| diff --git a/pipeline/pipeline.caching/pom.xml b/pipeline/pipeline.caching/pom.xml index fba348a5b..2286dd55d 100644 --- a/pipeline/pipeline.caching/pom.xml +++ b/pipeline/pipeline.caching/pom.xml @@ -33,7 +33,7 @@ pipeline.caching 51Degrees :: Pipeline :: Caching - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.caching-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.cloudrequestengine/pom.xml b/pipeline/pipeline.cloudrequestengine/pom.xml index 38d1e2115..2871e320d 100644 --- a/pipeline/pipeline.cloudrequestengine/pom.xml +++ b/pipeline/pipeline.cloudrequestengine/pom.xml @@ -34,7 +34,7 @@ pipeline.cloudrequestengine 51Degrees :: Pipeline :: Cloud Request Engine Cloud request engine for the 51Degrees Pipeline API - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index 2e982f7e1..593c618d6 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -46,7 +46,7 @@ public class Messages { + "because your resource key does not include access to any " + "properties under '%s'. For more details on resource keys, " + "see our explainer: " - + "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; public static final String ProcessCloudEngineNotImplemented = "This method should be overridden in the class derived from " diff --git a/pipeline/pipeline.common/pom.xml b/pipeline/pipeline.common/pom.xml index 68da0f135..cf2089abf 100644 --- a/pipeline/pipeline.common/pom.xml +++ b/pipeline/pipeline.common/pom.xml @@ -33,7 +33,7 @@ pipeline.common 51Degrees :: Pipeline :: Common - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.common-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.core/pom.xml b/pipeline/pipeline.core/pom.xml index 3da59fe4c..3e30f3f64 100644 --- a/pipeline/pipeline.core/pom.xml +++ b/pipeline/pipeline.core/pom.xml @@ -38,7 +38,7 @@ 51Degrees :: Pipeline :: Core Core library for the 51Degrees Pipeline API - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.core-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.did/pom.xml b/pipeline/pipeline.did/pom.xml index 21a2074c1..c5ba43ddc 100644 --- a/pipeline/pipeline.did/pom.xml +++ b/pipeline/pipeline.did/pom.xml @@ -39,7 +39,11 @@ value returned by the 51Degrees Cloud service. Parses the OWID envelope and exposes the Flags, License Id and match key plus the identifier type. Compare match keys, never envelopes. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.did-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java + + 51Degrees + https://51degrees.com/developers/51did?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.did&utm_term=organization + - 4.0.0 @@ -31,12 +31,12 @@ pom 51Degrees :: Pipeline - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java The 51Degrees Pipeline API provides architecture for consuming real-time digital data services. 51Degrees - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url-2 + https://51degrees.com/build-your-own?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline&utm_term=organization From 377323863270fdd3362483330f2691e58e1f30b8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:08:44 +0000 Subject: [PATCH 12/95] Describe the rc approach to releases in AGENTS.md (#39) From 59e13f17fb17fa8d16f9db5e2020e360f7fc34a8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 20:51:42 +0000 Subject: [PATCH 13/95] Check the callback URL before the JavaScript builder writes it (#40) --- .../flowelements/CallbackUrl.java | 358 ++++++++++++++++++ .../JavaScriptBuilderElement.java | 72 ++-- .../javascriptbuilder/JavaScriptUrlTests.java | 129 +++++++ .../flowelements/CallbackUrlTests.java | 283 ++++++++++++++ 4 files changed, 816 insertions(+), 26 deletions(-) create mode 100644 pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java create mode 100644 pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java new file mode 100644 index 000000000..4118f86f9 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java @@ -0,0 +1,358 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.flowelements; + +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * The callback URL the client script posts to for updated JSON. The protocol + * and host usually come from the request, and the script holds the URL in a + * single-quoted string, so the URL is checked before it is written into the + * script. A URL that is not valid is refused. In the user information, in + * anything after the host and port, and in the endpoint, the characters a + * URL cannot hold or that could end the string or the script are + * percent-encoded. + * @see Specification + */ +final class CallbackUrl { + + /** + * The largest port number. + */ + private static final int MAX_PORT = 65535; + + /** + * The most digits a port can have once its leading zeros are dropped. + */ + private static final int MAX_PORT_DIGITS = 5; + + /** + * Written in place of a lone surrogate, which has no UTF-8 encoding, as a + * browser writes it. + */ + private static final int REPLACEMENT_CHARACTER = 0xFFFD; + + /** + * The printable ASCII characters that are percent-encoded, besides the + * control characters, the space and everything outside ASCII. + */ + private static final String ENCODED = "\"'<>\\^`{|}"; + + private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray(); + + /** + * Text made only of the Unicode White_Space characters, which counts as + * a part that is not set. + */ + private static final Pattern BLANK = Pattern.compile( + "[\\t\\n\\x0B\\f\\r \\u0085\\u00A0\\u1680\\u2000-\\u200A" + + "\\u2028\\u2029\\u202F\\u205F\\u3000]*"); + + /** + * A scheme as RFC 3986 section 3.1 defines one, being a letter followed + * by letters, digits, '+', '-' and '.'. + */ + private static final Pattern SCHEME = + Pattern.compile("[A-Za-z][A-Za-z0-9+.-]*"); + + /** + * A label of a host name, being ASCII letters, digits, '-' and '_'. + */ + private static final Pattern LABEL = Pattern.compile("[A-Za-z0-9_-]+"); + + /** + * An IPv6 zone identifier, being unreserved characters as RFC 3986 + * section 2.3 defines them. + */ + private static final Pattern ZONE = Pattern.compile("[A-Za-z0-9._~-]+"); + + /** + * A port as it follows the host, being ':' and any number of digits. + */ + private static final Pattern PORT = Pattern.compile(":[0-9]*"); + + /** + * One group of an IPv6 address. + */ + private static final Pattern HEX_GROUP = + Pattern.compile("[0-9A-Fa-f]{1,4}"); + + /** + * An IPv4 address as RFC 3986 section 3.2.2 defines one, being four + * numbers from 0 to 255 with no leading zeros. + */ + private static final Pattern IPV4 = Pattern.compile( + "(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])" + + "(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}"); + + /** + * Whether the URL can be written into the script. + */ + final boolean valid; + + /** + * The URL to write into the script when it is valid, and otherwise the + * URL as it was built, for the log. + */ + final String url; + + private CallbackUrl(boolean valid, String url) { + this.valid = valid; + this.url = url; + } + + /** + * Builds the callback URL from its parts and checks it. One slash + * separates the host and the endpoint, being added when neither has one + * and removed when both do. The protocol and the host are written in + * lower case and the rest as given, apart from the percent-encoding. + * @param protocol the protocol, such as https + * @param host the host, which can start with user information and end + * with a port, a path, a query or a fragment + * @param endpoint the endpoint + * @return the callback URL, or null when the protocol, host or endpoint + * is not set + */ + static CallbackUrl of(String protocol, String host, String endpoint) { + if (isBlank(protocol) || isBlank(host) || isBlank(endpoint)) { + return null; + } + boolean hostSlash = host.endsWith("/"); + boolean endpointSlash = endpoint.startsWith("/"); + if (hostSlash == false && endpointSlash == false) { + endpoint = "/" + endpoint; + } else if (hostSlash && endpointSlash) { + endpoint = endpoint.substring(1); + } + String built = protocol + "://" + host + endpoint; + if (SCHEME.matcher(protocol).matches() == false) { + return new CallbackUrl(false, built); + } + int end = authorityEnd(host); + String authority = checkedAuthority(host.substring(0, end)); + if (authority == null) { + return new CallbackUrl(false, built); + } + return new CallbackUrl( + true, + protocol.toLowerCase(Locale.ROOT) + "://" + authority + + encoded(host.substring(end)) + encoded(endpoint)); + } + + /** + * Whether a part of the URL is not set, being null, empty or only + * whitespace. + * @param text the part + * @return true if the part is not set + */ + static boolean isBlank(String text) { + return text == null || BLANK.matcher(text).matches(); + } + + /** + * Where the authority ends, which is at the first '/', '?' or '#' of the + * host, or at its end. + */ + private static int authorityEnd(String host) { + for (int i = 0; i < host.length(); i++) { + char c = host.charAt(i); + if (c == '/' || c == '?' || c == '#') { + return i; + } + } + return host.length(); + } + + /** + * The user information, host and port as they are written into the URL, + * or null when they are not valid. The user information ends at the + * first '@', so a second '@' is part of the host and refused. + */ + private static String checkedAuthority(String authority) { + int at = authority.indexOf('@'); + if (at < 0) { + return checkedHostPort(authority); + } + String hostPort = checkedHostPort(authority.substring(at + 1)); + return hostPort == null + ? null + : encoded(authority.substring(0, at)) + "@" + hostPort; + } + + /** + * The host and optional port in lower case, or null when they are not + * valid. The host is an IPv6 address in brackets with an optional zone + * identifier, or a name of labels separated by dots with a dot allowed + * at the end. The port is ':' followed by digits up to 65535, or by + * nothing. + */ + private static String checkedHostPort(String hostPort) { + String host; + String port; + if (hostPort.startsWith("[")) { + int close = hostPort.indexOf(']'); + if (close < 0) { + return null; + } + String address = hostPort.substring(1, close); + int zone = address.indexOf("%25"); + if (zone >= 0) { + if (ZONE.matcher(address.substring(zone + 3)).matches() == + false) { + return null; + } + address = address.substring(0, zone); + } + if (isIpv6(address) == false) { + return null; + } + host = hostPort.substring(0, close + 1); + port = hostPort.substring(close + 1); + } else { + int colon = hostPort.indexOf(':'); + int end = colon < 0 ? hostPort.length() : colon; + host = hostPort.substring(0, end); + port = hostPort.substring(end); + String name = host.endsWith(".") + ? host.substring(0, host.length() - 1) + : host; + for (String label : name.split("\\.", -1)) { + if (LABEL.matcher(label).matches() == false) { + return null; + } + } + } + if (port.isEmpty() == false && isPort(port) == false) { + return null; + } + return host.toLowerCase(Locale.ROOT) + port; + } + + /** + * Whether the text is ':' followed by digits whose value is no more than + * 65535, or by nothing. + */ + private static boolean isPort(String text) { + if (PORT.matcher(text).matches() == false) { + return false; + } + String digits = text.substring(1).replaceFirst("^0+", ""); + return digits.length() <= MAX_PORT_DIGITS && + (digits.isEmpty() || Integer.parseInt(digits) <= MAX_PORT); + } + + /** + * Whether the text is an IPv6 address as RFC 3986 section 3.2.2 defines + * one. That is eight groups of one to four hexadecimal digits separated + * by ':', where one '::' can stand for one or more groups of zeros and + * the last two groups can be written as an IPv4 address. + */ + private static boolean isIpv6(String text) { + int gap = text.indexOf("::"); + if (gap < 0) { + return groups(text, true) == 8; + } + if (text.indexOf("::", gap + 1) >= 0) { + return false; + } + int head = groups(text.substring(0, gap), false); + int tail = groups(text.substring(gap + 2), true); + return head >= 0 && tail >= 0 && head + tail <= 7; + } + + /** + * The number of 16 bit groups in a run of groups separated by ':', or -1 + * when the run is not valid. An empty run has none, and an IPv4 address + * at the end of the run, where one is allowed, counts as two. + */ + private static int groups(String run, boolean ipv4AtEnd) { + if (run.isEmpty()) { + return 0; + } + String[] parts = run.split(":", -1); + int count = 0; + for (int i = 0; i < parts.length; i++) { + if (HEX_GROUP.matcher(parts[i]).matches()) { + count++; + } else if (ipv4AtEnd && i == parts.length - 1 && + IPV4.matcher(parts[i]).matches()) { + count += 2; + } else { + return -1; + } + } + return count; + } + + /** + * The text with each byte of its UTF-8 encoding percent-encoded where + * the byte is a control character, a space, outside ASCII or one of + * ENCODED, and with a '%' encoded where two hexadecimal digits do not + * follow it. + */ + private static String encoded(String text) { + byte[] bytes = utf8(text); + StringBuilder out = new StringBuilder(bytes.length); + for (int i = 0; i < bytes.length; i++) { + int b = bytes[i] & 0xFF; + boolean keep = b == '%' + ? i + 2 < bytes.length && + isHexDigit(bytes[i + 1]) && isHexDigit(bytes[i + 2]) + : b > ' ' && b < 0x7F && ENCODED.indexOf(b) < 0; + if (keep) { + out.append((char) b); + } else { + out.append('%') + .append(HEX_DIGITS[b >> 4]) + .append(HEX_DIGITS[b & 0x0F]); + } + } + return out.toString(); + } + + private static boolean isHexDigit(byte b) { + return (b >= '0' && b <= '9') || + (b >= 'A' && b <= 'F') || + (b >= 'a' && b <= 'f'); + } + + /** + * The UTF-8 encoding of the text, with each lone surrogate written as + * U+FFFD. + */ + private static byte[] utf8(String text) { + StringBuilder whole = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + int c = text.codePointAt(i); + i += Character.charCount(c); + whole.appendCodePoint( + c >= Character.MIN_SURROGATE && c <= Character.MAX_SURROGATE + ? REPLACEMENT_CHARACTER + : c); + } + return whole.toString().getBytes(StandardCharsets.UTF_8); + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 5ad34579e..f8d828dc5 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -42,6 +42,7 @@ import java.io.*; import java.net.URLEncoder; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static fiftyone.pipeline.core.Constants.*; @@ -68,6 +69,12 @@ public class JavaScriptBuilderElement protected final boolean enableCookies; private final Mustache mustache; + /** + * Set once a callback URL that is not valid has been logged, because a + * misconfigured element would otherwise log on every request. + */ + private final AtomicBoolean invalidUrlLogged = new AtomicBoolean(false); + /** * The object name is written into the script as the name of a global * variable, as a session storage key and inside string literals, with no @@ -392,35 +399,48 @@ private String serializeParameters(Map parameters) { return jsonObject.toString(0); } + /** + * Gets the callback URL, or null when the protocol, host or endpoint is + * not set or the URL is not valid, as {@link CallbackUrl} decides. A URL + * that is not valid is logged as an error the first time only, quoted + * and escaped so that it cannot start a new log line. + */ private String getUrl(String protocol, String host) { - String url = null; - if (protocol != null && !protocol.isEmpty() && - host != null && !host.isEmpty() && - endpoint != null && !endpoint.isEmpty()) { - boolean contextRootPopulated = contextRoot != null && - !contextRoot.isEmpty() && !contextRoot.equals("/"); - - // Make sure that each part of the URL except host starts with a '/' - // and each part except endpoint does NOT end with one. - if (endpoint.charAt(0) != '/') { - endpoint = "/" + endpoint; - } - if (contextRootPopulated && contextRoot.charAt(0) != '/') { - contextRoot = "/" + contextRoot; - } - if (host.charAt(host.length() - 1) == '/') { - host = host.substring(0, host.length() - 1); - } - if (contextRootPopulated && contextRoot.charAt(contextRoot.length() - 1) == '/') { - contextRoot = contextRoot.substring(0, contextRoot.length() - 1); - } - - url = protocol + "://" + host + - (contextRootPopulated ? contextRoot : "") + - endpoint; + CallbackUrl url = CallbackUrl.of(protocol, host, getPath()); + if (url == null) { + return null; + } + if (url.valid) { + return url.url; } + if (invalidUrlLogged.compareAndSet(false, true)) { + logger.error("The generated URL " + JSONObject.quote(url.url) + + " is not valid, so the JavaScript has no callback URL. " + + "Check the JavaScript builder's protocol, host, context " + + "root and endpoint, and the protocol and host evidence. " + + "This is reported once."); + } + return null; + } - return url; + /** + * Gets the endpoint with the context root in front of it, where there is + * a context root. The context root starts with a '/' and does not end + * with one, and a '/' joins it to the endpoint. + */ + private String getPath() { + boolean contextRootPopulated = contextRoot != null && + contextRoot.isEmpty() == false && contextRoot.equals("/") == false; + if (contextRootPopulated == false || CallbackUrl.isBlank(endpoint)) { + return endpoint; + } + String root = contextRoot.charAt(0) == '/' + ? contextRoot + : "/" + contextRoot; + if (root.charAt(root.length() - 1) == '/') { + root = root.substring(0, root.length() - 1); + } + return root + (endpoint.charAt(0) == '/' ? endpoint : "/" + endpoint); } /** diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index 4f3fc0dea..a94604104 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.javascriptbuilder; +import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -145,4 +146,132 @@ public void Url_KeepsContextRoot() throws Exception { assertEquals("https://example.com/app/json", url); } } + + /** + * Loggers for the tests that read what the element logged. No logger is + * behind them, so each one keeps the name it was created with. + */ + private final TestLoggerFactory hostLoggerFactory = + new TestLoggerFactory(null); + + /** + * Renders one script for each host given, all through the same element, + * with the host taken from the request's evidence. + */ + private List renderWithHosts(String... hosts) throws Exception { + TestLoggerFactory factory = hostLoggerFactory; + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(factory) + .setEndpoint("/json") + .build(); + Pipeline pipeline = new PipelineBuilder(factory) + .addFlowElement(new SequenceElementBuilder(factory).build()) + .addFlowElement(new JsonBuilderElementBuilder(factory).build()) + .addFlowElement(jsElement) + .build(); + List scripts = new ArrayList<>(); + for (String host : hosts) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(Constants.EVIDENCE_HOST_KEY, host); + flowData.process(); + scripts.add( + flowData.getFromElement(jsElement).getJavaScript()); + } + } + return scripts; + } + + /** + * The errors the JavaScript builder element logged. + */ + private List builderErrors() { + List errors = new ArrayList<>(); + for (TestLogger logger : hostLoggerFactory.loggers) { + if (JavaScriptBuilderElement.class.getName().equals( + logger.getName())) { + errors.addAll(logger.errorsLogged); + } + } + return errors; + } + + /** + * A host the check refuses gives a script with no callback, and the + * refusal is logged as an error the first time only, whatever the + * requests in between. + */ + @Test + public void Url_RefusedHost_NoCallbackAndLoggedOnce() throws Exception { + List scripts = renderWithHosts( + "exa mple.com", "exa mple.com", "example.com", "exa mple.com"); + + for (int i = 0; i < scripts.size(); i++) { + String script = scripts.get(i); + assertTrue(script.contains("new fiftyoneDegreesManager()"), + "The script was not rendered."); + if (i == 2) { + assertFalse(requestUrls(script).isEmpty(), + "The valid host lost its callback."); + } else { + assertTrue(requestUrls(script).isEmpty(), + "A refused URL reached the script."); + assertFalse(script.contains("exa mple.com"), + "The refused host reached the script."); + } + } + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + assertTrue(errors.get(0).contains("\"https://exa mple.com/json\""), + errors.get(0)); + } + + /** + * The URL is quoted and escaped in the log, so a line break in it cannot + * start a new log line. + */ + @Test + public void Url_RefusedHost_LoggedOnOneLine() throws Exception { + renderWithHosts("a\nb.com\r\u2028"); + + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + String error = errors.get(0); + for (char c : new char[] {'\n', '\r', '\u2028'}) { + assertTrue(error.indexOf(c) < 0, error); + } + assertTrue(error.contains("a\\nb.com\\r\\u2028"), error); + } + + /** + * A host made only of whitespace is not set, so the script has no + * callback and nothing is logged. + */ + @Test + public void Url_BlankHost_NoCallbackAndNothingLogged() throws Exception { + String script = renderWithHosts(" ").get(0); + + assertTrue(requestUrls(script).isEmpty(), + "A blank host gave a callback."); + assertTrue(builderErrors().isEmpty(), + String.join("\n", builderErrors())); + } + + /** + * A host from the request reaches the script as the check writes it, + * in lower case and with anything that could end the string the script + * holds it in percent-encoded. + */ + @Test + public void Url_HostFromEvidence_WrittenAsChecked() throws Exception { + String script = + renderWithHosts("EXAMPLE.com/');alert(1);//").get(0); + + List urls = requestUrls(script); + assertFalse(urls.isEmpty(), "No request URL found in the script."); + for (String url : urls) { + assertEquals("https://example.com/%27);alert(1);//json", url); + } + assertFalse(script.contains("');alert(1)"), + "The host ended the string."); + } } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java new file mode 100644 index 000000000..fb7f6d91d --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java @@ -0,0 +1,283 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.flowelements; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Checks the callback URL against the cases every implementation of the + * JavaScript builder shares, in the same order, so that each one refuses + * and writes the same URLs. + */ +public class CallbackUrlTests { + + private static final String ENDPOINT = "/51dpipeline/json"; + + private static CallbackUrl url(String protocol, String host) { + return CallbackUrl.of(protocol, host, ENDPOINT); + } + + private static void assertValid(String expected, CallbackUrl url, + String given) { + assertNotNull(url, given + " gave no URL"); + assertTrue(url.valid, given + " was refused as " + url.url); + assertEquals(expected, url.url, given); + } + + private static void assertInvalid(CallbackUrl url, String given) { + assertNotNull(url, given + " gave no URL"); + assertFalse(url.valid, given + " was accepted as " + url.url); + } + + @Test + public void CallbackUrl_AddsASingleSlash() { + assertValid( + "https://example.com/51dpipeline/json", + CallbackUrl.of("https", "example.com", "51dpipeline/json"), + "an endpoint without a slash"); + } + + @Test + public void CallbackUrl_KeepsASingleSlashOnTheEndpoint() { + assertValid( + "https://example.com/51dpipeline/json", + CallbackUrl.of("https", "example.com", "/51dpipeline/json"), + "an endpoint with a slash"); + } + + @Test + public void CallbackUrl_CollapsesADoubleSlash() { + assertValid( + "http://example.com/json", + CallbackUrl.of("http", "example.com/", "/json"), + "a slash on both"); + } + + @Test + public void CallbackUrl_MissingWhenAPartIsEmpty() { + assertNull(CallbackUrl.of("https", "", "/json")); + assertNull(CallbackUrl.of("", "example.com", "/json")); + assertNull(CallbackUrl.of("https", "example.com", "")); + assertNull(CallbackUrl.of("https", " ", "/json")); + } + + static Stream acceptedHosts() { + return Stream.of( + Arguments.of("EXAMPLE.com", + "https://example.com/51dpipeline/json"), + Arguments.of("example.com:8080", + "https://example.com:8080/51dpipeline/json"), + Arguments.of("example.com:", + "https://example.com:/51dpipeline/json"), + Arguments.of("example.com:65535", + "https://example.com:65535/51dpipeline/json"), + Arguments.of("a_b.com", + "https://a_b.com/51dpipeline/json"), + Arguments.of("-ab.com", + "https://-ab.com/51dpipeline/json"), + Arguments.of("127.0.0.1:80", + "https://127.0.0.1:80/51dpipeline/json"), + Arguments.of("[::1]", + "https://[::1]/51dpipeline/json"), + Arguments.of("[::1]:8080", + "https://[::1]:8080/51dpipeline/json"), + Arguments.of("[fe80::1%25eth0]", + "https://[fe80::1%25eth0]/51dpipeline/json"), + Arguments.of("user:pw@example.com", + "https://user:pw@example.com/51dpipeline/json"), + Arguments.of("u x@example.com", + "https://u%20x@example.com/51dpipeline/json"), + Arguments.of("u\tx@example.com", + "https://u%09x@example.com/51dpipeline/json"), + Arguments.of("example.com.", + "https://example.com./51dpipeline/json"), + Arguments.of("example.com/path", + "https://example.com/path/51dpipeline/json"), + Arguments.of("example.com?x", + "https://example.com?x/51dpipeline/json"), + Arguments.of("example.com/a b", + "https://example.com/a%20b/51dpipeline/json"), + Arguments.of("example.com/a%zzb", + "https://example.com/a%25zzb/51dpipeline/json"), + Arguments.of("example.com/a%41b", + "https://example.com/a%41b/51dpipeline/json"), + Arguments.of("example.com/\u00fc", + "https://example.com/%C3%BC/51dpipeline/json"), + Arguments.of("example.com/a\nb", + "https://example.com/a%0Ab/51dpipeline/json"), + Arguments.of("example.com/a\u2028b", + "https://example.com/a%E2%80%A8b/51dpipeline/json")); + } + + @ParameterizedTest(name = "[{index}]") + @MethodSource("acceptedHosts") + public void CallbackUrl_AcceptsHost(String host, String expected) { + assertValid(expected, url("https", host), "host " + host); + } + + static Stream refusedHosts() { + return Stream.of( + "example.com:65536", + "example.com:99999", + "example.com:abc", + "exa mple.com", + "example.com');alert(1);//", + "a\"b.com", + "a'b.com", + "a hostsThatCouldEndTheString() { + return Stream.of( + "example.com/');alert(1);//", + "example.com?');alert(1);//", + "example.com#');alert(1);//", + "u'x@example.com", + "u\\x@example.com", + "example.com/a\\b", + "example.com/"); + } + + /** + * A quote, a backslash or a line break never reaches the script, wherever + * in the URL it was given. + */ + @ParameterizedTest(name = "[{index}]") + @MethodSource("hostsThatCouldEndTheString") + public void CallbackUrl_NothingGivenCanEndTheString(String host) { + CallbackUrl url = url("https", host); + assertNotNull(url, "host " + host); + assertTrue(url.valid, "host " + host + " was refused"); + for (char c : new char[] { + '\'', '"', '\\', '<', '>', '\n', '\r', '\u2028', '\u2029'}) { + assertTrue(url.url.indexOf(c) < 0, + "host " + host + " gave " + url.url); + } + } + + @Test + public void CallbackUrl_EncodesTheEndpoint() { + assertValid( + "https://example.com/json%27x", + CallbackUrl.of("https", "example.com", "/json'x"), + "the endpoint /json'x"); + } + + /** + * A host must be ASCII, so an internationalized name is given as + * A-labels, and anything else outside ASCII is refused. + */ + @ParameterizedTest(name = "[{index}]") + @ValueSource(strings = { + "b\u00fccher.de", "a\u00a0b.com", "a\u200bb.com", "a\u2028b.com"}) + public void CallbackUrl_RefusesHostOutsideAscii(String host) { + assertInvalid(url("https", host), "host " + host); + } + + /** + * Not one of the shared cases, because only some languages' strings can + * hold a lone surrogate. It has no UTF-8 encoding, so it is written as + * U+FFFD, as a browser writes it. + */ + @Test + public void CallbackUrl_WritesALoneSurrogateAsTheReplacement() { + assertValid( + "https://example.com/%EF%BF%BD/51dpipeline/json", + url("https", "example.com/" + (char) 0xD800), + "a lone surrogate"); + } +} From 68101da77992cf79df74660a3594ef1132a02490 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 23:29:36 +0000 Subject: [PATCH 14/95] Close every FlowData, Pipeline and metadata collection, and fail the build when one is left open (#44) --- ci/pmd-resource-ruleset.xml | 99 +++ .../onpremise/data/ComponentMetaDataHash.java | 3 +- .../onpremise/data/ProfileMetaDataHash.java | 16 +- .../onpremise/data/PropertyMetaDataHash.java | 3 +- .../hash/engine/onpremise/WrapperHash.java | 7 +- .../onpremise/data/DataValidatorHash.java | 13 +- .../onpremise/data/MetaDataHashTests.java | 8 +- .../onpremise/data/MetaDataHasherHash.java | 75 +- .../onpremise/flowelements/EngineTests.java | 7 +- .../shared/testhelpers/Wrapper.java | 7 +- .../testhelpers/data/MetaDataHasher.java | 8 +- .../shared/testhelpers/data/ValueTests.java | 7 +- .../CloudRequestOriginTests.java | 5 +- ...DetectionOnPremisePipelineBuilderTest.java | 211 +++--- device-detection/pom.xml | 35 + .../examples/console/MetadataOnPrem.java | 135 ++-- .../examples/console/ShareUsageExample.java | 5 +- examples/device-detection/pom.xml | 35 + .../console/GettingStartedCloudMixed.java | 3 + .../examples/console/MetadataOnPrem.java | 103 +-- .../examples/console/MetricsOnPrem.java | 10 +- examples/ip-intelligence/pom.xml | 35 + .../examples/web/GettingStartedApi.java | 15 +- .../web/GettingStartedWebCloudMixed.java | 12 +- .../examples/web/GettingStartedWebCloud.java | 16 +- .../examples/web/GettingStartedWebMixed.java | 12 +- .../examples/web/GettingStartedWebOnPrem.java | 16 +- .../onpremise/data/ComponentMetaDataIPI.java | 3 +- .../onpremise/data/ProfileMetaDataIPI.java | 16 +- .../onpremise/data/PropertyMetaDataIPI.java | 3 +- .../engine/onpremise/WrapperHash.java | 7 +- .../onpremise/data/MetaDataHashTests.java | 8 +- .../onpremise/data/MetaDataHasherHash.java | 75 +- .../onpremise/flowelements/EngineTests.java | 7 +- .../shared/testhelpers/Wrapper.java | 7 +- .../testhelpers/data/MetaDataHasher.java | 8 +- .../CloudRequestOriginTests.java | 5 +- ...elligenceOnPremisePipelineBuilderTest.java | 148 ++-- ip-intelligence/pom.xml | 2 + .../examples/CombiningServices.java | 18 +- .../examples/cloud/GettingStarted.java | 13 +- location/pom.xml | 35 + .../CloudAspectEngineBaseTests.java | 33 +- .../flowelements/CloudRequestEngineTests.java | 71 +- .../core/flowelements/PipelineBuilder.java | 63 +- .../core/flowelements/FlowDataTests.java | 9 +- .../flowelements/ParallelElementsTests.java | 55 +- .../flowelements/PipelineBuilderTests.java | 149 ++-- .../core/flowelements/PipelineTests.java | 45 +- .../integration/PipelineIntegrationTests.java | 34 +- .../flowelements/RecordingElementBuilder.java | 47 ++ .../developerexamples/cloudengine/Main.java | 49 +- .../developerexamples/flowelement/Main.java | 31 +- .../onpremiseengine/Main.java | 32 +- .../developerexamples/usagesharing/Main.java | 40 +- .../fiftyone/data/ComponentMetaData.java | 3 +- .../data/FiftyOneAspectPropertyMetaData.java | 3 +- ...FiftyOneAspectPropertyMetaDataDefault.java | 6 +- .../fiftyone/data/ProfileMetaData.java | 6 +- .../flowelements/SetHeadersElementTests.java | 96 +-- .../flowelements/ShareUsageElementTests.java | 685 +++++++++--------- .../performance/ShareUsageOverheadTests.java | 103 +-- .../javascriptbuilder/CookieTests.java | 15 +- .../javascriptbuilder/JavaScriptUrlTests.java | 23 +- .../jsonbuilder/JsonBuilderTests.java | 8 +- .../mvc/components/FiftyOneInterceptor.java | 25 +- .../web/mvc/FiftyOneInterceptorTests.java | 15 + .../fiftyone/pipeline/web/PipelineFilter.java | 41 +- .../services/PipelineResultServiceCore.java | 23 +- .../pipeline/web/PipelineFilterTests.java | 41 +- pipeline/pom.xml | 35 + 71 files changed, 1864 insertions(+), 1178 deletions(-) create mode 100644 ci/pmd-resource-ruleset.xml create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java diff --git a/ci/pmd-resource-ruleset.xml b/ci/pmd-resource-ruleset.xml new file mode 100644 index 000000000..5f99fad8e --- /dev/null +++ b/ci/pmd-resource-ruleset.xml @@ -0,0 +1,99 @@ + + + + + Fails the build when a FlowData, a Pipeline or a collection of + metadata is not closed on every path. The native results in a FlowData + and the native metadata collections each hold the engine's data set, + so while one is open, closing the Pipeline cannot free the data set. + Java never frees the native results by itself. + + Where something is closed by its owner elsewhere, such as the FlowData + the web integration closes at the end of a request, the code says so + with @SuppressWarnings and a comment naming the owner. + + + + + Every FlowData created is closed by a try-with-resources statement, + unless it is returned to the caller or kept in a field. + + 3 + + + + + + + + + + + + Every CloseableIterable a method returns, such as the profiles or + the components of an engine, is closed by a try-with-resources + statement, unless it is returned to the caller. + + 3 + + + + + + + + + + + + A local variable holding a Pipeline that a builder has just built + is a try-with-resources resource, is closed in a finally block of + the same method, or is returned to the caller. + + 3 + + + + + + + + + + diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java index 30d18118b..63cfdbf6f 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.hash.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.interop.PropertyIterable; import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ComponentMetaDataSwig; @@ -77,7 +78,7 @@ public ProfileMetaData getDefaultProfile() { } @Override - public Iterable getProperties() { + public CloseableIterable getProperties() { return new PropertyIterable( engine, properties, diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java index 7603773ae..afadcd5b2 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java @@ -114,11 +114,19 @@ public ComponentMetaData getComponent() { @Override public String getName() { List values = new ArrayList<>(); - for (ValueMetaData value : getValues()) { - if (value.getProperty().getDisplayOrder() > 0 && - value.getName().equalsIgnoreCase("N/A") == false) { - values.add(value); + // The native collection holds the data set until it is closed. The + // values taken from it are separate objects, so they outlive it. + try (CloseableIterable profileValues = getValues()) { + for (ValueMetaData value : profileValues) { + if (value.getProperty().getDisplayOrder() > 0 && + value.getName().equalsIgnoreCase("N/A") == false) { + values.add(value); + } } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); } ValueMetaData[] valuesArray = new ValueMetaData[0]; valuesArray = values.toArray(valuesArray); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java index 3dbf2b5d5..6a6b5a92a 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.hash.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.interop.Swig; import fiftyone.devicedetection.hash.engine.onpremise.interop.ValueIterable; @@ -136,7 +137,7 @@ public ComponentMetaData getComponent() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return new ValueIterable( engine, engine.getMetaData().getValuesForProperty(source)); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java index 2d034f6ff..21bdb6fb7 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -70,15 +71,15 @@ public Iterable getProperties() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return engine.getValues(); } - public Iterable getProfiles() { + public CloseableIterable getProfiles() { return engine.getProfiles(); } - public Iterable getComponents() { + public CloseableIterable getComponents() { return engine.getComponents(); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java index 336a52141..ddd55d599 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java @@ -29,6 +29,7 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.exceptions.NoValueException; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; @@ -91,8 +92,16 @@ public void validateData(FlowData data, boolean validEvidence) throws NoValueExc // rather than a fixed value, and that valid evidence produces at least // one result. int componentCount = 0; - for (ComponentMetaData component : engine.getComponents()) { - componentCount++; + // The collection holds the engine's data set until it is closed. + try (CloseableIterable components = + engine.getComponents()) { + for (ComponentMetaData component : components) { + componentCount++; + } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); } int matchedUserAgents = elementData.getUserAgents().getValue().size(); assertTrue( diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java index 4d0a9e220..8e4fa034a 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java @@ -65,7 +65,7 @@ public void MetaData_Hash_Reload() throws ExecutionException, InterruptedExcepti * assumption rather than failing. */ @Test - public void MetaData_Performance_Properties() { + public void MetaData_Performance_Properties() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProperties(0, getWrapper()); @@ -83,7 +83,7 @@ public void MetaData_Performance_Properties() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Components() { + public void MetaData_Performance_Components() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashComponents(0, getWrapper()); @@ -101,7 +101,7 @@ public void MetaData_Performance_Components() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Values() { + public void MetaData_Performance_Values() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashValues(0, getWrapper()); @@ -119,7 +119,7 @@ public void MetaData_Performance_Values() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Profiles() { + public void MetaData_Performance_Profiles() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProfiles(0, getWrapper()); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java index f793a46fc..aa5a619ba 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java @@ -25,6 +25,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.Constants; import fiftyone.devicedetection.shared.testhelpers.Wrapper; import fiftyone.devicedetection.shared.testhelpers.data.MetaDataHasher; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -34,7 +35,7 @@ public class MetaDataHasherHash implements MetaDataHasher { @Override - public int hashProperties(int hash, Wrapper wrapper) { + public int hashProperties(int hash, Wrapper wrapper) throws Exception { int i = 0; Iterable properties = wrapper.getProperties(); for (FiftyOneAspectPropertyMetaData property : properties) { @@ -42,11 +43,13 @@ public int hashProperties(int hash, Wrapper wrapper) { hash ^= property.hashCode(); int j = 0; - for (ValueMetaData value : property.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable propertyValues = property.getValues()) { + for (ValueMetaData value : propertyValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; } - j++; } // Match metric property does not have component so don't include if (!property.getCategory().equals(Constants.MatchMetrics.CATEGORY)) { @@ -62,50 +65,60 @@ public int hashProperties(int hash, Wrapper wrapper) { } @Override - public int hashValues(int hash, Wrapper wrapper) { + public int hashValues(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ValueMetaData value : wrapper.getValues()) { - if (i % 100 == 0) { - hash ^= value.hashCode(); - hash ^= value.getProperty().hashCode(); + try (CloseableIterable values = wrapper.getValues()) { + for (ValueMetaData value : values) { + if (i % 100 == 0) { + hash ^= value.hashCode(); + hash ^= value.getProperty().hashCode(); + } + i++; } - i++; } return hash; } @Override - public int hashComponents(int hash, Wrapper wrapper) { - for (ComponentMetaData component : wrapper.getComponents()) { - hash ^= component.hashCode(); - int i = 0; - for (FiftyOneAspectPropertyMetaData property : component.getProperties()) { - if (i % 10 == 0) { - hash ^= property.hashCode(); + public int hashComponents(int hash, Wrapper wrapper) throws Exception { + try (CloseableIterable components = wrapper.getComponents()) { + for (ComponentMetaData component : components) { + hash ^= component.hashCode(); + int i = 0; + try (CloseableIterable componentProperties = component.getProperties()) { + for (FiftyOneAspectPropertyMetaData property : componentProperties) { + if (i % 10 == 0) { + hash ^= property.hashCode(); + } + i++; + } } - i++; + hash ^= component.getDefaultProfile().hashCode(); } - hash ^= component.getDefaultProfile().hashCode(); } return hash; } @Override - public int hashProfiles(int hash, Wrapper wrapper) { + public int hashProfiles(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ProfileMetaData profile : wrapper.getProfiles()) { - if (i % 100 == 0) { - hash ^= profile.hashCode(); - hash ^= profile.getComponent().hashCode(); - int j = 0; - for (ValueMetaData value : profile.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable profiles = wrapper.getProfiles()) { + for (ProfileMetaData profile : profiles) { + if (i % 100 == 0) { + hash ^= profile.hashCode(); + hash ^= profile.getComponent().hashCode(); + int j = 0; + try (CloseableIterable profileValues = profile.getValues()) { + for (ValueMetaData value : profileValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; + } } - j++; } + i++; } - i++; } return hash; } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index 0b58bc098..14b30de62 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -62,9 +62,10 @@ private void testDataSourceTier(String fileName) throws Exception { assumeTrue(e.getMessage(), false); } - DeviceDetectionHashEngine engine = createEngine(dataFile); - String tier = engine.getDataSourceTier(); - engine.close(); + String tier; + try (DeviceDetectionHashEngine engine = createEngine(dataFile)) { + tier = engine.getDataSourceTier(); + } if (fileName.equals(LITE_HASH_DATA_FILE_PATH)) { assertEquals("Lite", tier); diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java index 45fc54abe..82a7b6454 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java @@ -24,6 +24,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -38,11 +39,11 @@ public interface Wrapper extends Closeable { Iterable getProperties(); - Iterable getValues(); + CloseableIterable getValues(); - Iterable getProfiles(); + CloseableIterable getProfiles(); - Iterable getComponents(); + CloseableIterable getComponents(); FiftyOneAspectEngine getEngine(); } \ No newline at end of file diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java index 49240aecd..17fd7796e 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java @@ -25,11 +25,11 @@ import fiftyone.devicedetection.shared.testhelpers.Wrapper; public interface MetaDataHasher { - int hashProperties(int hash, Wrapper wrapper); + int hashProperties(int hash, Wrapper wrapper) throws Exception; - int hashValues(int hash, Wrapper wrapper); + int hashValues(int hash, Wrapper wrapper) throws Exception; - int hashComponents(int hash, Wrapper wrapper); + int hashComponents(int hash, Wrapper wrapper) throws Exception; - int hashProfiles(int hash, Wrapper wrapper); + int hashProfiles(int hash, Wrapper wrapper) throws Exception; } diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java index 148e3a6e0..6826fb4d2 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java @@ -30,6 +30,7 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; @@ -67,8 +68,10 @@ public static void matchedUserAgents(Wrapper wrapper) throws Exception { // depends on the data file, so assert the bound here and validate // every matched substring below. int componentCount = 0; - for (ComponentMetaData component : wrapper.getComponents()) { - componentCount++; + try (CloseableIterable components = wrapper.getComponents()) { + for (ComponentMetaData component : components) { + componentCount++; + } } int matchedCount = device.getUserAgents().getValue().size(); assertTrue( diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java index 72e4e2335..bb311b1b3 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java @@ -86,12 +86,11 @@ public void ResourceKeyWithOrigin() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); try { - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory, http) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory, http) .useCloud(resourceKey) .setCloudRequestOrigin(this.origin) .build(); - - try(FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { data.addEvidence("query.user-agent", "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL " + "Build/OPD1.170816.004) AppleWebKit/537.36 " + diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java index c9430a527..21b9e5bd4 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java @@ -87,12 +87,13 @@ private ConfigHashSwig getConfigHashSwig(Pipeline pipeline) throws NoSuchFieldEx @Test public void setFilename_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setFilename(HASH_DATA_FILE_NAME, true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(HASH_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); - assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(HASH_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); + assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + } } @Test @@ -118,210 +119,232 @@ public void setFilename_OtherExtension_ExceptionThrown() { @Test public void setEngineData_PropertyPropagatedToDataFileConfiguration() throws Exception { byte[] data = Files.readAllBytes(datafile.toPath()); - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(new byte[]{}, Enums.DeviceDetectionAlgorithm.Hash) .setEngineData(data) .setShareUsage(false) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(data, dataFileConfiguration.getData()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(data, dataFileConfiguration.getData()); + } } @Test public void setEngineData_FilenameIsSet_PropertyNotPropagated() throws Exception { byte[] data = Files.readAllBytes(datafile.toPath()); - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setEngineData(data) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertNull(dataFileConfiguration.getData()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertNull(dataFileConfiguration.getData()); + } } @Test public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(false) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(1, flowElements.size()); - assertFalse(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(1, flowElements.size()); + assertFalse(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setShareUsage_True_SharedFlowElementAdded() throws Exception { - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(true) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(2, flowElements.size()); - assertTrue(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(2, flowElements.size()); + assertTrue(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setAutoUpdate(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + } } @Test public void setDataFileSystemWatcher_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataFileSystemWatcher(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + } } @Test public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); - Pipeline pipeline = builder + try (Pipeline ignored = builder .setAutoUpdate(true) .setDataUpdateService(dataUpdateServiceDefault) - .build(); - verify(dataUpdateServiceDefault).registerDataFile(any()); + .build()) { + verify(dataUpdateServiceDefault).registerDataFile(any()); + } } @Test public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getUpdateOnStartup()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getUpdateOnStartup()); + } } @Test public void setUpdatePollingInterval_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingInterval(10) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdatePollingIntervalMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingIntervalMillis(10000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdateRandomisationMax_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMax(123) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setUpdateRandomisationMaxMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMaxMillis(123000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setDataUpdateLicenseKey_PropertyPropagatedToDataFileConfiguration() throws Exception { String key = "licence_key"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateLicenseKey(key) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + } } @Test public void setConcurrency_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setConcurrency(4) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertEquals(4, swig.getConcurrency()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertEquals(4, swig.getConcurrency()); + } } @Test public void setDifference_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDifference(123) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertEquals(123, swig.getDifference()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertEquals(123, swig.getDifference()); + } } @Test public void setAllowUnmatched_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setAllowUnmatched(true) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertTrue(swig.getAllowUnmatched()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertTrue(swig.getAllowUnmatched()); + } } @Test public void setDrift_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDrift(123) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertEquals(123, swig.getDrift()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertEquals(123, swig.getDrift()); + } } @Test public void setUsePerformanceGraph_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUsePerformanceGraph(true) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertTrue(swig.getUsePerformanceGraph()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertTrue(swig.getUsePerformanceGraph()); + } } @Test public void setUsePredictiveGraph_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUsePredictiveGraph(true) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertTrue(swig.getUsePredictiveGraph()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertTrue(swig.getUsePredictiveGraph()); + } } @Test public void setDataUpdateUrl_PropertyPropagatedToDataFileConfiguration() throws Exception { String url = "https://update.me.com"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrl(url) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + } } @Test public void setDataUpdateVerifyMd5_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateVerifyMd5(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getVerifyMd5()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getVerifyMd5()); + } } @Test public void setDataUpdateUrlFormatter_PropertyPropagatedToDataFileConfiguration() throws Exception { DataUpdateUrlFormatter urlFormatter = mock(DataUpdateUrlFormatter.class); - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrlFormatter(urlFormatter) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + } } } \ No newline at end of file diff --git a/device-detection/pom.xml b/device-detection/pom.xml index ce56dd201..b255d1e53 100644 --- a/device-detection/pom.xml +++ b/device-detection/pom.xml @@ -56,6 +56,7 @@ 3.7.1 6.5.3 0.7.0 + 3.26.0 true false @@ -308,6 +309,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 5b6939705..350172405 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -21,11 +21,13 @@ * ********************************************************************* */ package fiftyone.devicedetection.examples.console; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; import fiftyone.pipeline.engines.fiftyone.data.ValueMetaData; @@ -134,71 +136,92 @@ private static void outputEvidenceKeyDetails(DeviceDetectionHashEngine ddEngine, } private static void outputProfileDetails(DeviceDetectionHashEngine ddEngine, - PrintWriter output) { - // Group the profiles by component and then output the number of profiles + PrintWriter output) throws Exception { + // Group the profiles by component and then output the number of profiles // for each component. - Map> groups = - StreamSupport.stream(ddEngine.getProfiles().spliterator(), false) - .collect(Collectors.groupingBy(p -> p.getComponent().getName())); - groups.forEach((k,v)->output.format("%s Profiles: %d\n", k , v.size())); + // + // A try-with-resource block MUST be used for the collections of + // metadata returned by the engine. Each one holds a reference to the + // engine's data set until it is closed, so if it is left open the + // data set cannot be freed when the engine is closed. + Map groups; + try (CloseableIterable profiles = ddEngine.getProfiles()) { + groups = StreamSupport.stream(profiles.spliterator(), false) + .collect(Collectors.groupingBy( + p -> p.getComponent().getName(), + Collectors.counting())); + } + groups.forEach((k,v)->output.format("%s Profiles: %d\n", k , v)); } // Output the component name as well as a list of all the associated properties. // If we're outputting to console then we also add some formatting to make it // more readable. - private static void outputComponents(DeviceDetectionHashEngine ddEngine, PrintWriter output){ - ddEngine.getComponents().forEach(c -> { - output.println("Component - "+ c.getName()); - outputProperties(c, output); - }); + private static void outputComponents(DeviceDetectionHashEngine ddEngine, PrintWriter output) + throws Exception { + // See outputProfileDetails for why this is a try-with-resource block. + try (CloseableIterable components = ddEngine.getComponents()) { + for (ComponentMetaData c : components) { + output.println("Component - "+ c.getName()); + outputProperties(c, output); + } + } } - private static void outputProperties(ComponentMetaData component, PrintWriter output) { - if (component.getProperties().iterator().hasNext() == false) { - output.println(" ... no properties"); - return; + private static void outputProperties(ComponentMetaData component, PrintWriter output) + throws Exception { + // The properties of a component, and the values of a property, are + // collections that are closed in the same way. + try (CloseableIterable properties = + component.getProperties()) { + if (properties.iterator().hasNext() == false) { + output.println(" ... no properties"); + return; + } + for (FiftyOneAspectPropertyMetaData property : properties) { + // Output some details about the property. + // If we're outputting to console then we also add some formatting to make it + // more readable. + output.format(" Property - %s [Category: %s] (%s)\n " + + "Description: %s\n", + property.getName(), + property.getCategory(), + property.getType().getName(), + property.getDescription()); + + // Next, output a list of the possible values this property can have. + // Most properties in the Device Metrics category do not have defined + // values so exclude them. + if (property.getCategory().equals("Device Metrics")==false) { + StringBuilder values = new StringBuilder(" Possible " + + "values: "); + try (CloseableIterable propertyValues = + property.getValues()) { + Spliterator spliterator2 = + propertyValues.spliterator(); + StreamSupport.stream(spliterator2, false) + .limit(20) + .forEach(value -> { + // add value + values.append(truncateToNl(value.getName())); + // add description if exists + String d = value.getDescription(); + if (Objects.nonNull(d) && d.isEmpty() == false) { + values.append("(") + .append(d) + .append(")"); + } + values.append(","); + }); + + if (spliterator2.estimateSize() > 20) { + values.append(" + more ..."); + } + } + output.println(values); + } + } } - component.getProperties() - .forEach(property-> { - // Output some details about the property. - // If we're outputting to console then we also add some formatting to make it - // more readable. - output.format(" Property - %s [Category: %s] (%s)\n " + - "Description: %s\n", - property.getName(), - property.getCategory(), - property.getType().getName(), - property.getDescription()); - - // Next, output a list of the possible values this property can have. - // Most properties in the Device Metrics category do not have defined - // values so exclude them. - if (property.getCategory().equals("Device Metrics")==false) { - StringBuilder values = new StringBuilder(" Possible " + - "values: "); - Spliterator spliterator2 = - property.getValues().spliterator(); - StreamSupport.stream(spliterator2, false) - .limit(20) - .forEach(value -> { - // add value - values.append(truncateToNl(value.getName())); - // add description if exists - String d = value.getDescription(); - if (Objects.nonNull(d) && d.isEmpty() == false) { - values.append("(") - .append(d) - .append(")"); - } - values.append(","); - }); - - if (spliterator2.estimateSize() > 20) { - values.append(" + more ..."); - } - output.println(values); - } - }); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java index 621ef0761..50d44c364 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java @@ -137,9 +137,12 @@ public static void run(InputStream is) throws Exception { // .setIncludedQueryStringParameter("campaign") .build(); - // Build a pipeline with only the ShareUsageElement + // Build a pipeline with only the ShareUsageElement. The pipeline is + // asked to close the element, which sends any data still queued, as + // it is not closed anywhere else. try (Pipeline pipeline = new PipelineBuilder() .addFlowElement(shareUsageElement) + .setAutoCloseElements(true) .build()) { logger.info("Pipeline built successfully with ShareUsageElement"); diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index d7617e6e6..fa55e5534 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -62,6 +62,7 @@ 2.0.20 1.3.16 + 3.26.0 @@ -165,6 +166,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java index e5bf11069..efca03fdc 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java @@ -144,10 +144,13 @@ public static void run(String resourceKey, IPIntelligenceCloudEngine ipIntelligenceEngine = new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + // The engines are not closed anywhere else, so the pipeline is asked + // to close them when it is closed. try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(cloudRequestEngine) .addFlowElement(deviceDetectionEngine) .addFlowElement(ipIntelligenceEngine) + .setAutoCloseElements(true) .build()) { // carry out some sample detections diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 7a96b562f..6e127722d 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -21,6 +21,7 @@ * ********************************************************************* */ package fiftyone.ipintelligence.examples.console; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; @@ -175,59 +176,67 @@ private static void outputComponents(IPIntelligenceOnPremiseEngine engine, Print throws Exception { // See outputProfileDetails for why this is a try-with-resource block. try (CloseableIterable components = engine.getComponents()) { - components.forEach(c -> { + for (ComponentMetaData c : components) { output.println("Component - "+ c.getName()); outputProperties(c, output); - }); + } } } - private static void outputProperties(ComponentMetaData component, PrintWriter output) { - if (component.getProperties().iterator().hasNext() == false) { - output.println(" ... no properties"); - return; + private static void outputProperties(ComponentMetaData component, PrintWriter output) + throws Exception { + // The properties of a component, and the values of a property, are + // collections that are closed in the same way. + try (CloseableIterable properties = + component.getProperties()) { + if (properties.iterator().hasNext() == false) { + output.println(" ... no properties"); + return; + } + for (FiftyOneAspectPropertyMetaData property : properties) { + // Output some details about the property. + // If we're outputting to console then we also add some formatting to make it + // more readable. + output.format(" Property - %s [Category: %s] (%s)\n " + + "Description: %s\n", + property.getName(), + property.getCategory(), + property.getType().getName(), + property.getDescription()); + + // Next, output a list of the possible values this property can have. + // Most properties in the Metrics category do not have defined + // values so exclude them. + if (property.getCategory().equals("Metrics")==false) { + StringBuilder values = new StringBuilder(" Possible " + + "values: "); + try (CloseableIterable propertyValues = + property.getValues()) { + Spliterator spliterator2 = + propertyValues.spliterator(); + StreamSupport.stream(spliterator2, false) + .limit(20) + .forEach(value -> { + // add value + values.append(truncateToNl(value.getName())); + // add description if exists + String d = value.getDescription(); + if (Objects.nonNull(d) && d.isEmpty() == false) { + values.append("(") + .append(d) + .append(")"); + } + values.append(","); + }); + + if (spliterator2.estimateSize() > 20) { + values.append(" + more ..."); + } + } + output.println(values); + } + } } - component.getProperties() - .forEach(property-> { - // Output some details about the property. - // If we're outputting to console then we also add some formatting to make it - // more readable. - output.format(" Property - %s [Category: %s] (%s)\n " + - "Description: %s\n", - property.getName(), - property.getCategory(), - property.getType().getName(), - property.getDescription()); - - // Next, output a list of the possible values this property can have. - // Most properties in the Metrics category do not have defined - // values so exclude them. - if (property.getCategory().equals("Metrics")==false) { - StringBuilder values = new StringBuilder(" Possible " + - "values: "); - Spliterator spliterator2 = - property.getValues().spliterator(); - StreamSupport.stream(spliterator2, false) - .limit(20) - .forEach(value -> { - // add value - values.append(truncateToNl(value.getName())); - // add description if exists - String d = value.getDescription(); - if (Objects.nonNull(d) && d.isEmpty() == false) { - values.append("(") - .append(d) - .append(")"); - } - values.append(","); - }); - - if (spliterator2.estimateSize() > 20) { - values.append(" + more ..."); - } - output.println(values); - } - }); } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 6c66b486f..347c35b60 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -507,11 +507,13 @@ private static String getValue(ProfileMetaData profile, String name) String result = null; // Every value is closed, not only the first, as each one wraps a // native object of its own. - for (ValueMetaData value : profile.getValues(name)) { - if (result == null) { - result = value.getName(); + try (CloseableIterable values = profile.getValues(name)) { + for (ValueMetaData value : values) { + if (result == null) { + result = value.getName(); + } + value.close(); } - value.close(); } return result; } diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 945f34223..2e3624a91 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -63,6 +63,7 @@ 2.0.20 1.3.16 + 3.26.0 @@ -165,6 +166,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 6e3fa2e8a..9887afc57 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -127,16 +127,29 @@ public void init() throws ServletException { .setAutoUpdate(false) .build(dataFileLocation, false); // The SequenceElement provides the sequence evidence required by - // the JsonBuilderElement. + // the JsonBuilderElement. The pipeline closes all three elements, + // the engine and its data file included, when destroy closes it. pipeline = new PipelineBuilder(LoggerFactory.getILoggerFactory()) .addFlowElement(new SequenceElementBuilder( LoggerFactory.getILoggerFactory()).build()) .addFlowElement(engine) .addFlowElement(new JsonBuilderElementBuilder( LoggerFactory.getILoggerFactory()).build()) + .setAutoCloseElements(true) .build(); DataFileHelper.logDataFileInfo(engine); } catch (Exception e) { + // A servlet whose init fails is never destroyed, so whatever was + // built is closed here. + try { + if (pipeline != null) { + pipeline.close(); + } else if (engine != null) { + engine.close(); + } + } catch (Exception closeException) { + e.addSuppressed(closeException); + } throw new ServletException("Failed to create pipeline from data file '" + dataFileLocation + "'", e); } diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java index 4090fc110..977e3b3fa 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java @@ -118,7 +118,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // Get the IP address parameter from the request for custom lookup String inputIpAddress = request.getParameter("client-ip"); - // The detection has already been carried out by the PipelineFilter + // The detection has already been carried out by the PipelineFilter, + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -149,14 +151,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, deviceData, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java index dbcb50e7f..2e0fbb58f 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java @@ -47,8 +47,9 @@ * The `PipelineFilter` is used to intercept requests and perform IP Intelligence. The results * will be stored in the HttpServletRequest object. * - * The results of the lookup can be accessed by using a FlowDataProvider which - * is responsible for managing the lifecycle of the flowData - do NOT dispose + * The results of the lookup can be accessed by using a FlowDataProvider. The + * PipelineFilter closes the FlowData at the end of the request, so it is not + * closed by the example. * ```{java} * FlowData flowData = flowDataProvider.getFlowData(request); * IPIntelligenceData ipData = flowData.get(IPIntelligenceData.class); @@ -137,7 +138,8 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String inputIpAddress = request.getParameter("client-ip"); // The lookup has already been carried out by the PipelineFilter - // which is responsible for the lifecycle of the flowData - do NOT dispose + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -165,14 +167,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 5b7331767..b6addc8d7 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -140,7 +140,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // Get the IP address parameter from the request for custom lookup String inputIpAddress = request.getParameter("client-ip"); - // The detection has already been carried out by the PipelineFilter + // The detection has already been carried out by the PipelineFilter, + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -171,14 +173,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, deviceData, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index d73bafdf8..ec48e2df9 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -45,8 +45,9 @@ * The `PipelineFilter` is used to intercept requests and perform IP Intelligence. The results * will be stored in the HttpServletRequest object. * - * The results of the lookup can be accessed by using a FlowDataProvider which - * is responsible for managing the lifecycle of the flowData - do NOT dispose + * The results of the lookup can be accessed by using a FlowDataProvider. The + * PipelineFilter closes the FlowData at the end of the request, so it is not + * closed by the example. * ```{java} * FlowData flowData = flowDataProvider.getFlowData(request); * IPIntelligenceData ipData = flowData.get(IPIntelligenceData.class); @@ -155,7 +156,8 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String inputIpAddress = request.getParameter("client-ip"); // The lookup has already been carried out by the PipelineFilter - // which is responsible for the lifecycle of the flowData - do NOT dispose + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -183,14 +185,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java index 7ee5100d7..528e1d737 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.interop.PropertyIterable; import fiftyone.ipintelligence.engine.onpremise.interop.swig.ComponentMetaDataSwig; @@ -77,7 +78,7 @@ public ProfileMetaData getDefaultProfile() { } @Override - public Iterable getProperties() { + public CloseableIterable getProperties() { return new PropertyIterable( engine, properties, diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java index b9ee322df..3f2645723 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java @@ -114,11 +114,19 @@ public ComponentMetaData getComponent() { @Override public String getName() { List values = new ArrayList<>(); - for (ValueMetaData value : getValues()) { - if (value.getProperty().getDisplayOrder() > 0 && - value.getName().equalsIgnoreCase("N/A") == false) { - values.add(value); + // The native collection holds the data set until it is closed. The + // values taken from it are separate objects, so they outlive it. + try (CloseableIterable profileValues = getValues()) { + for (ValueMetaData value : profileValues) { + if (value.getProperty().getDisplayOrder() > 0 && + value.getName().equalsIgnoreCase("N/A") == false) { + values.add(value); + } } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); } ValueMetaData[] valuesArray = new ValueMetaData[0]; valuesArray = values.toArray(valuesArray); diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java index 527df5b74..eabb2aecb 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.interop.Swig; import fiftyone.ipintelligence.engine.onpremise.interop.ValueIterable; @@ -136,7 +137,7 @@ public ComponentMetaData getComponent() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return new ValueIterable( engine, engine.getMetaData().getValuesForProperty(source)); diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java index d9e861c0f..a001f6e34 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -69,15 +70,15 @@ public Iterable getProperties() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return engine.getValues(); } - public Iterable getProfiles() { + public CloseableIterable getProfiles() { return engine.getProfiles(); } - public Iterable getComponents() { + public CloseableIterable getComponents() { return engine.getComponents(); } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java index d229af08b..f9f9a03eb 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java @@ -67,7 +67,7 @@ public void MetaData_Hash_Reload() throws ExecutionException, InterruptedExcepti * assumption rather than failing. */ @Test - public void MetaData_Performance_Properties() { + public void MetaData_Performance_Properties() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProperties(0, getWrapper()); @@ -85,7 +85,7 @@ public void MetaData_Performance_Properties() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Components() { + public void MetaData_Performance_Components() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashComponents(0, getWrapper()); @@ -103,7 +103,7 @@ public void MetaData_Performance_Components() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Values() { + public void MetaData_Performance_Values() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashValues(0, getWrapper()); @@ -121,7 +121,7 @@ public void MetaData_Performance_Values() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Profiles() { + public void MetaData_Performance_Profiles() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProfiles(0, getWrapper()); diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java index 4683157a5..2bd50f8b6 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java @@ -24,6 +24,7 @@ import fiftyone.ipintelligence.shared.testhelpers.Wrapper; import fiftyone.ipintelligence.shared.testhelpers.data.MetaDataHasher; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -33,7 +34,7 @@ public class MetaDataHasherHash implements MetaDataHasher { @Override - public int hashProperties(int hash, Wrapper wrapper) { + public int hashProperties(int hash, Wrapper wrapper) throws Exception { int i = 0; Iterable properties = wrapper.getProperties(); for (FiftyOneAspectPropertyMetaData property : properties) { @@ -41,11 +42,13 @@ public int hashProperties(int hash, Wrapper wrapper) { hash ^= property.hashCode(); int j = 0; - for (ValueMetaData value : property.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable propertyValues = property.getValues()) { + for (ValueMetaData value : propertyValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; } - j++; } if (property.getDefaultValue() != null) { hash ^= property.getDefaultValue().hashCode(); @@ -57,50 +60,60 @@ public int hashProperties(int hash, Wrapper wrapper) { } @Override - public int hashValues(int hash, Wrapper wrapper) { + public int hashValues(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ValueMetaData value : wrapper.getValues()) { - if (i % 100 == 0) { - hash ^= value.hashCode(); - hash ^= value.getProperty().hashCode(); + try (CloseableIterable values = wrapper.getValues()) { + for (ValueMetaData value : values) { + if (i % 100 == 0) { + hash ^= value.hashCode(); + hash ^= value.getProperty().hashCode(); + } + i++; } - i++; } return hash; } @Override - public int hashComponents(int hash, Wrapper wrapper) { - for (ComponentMetaData component : wrapper.getComponents()) { - hash ^= component.hashCode(); - int i = 0; - for (FiftyOneAspectPropertyMetaData property : component.getProperties()) { - if (i % 10 == 0) { - hash ^= property.hashCode(); + public int hashComponents(int hash, Wrapper wrapper) throws Exception { + try (CloseableIterable components = wrapper.getComponents()) { + for (ComponentMetaData component : components) { + hash ^= component.hashCode(); + int i = 0; + try (CloseableIterable componentProperties = component.getProperties()) { + for (FiftyOneAspectPropertyMetaData property : componentProperties) { + if (i % 10 == 0) { + hash ^= property.hashCode(); + } + i++; + } } - i++; + hash ^= component.getDefaultProfile().hashCode(); } - hash ^= component.getDefaultProfile().hashCode(); } return hash; } @Override - public int hashProfiles(int hash, Wrapper wrapper) { + public int hashProfiles(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ProfileMetaData profile : wrapper.getProfiles()) { - if (i % 100 == 0) { - hash ^= profile.hashCode(); - hash ^= profile.getComponent().hashCode(); - int j = 0; - for (ValueMetaData value : profile.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable profiles = wrapper.getProfiles()) { + for (ProfileMetaData profile : profiles) { + if (i % 100 == 0) { + hash ^= profile.hashCode(); + hash ^= profile.getComponent().hashCode(); + int j = 0; + try (CloseableIterable profileValues = profile.getValues()) { + for (ValueMetaData value : profileValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; + } } - j++; } + i++; } - i++; } return hash; } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java index 4f0d9005b..e12f6cc10 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java @@ -62,9 +62,10 @@ private void testDataSourceTier(String fileName) throws Exception { assumeTrue(e.getMessage(), false); } - IPIntelligenceOnPremiseEngine engine = createEngine(dataFile); - String tier = engine.getDataSourceTier(); - engine.close(); + String tier; + try (IPIntelligenceOnPremiseEngine engine = createEngine(dataFile)) { + tier = engine.getDataSourceTier(); + } if (fileName.equals(LITE_IPI_DATA_FILE_NAME)) { assertEquals("Lite", tier); diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java index 39378f9fa..d74e56ff7 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java @@ -24,6 +24,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -38,11 +39,11 @@ public interface Wrapper extends Closeable { Iterable getProperties(); - Iterable getValues(); + CloseableIterable getValues(); - Iterable getProfiles(); + CloseableIterable getProfiles(); - Iterable getComponents(); + CloseableIterable getComponents(); FiftyOneAspectEngine getEngine(); } \ No newline at end of file diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java index 43ffbbfa7..da6e9b7bd 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java @@ -25,11 +25,11 @@ import fiftyone.ipintelligence.shared.testhelpers.Wrapper; public interface MetaDataHasher { - int hashProperties(int hash, Wrapper wrapper); + int hashProperties(int hash, Wrapper wrapper) throws Exception; - int hashValues(int hash, Wrapper wrapper); + int hashValues(int hash, Wrapper wrapper) throws Exception; - int hashComponents(int hash, Wrapper wrapper); + int hashComponents(int hash, Wrapper wrapper) throws Exception; - int hashProfiles(int hash, Wrapper wrapper); + int hashProfiles(int hash, Wrapper wrapper) throws Exception; } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java index c4d13d293..54b262730 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java @@ -88,12 +88,11 @@ public void ResourceKeyWithOrigin() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); try { - Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory, http) + try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory, http) .useCloud(resourceKey) .setCloudRequestOrigin(this.origin) .build(); - - try(FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { data.addEvidence("query.client-ip", "8.8.8.8") .process(); } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java index 98b386e97..d220eb776 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java @@ -88,12 +88,13 @@ private ConfigIpiSwig getConfigIpiSwig(Pipeline pipeline) throws NoSuchFieldExce @Test public void setFilename_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setFilename(IPI_DATA_FILE_NAME, true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(IPI_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); - assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(IPI_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); + assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + } } @Test @@ -118,144 +119,159 @@ public void setFilename_OtherExtension_ExceptionThrown() { @Test public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { - Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) + try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(false) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(1, flowElements.size()); - assertFalse(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(1, flowElements.size()); + assertFalse(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setShareUsage_True_SharedFlowElementAdded() throws Exception { - Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) + try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(true) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(2, flowElements.size()); - assertTrue(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(2, flowElements.size()); + assertTrue(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setAutoUpdate(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + } } @Test public void setDataFileSystemWatcher_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataFileSystemWatcher(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + } } @Test public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); - builder + try (Pipeline ignored = builder .setAutoUpdate(true) .setDataUpdateService(dataUpdateServiceDefault) - .build(); - verify(dataUpdateServiceDefault).registerDataFile(any()); + .build()) { + verify(dataUpdateServiceDefault).registerDataFile(any()); + } } @Test public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getUpdateOnStartup()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getUpdateOnStartup()); + } } @Test public void setUpdatePollingInterval_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingInterval(10) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdatePollingIntervalMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingIntervalMillis(10000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdateRandomisationMax_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMax(123) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setUpdateRandomisationMaxMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMaxMillis(123000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setDataUpdateLicenseKey_PropertyPropagatedToDataFileConfiguration() throws Exception { String key = "licence_key"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateLicenseKey(key) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + } } @Test public void setConcurrency_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setConcurrency(4) - .build(); - ConfigIpiSwig swig = getConfigIpiSwig(pipeline); - assertEquals(4, swig.getConcurrency()); + .build()) { + ConfigIpiSwig swig = getConfigIpiSwig(pipeline); + assertEquals(4, swig.getConcurrency()); + } } @Test public void setDataUpdateUrl_PropertyPropagatedToDataFileConfiguration() throws Exception { String url = "https://update.me.com"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrl(url) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + } } @Test public void setDataUpdateVerifyMd5_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateVerifyMd5(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getVerifyMd5()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getVerifyMd5()); + } } @Test public void setDataUpdateUrlFormatter_PropertyPropagatedToDataFileConfiguration() throws Exception { DataUpdateUrlFormatter urlFormatter = mock(DataUpdateUrlFormatter.class); - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrlFormatter(urlFormatter) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + } } } \ No newline at end of file diff --git a/ip-intelligence/pom.xml b/ip-intelligence/pom.xml index 16a23b1a3..1355b5932 100644 --- a/ip-intelligence/pom.xml +++ b/ip-intelligence/pom.xml @@ -179,6 +179,8 @@ ${maven.multiModuleProjectDirectory}/ci/pmd-ruleset.xml + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml true diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java index 748499f05..c3088a99f 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java @@ -74,12 +74,14 @@ * .build(Enums.GeoLocationProvider.FiftyOneDegrees); * ``` * - * 2. Build a new Pipeline containing the engines which were just created. + * 2. Build a new Pipeline containing the engines which were just created, + * and ask it to close them when it is closed. * ``` * Pipeline pipeline = new PipelineBuilder(loggerFactory) * .addFlowElement(cloudRequestEngine) * .addFlowElement(deviceDetectionEngine) * .addFlowElement(locationEngine) + * .setAutoCloseElements(true) * .build(); * ``` * @@ -92,9 +94,10 @@ * .build(); * ``` * 3. Create a new FlowData instance ready to be populated with evidence for - * the Pipeline. + * the Pipeline, in a try-with-resources block so that it is closed when it + * is finished with. * ``` - * FlowData data = pipeline.createFlowData(); + * try (FlowData data = pipeline.createFlowData()) { * ``` * * 4. Process a longitude, latitude and User-Agent to retrieve the values @@ -148,14 +151,16 @@ public static void main(String[] args) throws Exception { new GeoLocationCloudEngineBuilder(loggerFactory) .build(Enums.GeoLocationProvider.FiftyOneDegrees); - Pipeline pipeline = + // The engines are not closed anywhere else, so the pipeline is + // asked to close them when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(cloudRequestEngine) .addFlowElement(deviceDetectionEngine) .addFlowElement(locationEngine) + .setAutoCloseElements(true) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData .addEvidence(EVIDENCE_HEADER_USERAGENT_KEY, mobileUserAgent) .addEvidence(EVIDENCE_GEO_LAT_PARAM_KEY, "51.458048") @@ -173,7 +178,6 @@ public static void main(String[] args) throws Exception { System.out.println("Country: " + country.toString()); System.out.println("IsMobile: " + isMobile.toString()); } - pipeline.close(); } } diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java index 553517c6a..8c86dfe01 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java @@ -57,9 +57,10 @@ * ``` * * 2. Create a new FlowData instance ready to be populated with evidence for - * the Pipeline. + * the Pipeline, in a try-with-resources block so that it is closed when it + * is finished with. * ``` - * FlowData data = pipeline.createFlowData(); + * try (FlowData data = pipeline.createFlowData()) { * ``` * * 3. Process a longitude and latitude to retrieve the values associated with @@ -92,14 +93,13 @@ public static void main(String[] args) throws Exception { "property as it is used by this example."); } else { - Pipeline pipeline = new GeoLocationPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new GeoLocationPipelineBuilder(loggerFactory) .useCloud(resourceKey, Enums.GeoLocationProvider.FiftyOneDegrees) .setEndPoint("https://cloud.51degrees.com/api/v4") .useLazyLoading(1000) .setAutoCloseElements(true) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(EVIDENCE_GEO_LAT_PARAM_KEY, "51.458048") .addEvidence(EVIDENCE_GEO_LON_PARAM_KEY, "-0.9822207999999999") .process(); @@ -115,8 +115,7 @@ public static void main(String[] args) throws Exception { System.out.println("Country: " + country.toString()); System.out.println(flowData.get(GeoData.class).getRegion()); - } - pipeline.close(); + } } } diff --git a/location/pom.xml b/location/pom.xml index 1939c6520..930a853f5 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -78,6 +78,7 @@ 3.0.0 3.0.1 0.7.0 + 3.26.0 true false @@ -300,6 +301,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 54b2c8f96..514a9369e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -181,9 +181,15 @@ public void init() { } @AfterEach - public void cleanUp() { - loggerFactory.assertMaxWarnings(maxWarnings); - loggerFactory.assertMaxErrors(maxErrors); + public void cleanUp() throws Exception { + try { + loggerFactory.assertMaxWarnings(maxWarnings); + loggerFactory.assertMaxErrors(maxErrors); + } finally { + if (pipeline != null) { + pipeline.close(); + } + } } private boolean propertiesContainName( @@ -370,9 +376,10 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .addFlowElement(mockRequestEngine) .addFlowElement(mockTestInstance) .build(); - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("query.user-agent", "iPhone"); - flowData.process(); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + } } catch (Exception e) { fail("An exception has occurred here."); @@ -422,9 +429,10 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .addFlowElement(mockRequestEngine) .addFlowElement(mockTestInstance) .build(); - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("query.user-agent", "iPhone"); - flowData.process(); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + } } catch (Exception e) { fail("An exception has occurred here."); @@ -480,9 +488,10 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .addFlowElement(mockTestInstance) .addFlowElement(mockRequestEngine) .build(); - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("query.user-agent", "iPhone"); - flowData.process(); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + } fail("Expected exception was not thrown"); } catch (Exception ex) { diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index c82645790..01b82a180 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -75,29 +75,30 @@ public void Process() throws Exception { .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine).build(); - FlowData data = pipeline.createFlowData(); - data.addEvidence("query.User-Agent", userAgent); - data.process(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", userAgent); + data.process(); - String result = data.getFromElement(engine).getJsonResponse(); - assertEquals("{'device':{'value':'1'}}", result); + String result = data.getFromElement(engine).getJsonResponse(); + assertEquals("{'device':{'value':'1'}}", result); - JSONObject jsonObj = new JSONObject(result); - assertEquals(1, jsonObj.getJSONObject("device").getInt("value")); + JSONObject jsonObj = new JSONObject(result); + assertEquals(1, jsonObj.getJSONObject("device").getInt("value")); - verify(httpClient, times(1)) // we expected a single external POST request - .postData( - argThat(c -> { - return c.getURL().equals(expectedUrl); // to this uri - }), - ArgumentMatchers.anyMap(), - argThat(bytes -> { - String string = new String(bytes); - return string.contains("resource=" + resourceKey) && // content contains resource key - string.contains("User-Agent=" + userAgent); // content contains user agent - })); + verify(httpClient, times(1)) // we expected a single external POST request + .postData( + argThat(c -> { + return c.getURL().equals(expectedUrl); // to this uri + }), + ArgumentMatchers.anyMap(), + argThat(bytes -> { + String string = new String(bytes); + return string.contains("resource=" + resourceKey) && // content contains resource key + string.contains("User-Agent=" + userAgent); // content contains user agent + })); + } } /** @@ -113,19 +114,20 @@ public void Process_SuppressProcessExceptions_Exceptions_Added_To_Errors() throw .setResourceKey(resourceKey) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .setSuppressProcessException(true) .addFlowElement(engine) .build(); - FlowData data = pipeline.createFlowData(); + FlowData data = pipeline.createFlowData()) { - data.addEvidence("query.User-Agent", userAgent); - data.process(); + data.addEvidence("query.User-Agent", userAgent); + data.process(); - assertFalse(data.getErrors().isEmpty()); - ArrayList errors = (ArrayList) data.getErrors(); - Throwable throwable = errors.get(0).getThrowable(); - assertInstanceOf(IOException.class, throwable); + assertFalse(data.getErrors().isEmpty()); + ArrayList errors = (ArrayList) data.getErrors(); + Throwable throwable = errors.get(0).getThrowable(); + assertInstanceOf(IOException.class, throwable); + } } /** @@ -330,11 +332,10 @@ public void evidencePrecidence(String name, boolean shouldWarn, String evidence1 .setResourceKey("resourcekey") .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(evidence1Parts[0], evidence1Parts[1]); flowData.addEvidence(evidence2Parts[0], evidence2Parts[1]); @@ -450,11 +451,10 @@ public void getFormData(String name, Map evidence, String expect .setResourceKey("resourcekey") .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .build(); - - try (FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { for (Map.Entry entry : evidence.entrySet()) { data.addEvidence(entry.getKey(), entry.getValue()); } @@ -479,11 +479,10 @@ public void getFormData_KeepsDotsInEvidenceName() throws Exception { .setResourceKey("resourcekey") .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .build(); - - try (FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { data.addEvidence("query.id.usage", "non-marketing"); data.addEvidence("header.id.email", "someone@example.com"); diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java index 20c846576..fbebda96e 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java @@ -122,9 +122,12 @@ public Pipeline buildFromConfiguration(PipelineOptions options) "The following builder parameters could not be processed: " + stringJoin(builderParams, ",")); } - } catch (PipelineConfigurationException ex) { - logger.debug("Problem with pipeline configuration, " + - "failed to create pipeline.", ex); + } catch (Exception | Error ex) { + if (ex instanceof PipelineConfigurationException) { + logger.debug("Problem with pipeline configuration, " + + "failed to create pipeline.", ex); + } + closeAfterFailure(flowElements, ex); throw ex; } // As the elements are all created within the builder, the user @@ -134,7 +137,32 @@ public Pipeline buildFromConfiguration(PipelineOptions options) // Build and return the pipeline using the list of flow elements // that have been created from the configuration options. - return build(); + try { + return build(); + } catch (Exception | Error ex) { + closeAfterFailure(flowElements, ex); + throw ex; + } + } + + /** + * Closes the elements built so far when building from configuration + * fails, as no pipeline exists to close them. A failure to close one is + * added to the original failure, which is the one reported. + * @param elements the elements built so far, emptied once closed + * @param failure the reason building failed + */ + private static void closeAfterFailure( + List elements, + Throwable failure) { + for (FlowElement element : elements) { + try { + element.close(); + } catch (Exception e) { + failure.addSuppressed(e); + } + } + elements.clear(); } /** @@ -377,17 +405,24 @@ private void addParallelElementsToList( // Iterate through the sub elements, creating them and // adding them to the list. int subCounter = 0; - for (ElementOptions subElement : elementOptions.subElements) { - if (subElement.subElements != null && - subElement.subElements.size() > 0) { - throw new PipelineConfigurationException( - "ElementOptions " + elementIndex + " contains nested " + - "sub elements. This is not supported."); - } else { - addElementToList(parallelElements, subElement, - "element " + subCounter + " in element " + elementIndex); + try { + for (ElementOptions subElement : elementOptions.subElements) { + if (subElement.subElements != null && + subElement.subElements.size() > 0) { + throw new PipelineConfigurationException( + "ElementOptions " + elementIndex + " contains nested " + + "sub elements. This is not supported."); + } else { + addElementToList(parallelElements, subElement, + "element " + subCounter + " in element " + elementIndex); + } + subCounter++; } - subCounter++; + } catch (RuntimeException | Error ex) { + // These elements are not in the builder's list yet, so they are + // closed here. + closeAfterFailure(parallelElements, ex); + throw ex; } // Now we've created all the elements, create the // ParallelElements instance and add it to the pipeline's diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java index 3e04cf231..613033fa2 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java @@ -318,14 +318,15 @@ public void FlowData_GetNotPresent_Element() { } @Test(expected = IllegalStateException.class) - public void FlowData_PipelineDisposed() { + public void FlowData_PipelineDisposed() throws Exception { when(pipeline.isClosed()).thenReturn(true); - FlowData flowData = new FlowDataDefault( + try (FlowData flowData = new FlowDataDefault( mock(Logger.class), pipeline, - new EvidenceDefault(mock(Logger.class))); + new EvidenceDefault(mock(Logger.class)))) { - flowData.process(); + flowData.process(); + } } @Test diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java index 8ddaa0435..617089b24 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java @@ -126,35 +126,36 @@ public TestElementData create(FlowData flowData) { Arrays.asList(element1, element2, element3)); - FlowData data = new FlowDataDefault( + try (FlowData data = new FlowDataDefault( mock(Logger.class), pipeline, - new EvidenceDefault(mock(Logger.class))); - data.process(); - - // Act - parallelElements.process(data); - - List startTimes = new ArrayList<>(); - startTimes.add((Date) data.get("element1").get("start")); - startTimes.add((Date) data.get("element2").get("start")); - startTimes.add((Date) data.get("element3").get("start")); - List endTimes = new ArrayList<>(); - endTimes.add((Date) data.get("element1").get("end")); - endTimes.add((Date) data.get("element2").get("end")); - endTimes.add((Date) data.get("element3").get("end")); - - // Assert - assertTrue("Expected no errors", - data.getErrors() == null || data.getErrors().size() == 0); - for (Date start : startTimes) { - for (Date end : endTimes) { - assertTrue( - "The elements were not processed sequentially. Times were: " + - startTimes.get(0) + " => " + endTimes.get(0) + "\n" + - startTimes.get(1) + " => " + endTimes.get(1) + "\n" + - startTimes.get(2) + " => " + endTimes.get(2), - start.before(end)); + new EvidenceDefault(mock(Logger.class)))) { + data.process(); + + // Act + parallelElements.process(data); + + List startTimes = new ArrayList<>(); + startTimes.add((Date) data.get("element1").get("start")); + startTimes.add((Date) data.get("element2").get("start")); + startTimes.add((Date) data.get("element3").get("start")); + List endTimes = new ArrayList<>(); + endTimes.add((Date) data.get("element1").get("end")); + endTimes.add((Date) data.get("element2").get("end")); + endTimes.add((Date) data.get("element3").get("end")); + + // Assert + assertTrue("Expected no errors", + data.getErrors() == null || data.getErrors().size() == 0); + for (Date start : startTimes) { + for (Date end : endTimes) { + assertTrue( + "The elements were not processed sequentially. Times were: " + + startTimes.get(0) + " => " + endTimes.get(0) + "\n" + + startTimes.get(1) + " => " + endTimes.get(1) + "\n" + + startTimes.get(2) + " => " + endTimes.get(2), + start.before(end)); + } } } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java index c28bf4438..f8e1c968b 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java @@ -42,6 +42,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -233,6 +234,35 @@ public void PipelineBuilder_BuildFromConfiguration_MandatoryParameterWrongType() builder.buildFromConfiguration(opts); } + /** + * When an element fails to build, the elements built before it are + * closed, and the failure reported is the original one. + */ + @Test + public void PipelineBuilder_BuildFromConfiguration_SecondElementFails_FirstClosed() throws Exception { + RecordingElementBuilder.BUILT.clear(); + PipelineOptions opts = new PipelineOptions(); + ElementOptions first = new ElementOptions(); + first.builderName = "RecordingElementBuilder"; + opts.elements.add(first); + // The mandatory 'multiple' parameter is not set, so this one fails. + ElementOptions second = new ElementOptions(); + second.builderName = "MultiplyByElementBuilder"; + opts.elements.add(second); + + maxErrors = 1; + + try { + builder.buildFromConfiguration(opts); + fail("Expected the second element to fail to build"); + } catch (PipelineConfigurationException e) { + assertEquals(0, e.getSuppressed().length); + } + assertEquals(1, RecordingElementBuilder.BUILT.size()); + assertTrue("The first element should have been closed", + RecordingElementBuilder.BUILT.get(0).isClosed()); + } + @Test(expected = PipelineConfigurationException.class) public void PipelineBuilder_BuildFromConfiguration_NoBuilder() throws Exception { // Create the configuration object. @@ -416,26 +446,27 @@ public void PipelineBuilder_BuildFromConfiguration_ParallelElements() throws Exc options.elements.add(elOpts); // Pass the configuration to the builder to create the pipeline. - Pipeline pipeline = builder.buildFromConfiguration(options); - // Get the elements - ListSplitterElement splitterElement = pipeline.getElement(ListSplitterElement.class); - MultiplyByElement multiplyByElement = pipeline.getElement(MultiplyByElement.class); - - // Create, populate and process flow data. - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence(splitterElement.evidenceKeys.get(0), "1,2,abc") - .addEvidence(multiplyByElement.evidenceKeys.get(0), 25) - .process(); + try (Pipeline pipeline = builder.buildFromConfiguration(options)) { + // Get the elements + ListSplitterElement splitterElement = pipeline.getElement(ListSplitterElement.class); + MultiplyByElement multiplyByElement = pipeline.getElement(MultiplyByElement.class); + + // Create, populate and process flow data. + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence(splitterElement.evidenceKeys.get(0), "1,2,abc") + .addEvidence(multiplyByElement.evidenceKeys.get(0), 25) + .process(); - // Get the results and verify them. - ListSplitterElementData splitterData = flowData.getFromElement(splitterElement); - TestElementData multiplyByData = flowData.getFromElement(multiplyByElement); + // Get the results and verify them. + ListSplitterElementData splitterData = flowData.getFromElement(splitterElement); + TestElementData multiplyByData = flowData.getFromElement(multiplyByElement); - assertEquals("1", splitterData.getResult().get(0)); - assertEquals("2", splitterData.getResult().get(1)); - assertEquals("abc", splitterData.getResult().get(2)); - assertEquals(75, multiplyByData.getResult()); + assertEquals("1", splitterData.getResult().get(0)); + assertEquals("2", splitterData.getResult().get(1)); + assertEquals("abc", splitterData.getResult().get(2)); + assertEquals(75, multiplyByData.getResult()); + } } } @@ -443,58 +474,60 @@ private void VerifyListSplitterElementPipeline( PipelineOptions options, SplitOption splitOn) throws Exception { // Pass the configuration to the builder to create the pipeline. - Pipeline pipeline = builder.buildFromConfiguration(options); + try (Pipeline pipeline = builder.buildFromConfiguration(options)) { - ListSplitterElement element = pipeline.getElement(ListSplitterElement.class); - // Check we've got the expected number of evidence keys. - assertEquals(1, element.evidenceKeys.size()); + ListSplitterElement element = pipeline.getElement(ListSplitterElement.class); + // Check we've got the expected number of evidence keys. + assertEquals(1, element.evidenceKeys.size()); - // Create, populate and process flow data. - try (FlowData flowData = pipeline.createFlowData()) { - flowData.addEvidence(element.evidenceKeys.get(0), "123,456|789,0") - .process(); + // Create, populate and process flow data. + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(element.evidenceKeys.get(0), "123,456|789,0") + .process(); - // Get the result and verify it. - ListSplitterElementData elementData = flowData.getFromElement(element); - switch (splitOn) { - case Comma: - assertEquals("123", elementData.getResult().get(0)); - assertEquals("456|789", elementData.getResult().get(1)); - assertEquals("0", elementData.getResult().get(2)); - break; - case Pipe: - assertEquals("123,456", elementData.getResult().get(0)); - assertEquals("789,0", elementData.getResult().get(1)); - break; - case CommaMaxLengthThree: - assertEquals("123", elementData.getResult().get(0)); - assertEquals("456", elementData.getResult().get(1)); - assertEquals("|78", elementData.getResult().get(2)); - assertEquals("9", elementData.getResult().get(3)); - assertEquals("0", elementData.getResult().get(4)); - break; - default: - break; + // Get the result and verify it. + ListSplitterElementData elementData = flowData.getFromElement(element); + switch (splitOn) { + case Comma: + assertEquals("123", elementData.getResult().get(0)); + assertEquals("456|789", elementData.getResult().get(1)); + assertEquals("0", elementData.getResult().get(2)); + break; + case Pipe: + assertEquals("123,456", elementData.getResult().get(0)); + assertEquals("789,0", elementData.getResult().get(1)); + break; + case CommaMaxLengthThree: + assertEquals("123", elementData.getResult().get(0)); + assertEquals("456", elementData.getResult().get(1)); + assertEquals("|78", elementData.getResult().get(2)); + assertEquals("9", elementData.getResult().get(3)); + assertEquals("0", elementData.getResult().get(4)); + break; + default: + break; + } } } } private void VerifyMultiplyByElementPipeline(PipelineOptions options) throws Exception { // Pass the configuration to the builder to create the pipeline. - Pipeline pipeline = builder.buildFromConfiguration(options); - MultiplyByElement element = pipeline.getElement(MultiplyByElement.class); + try (Pipeline pipeline = builder.buildFromConfiguration(options)) { + MultiplyByElement element = pipeline.getElement(MultiplyByElement.class); - // Check we've got the expected number of evidence keys. - assertEquals(1, element.evidenceKeys.size()); + // Check we've got the expected number of evidence keys. + assertEquals(1, element.evidenceKeys.size()); - // Create, populate and process flow data. - try (FlowData flowData = pipeline.createFlowData()) { - flowData.addEvidence(element.evidenceKeys.get(0), 5) - .process(); + // Create, populate and process flow data. + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(element.evidenceKeys.get(0), 5) + .process(); - // Get the result and verify it. - TestElementData elementData = flowData.getFromElement(element); - assertEquals(40, elementData.getResult()); + // Get the result and verify it. + TestElementData elementData = flowData.getFromElement(element); + assertEquals(40, elementData.getResult()); + } } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java index c688da1ee..e7d0f1d7b 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java @@ -102,30 +102,31 @@ public TestElementData create(FlowData flowData) { new FlowElement[]{element1, element2}); // Don't create the flow data via the pipeline as we just want // to test Process. - FlowData data = new FlowDataDefault( + try (FlowData data = new FlowDataDefault( mock(Logger.class), pipeline, - new EvidenceDefault(mock(Logger.class))); - - // Act - data.process(); - - // Assert - assertTrue("Expected no errors", - data.getErrors() == null || data.getErrors().size() == 0); - // Check that the resulting data has the expected values - assertTrue("data from element 1 is missing in the result", - data.getDataKeys().contains("element1")); - assertTrue("data from element 2 is missing in the result", data.getDataKeys().contains("element2")); - assertEquals("done", data.get("element1").get("key").toString()); - assertEquals("done", data.get("element2").get("key").toString()); - // Check that element 1 was called before element 2. - verify(element2, times(1)).process(argThat(new ArgumentMatcher() { - @Override - public boolean matches(FlowData flowData) { - return flowData.getDataKeys().contains("element1"); - } - })); + new EvidenceDefault(mock(Logger.class)))) { + + // Act + data.process(); + + // Assert + assertTrue("Expected no errors", + data.getErrors() == null || data.getErrors().size() == 0); + // Check that the resulting data has the expected values + assertTrue("data from element 1 is missing in the result", + data.getDataKeys().contains("element1")); + assertTrue("data from element 2 is missing in the result", data.getDataKeys().contains("element2")); + assertEquals("done", data.get("element1").get("key").toString()); + assertEquals("done", data.get("element2").get("key").toString()); + // Check that element 1 was called before element 2. + verify(element2, times(1)).process(argThat(new ArgumentMatcher() { + @Override + public boolean matches(FlowData flowData) { + return flowData.getDataKeys().contains("element1"); + } + })); + } } @Test diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java index c891b7181..1abdcd94a 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java @@ -54,11 +54,10 @@ public static void setup() { @Test public void PipelineIntegration_SingleElement() throws Exception { MultiplyByFiveElement fiveElement = new MultiplyByFiveElement(mock(Logger.class)); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(fiveElement) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(fiveElement.evidenceKeys.get(0), 2); flowData.process(); @@ -71,12 +70,11 @@ public void PipelineIntegration_SingleElement() throws Exception { public void PipelineIntegration_MultipleElements() throws Exception { MultiplyByFiveElement fiveElement = new MultiplyByFiveElement(mock(Logger.class)); MultiplyByTenElement tenElement = new MultiplyByTenElement(mock(Logger.class)); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(fiveElement) .addFlowElement(tenElement) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(fiveElement.evidenceKeys.get(0), 2); flowData.process(); @@ -91,11 +89,10 @@ public void PipelineIntegration_MultipleElements() throws Exception { public void PipelineIntegration_MultipleElementsParallel() throws Exception { MultiplyByFiveElement fiveElement = new MultiplyByFiveElement(mock(Logger.class)); MultiplyByTenElement tenElement = new MultiplyByTenElement(mock(Logger.class)); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElementsParallel(new FlowElement[]{fiveElement, tenElement}) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(fiveElement.evidenceKeys.get(0), 2); flowData.process(); @@ -113,19 +110,20 @@ public void PipelineIntegration_StopFlag() throws Exception { FlowElement testElement = mock(FlowElement.class); when(testElement.getElementDataKey()).thenReturn("test"); when(testElement.getProperties()).thenReturn(Collections.emptyList()); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(stopElement) .addFlowElement(testElement) - .build(); + .build()) { - // Create and process flow data - try (FlowData flowData = pipeline.createFlowData()) { - flowData.process(); + // Create and process flow data + try (FlowData flowData = pipeline.createFlowData()) { + flowData.process(); - // Check that the stop flag is set - assertTrue(flowData.isStopped()); - // Check that the second element was never processed - verify(testElement, never()).process(any(FlowData.class)); + // Check that the stop flag is set + assertTrue(flowData.isStopped()); + // Check that the second element was never processed + verify(testElement, never()).process(any(FlowData.class)); + } } } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java new file mode 100644 index 000000000..6ab3d4221 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java @@ -0,0 +1,47 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.testclasses.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.core.flowelements.FlowElement; + +import java.util.ArrayList; +import java.util.List; + +/** + * Builds list splitter elements and keeps each one, so that a test can check + * what happened to the elements a pipeline builder made. + */ +@ElementBuilder +public class RecordingElementBuilder { + + @SuppressWarnings("rawtypes") + public static final List BUILT = new ArrayList<>(); + + @SuppressWarnings("rawtypes") + public FlowElement build() { + FlowElement element = new ListSplitterElement(",", 100000); + BUILT.add(element); + return element; + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 819bcee46..5b7c1b325 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -105,33 +105,36 @@ public void run() throws Exception { loggerFactory) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the engines when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(cloudRequestEngine) .addFlowElement(ageElement) - .build(); + .setAutoCloseElements(true) + .build()) { - String dob = "18/12/1992"; + String dob = "18/12/1992"; - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence("cookie.date-of-birth", dob) - .process(); - - String starSign = - flowData.getFromElement(ageElement).getStarSign(); - if (starSign == null) { - // The service answered but offers no star sign - // product, which is what the 51Degrees cloud says, - // because star signs are only served by the example - // service. The connection, the property and evidence - // key negotiation and the request itself all worked. - System.out.println("The cloud service at " + endpoint() - + " does not offer the star sign product, so no " - + "star sign is available for a date of birth of " - + dob + "."); - } else { - System.out.println("With a date of birth of " + dob - + ", your star sign is " + starSign + "."); + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence("cookie.date-of-birth", dob) + .process(); + + String starSign = + flowData.getFromElement(ageElement).getStarSign(); + if (starSign == null) { + // The service answered but offers no star sign + // product, which is what the 51Degrees cloud says, + // because star signs are only served by the example + // service. The connection, the property and evidence + // key negotiation and the request itself all worked. + System.out.println("The cloud service at " + endpoint() + + " does not offer the star sign product, so no " + + "star sign is available for a date of birth of " + + dob + "."); + } else { + System.out.println("With a date of birth of " + dob + + ", your star sign is " + starSign + "."); + } } } //! [usage] diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java index 8406ac0aa..76d7e4473 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java @@ -45,22 +45,25 @@ public void run() throws Exception { new SimpleFlowElementBuilder(loggerFactory) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the element when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(starSignElement) - .build(); - Calendar dob = Calendar.getInstance(); - dob.set(1992, Calendar.DECEMBER, 18); + .setAutoCloseElements(true) + .build()) { + Calendar dob = Calendar.getInstance(); + dob.set(1992, Calendar.DECEMBER, 18); - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence("date-of-birth", dob.getTime()) - .process(); - - System.out.println("With a date of birth of " + - new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + - ", your star sign is " + - flowData.getFromElement(starSignElement).getStarSign() + - "."); + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence("date-of-birth", dob.getTime()) + .process(); + + System.out.println("With a date of birth of " + + new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + + ", your star sign is " + + flowData.getFromElement(starSignElement).getStarSign() + + "."); + } } //! [usage] } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java index 594cdbd5b..8d803d244 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java @@ -47,22 +47,26 @@ public void run() throws Exception { Main.class.getClassLoader().getResource("starsigns.csv").getPath(), false); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the engine, and with it the data file, + // when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(starSignEngine) - .build(); - Calendar dob = Calendar.getInstance(); - dob.set(1992, Calendar.DECEMBER, 18); + .setAutoCloseElements(true) + .build()) { + Calendar dob = Calendar.getInstance(); + dob.set(1992, Calendar.DECEMBER, 18); - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence("date-of-birth", dob.getTime()) - .process(); - - System.out.println("With a date of birth of " + - new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + - ", your star sign is " + - flowData.getFromElement(starSignEngine).getStarSign() + - "."); + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence("date-of-birth", dob.getTime()) + .process(); + + System.out.println("With a date of birth of " + + new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + + ", your star sign is " + + flowData.getFromElement(starSignEngine).getStarSign() + + "."); + } } //! [usage] } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java index 407759b4d..f6da01434 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java @@ -88,26 +88,30 @@ public void run() throws Exception { new File(Objects.requireNonNull(getClass().getClassLoader(). getResource("51d.xml")).getFile())); - // Build a new Pipeline from the configuration. - Pipeline pipeline = new PipelineBuilder() + // Build a new Pipeline from the configuration. Closing the + // pipeline closes the ShareUsageElement it created, which sends + // any usage data still queued. + try (Pipeline pipeline = new PipelineBuilder() .addService(http) - .buildFromConfiguration(options); - - // Alternatively, the commented code below shows how to - // configure the ShareUsageElement in code, rather than - // using a configuration file. - //ShareUsageElement usageElement = new ShareUsageBuilder(loggerFactory, http) - // .setSharePercentage(0.01) - // .setMinimumEntriesPerMessage(2500) - // .build(); - //Pipeline pipeline = new PipelineBuilder() - // .addFlowElement(usageElement) - // .build(); + .buildFromConfiguration(options)) { - System.out.println( - "Pipeline created with share usage element. Evidence processed " + - "with this pipeline will now be periodically shared with " + - "51Degrees using the specified configuration."); + // Alternatively, the commented code below shows how to + // configure the ShareUsageElement in code, rather than + // using a configuration file. + //ShareUsageElement usageElement = new ShareUsageBuilder(loggerFactory, http) + // .setSharePercentage(0.01) + // .setMinimumEntriesPerMessage(2500) + // .build(); + //Pipeline pipeline = new PipelineBuilder() + // .addFlowElement(usageElement) + // .setAutoCloseElements(true) + // .build(); + + System.out.println( + "Pipeline created with share usage element. Evidence processed " + + "with this pipeline will now be periodically shared with " + + "51Degrees using the specified configuration."); + } } } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java index 2c3b0ac4f..830f8d8ef 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java @@ -50,9 +50,10 @@ public interface ComponentMetaData extends Closeable { /** * List of the properties which come under the umbrella of this component. + * The caller closes the collection, which can hold native memory. * @return list of properties */ - Iterable getProperties(); + CloseableIterable getProperties(); /** * Get the property from the component which has the name provided. Null is diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java index 2d6357f34..da0c127dc 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java @@ -91,9 +91,10 @@ public interface FiftyOneAspectPropertyMetaData /** * Get the values which relate to the property i.e. the values which can be * returned for this property. + * The caller closes the collection, which can hold the engine's data set. * @return values which relate to the property */ - Iterable getValues(); + CloseableIterable getValues(); /** * Get the default value for the property. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java index cd80a737f..a460d8443 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java @@ -130,8 +130,8 @@ public ComponentMetaData getComponent() { } @Override - public Iterable getValues() { - return () -> { + public CloseableIterable getValues() { + return new CloseableIterableDefault(() -> { Iterator innerValues = values.iterator(); return new Iterator() { @@ -145,7 +145,7 @@ public ValueMetaData next() { return innerValues.next(); } }; - }; + }); } @Override diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java index bf0ddfabc..94ea3d982 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java @@ -39,16 +39,18 @@ public interface ProfileMetaData extends Closeable { /** * Get the values which are defined in the profile (for some Engines * multiple profiles are required to build a full set of results). + * The caller closes the collection, which can hold the engine's data set. * @return values defined in the profile */ - Iterable getValues(); + CloseableIterable getValues(); /** * Gets the values associated with the profile and the property name. * @param propertyName to get the values for + * The caller closes the collection, which can hold native memory. * @return values matching the property */ - Iterable getValues(String propertyName); + CloseableIterable getValues(String propertyName); /** * If there is a value for the profile with the property name and value then diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java index 66a433a25..cd25d63ac 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java @@ -33,6 +33,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import fiftyone.pipeline.engines.fiftyone.data.SetHeadersData; import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -66,6 +67,13 @@ public SetHeadersElementTests() { when(internalLogger.getLogger(anyString())).thenReturn(mock(Logger.class)); loggerFactory = new TestLoggerFactory(internalLogger); } + + @AfterEach + public void cleanUp() throws Exception { + if (pipeline != null) { + pipeline.close(); + } + } /** * Check that SetHeaderElement handles both 'String' and @@ -84,14 +92,15 @@ public void setHeadersElement(boolean valueIsAPV) throws Exception { new HashMap(); propertyNameValues.put("SetHeaderBrowserAccept-CH", value); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); - assertEquals("UA-Platform", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); + assertEquals("UA-Platform", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); + } } /** @@ -104,12 +113,13 @@ public void setHeadersElement_InvalidPropertyValues(Object sourcePropertyValue) Map propertyNameValues = new HashMap(); propertyNameValues.put("SetHeaderBrowserAccept-CH", sourcePropertyValue); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + } } /** @@ -139,12 +149,13 @@ public void setHeadersElement_APV_InvalidPropertyValues( Map propertyNameValues = new HashMap(); propertyNameValues.put("SetHeaderBrowserAccept-CH", value); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + } } /** @@ -167,10 +178,11 @@ public void setHeadersElement_InvalidPropertyNames( Map propertyNameValues = new HashMap(); propertyNameValues.put(sourcePropertyName, "TEST"); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - assertThrows(AggregateException.class, () -> { - data.process(); - }); + try (FlowData data = pipeline.createFlowData()) { + assertThrows(AggregateException.class, () -> { + data.process(); + }); + } } /** @@ -184,17 +196,18 @@ public void setHeadersElement_MultipleProperties() throws Exception { propertyNameValues.put("SetHeaderHardwareCritical-CH", "Sec-CH-UA-Model,Sec-CH-UA-Mobile"); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(2, typedOutput.getResponseHeaderDictionary().size()); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Critical-CH")); - assertEquals("Sec-CH-UA", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); - assertEquals("Sec-CH-UA-Model,Sec-CH-UA-Mobile", - typedOutput.getResponseHeaderDictionary().get("Critical-CH")); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(2, typedOutput.getResponseHeaderDictionary().size()); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Critical-CH")); + assertEquals("Sec-CH-UA", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); + assertEquals("Sec-CH-UA-Model,Sec-CH-UA-Mobile", + typedOutput.getResponseHeaderDictionary().get("Critical-CH")); + } } /** @@ -207,17 +220,18 @@ public void setHeadersElement_MultipleProperties_SameHeader() throws Exception { propertyNameValues.put("SetHeaderBrowserAccept-CH", "Sec-CH-UA"); propertyNameValues.put("SetHeaderHardwareAccept-CH", "Sec-CH-UA-Model,Sec-CH-UA-Mobile"); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); - String acceptCHValue = typedOutput.getResponseHeaderDictionary().get("Accept-CH"); - assertTrue(acceptCHValue.contains("Sec-CH-UA")); - assertTrue(acceptCHValue.contains("Sec-CH-UA-Model,Sec-CH-UA-Mobile")); - assertTrue(acceptCHValue.matches("^Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+$")); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); + String acceptCHValue = typedOutput.getResponseHeaderDictionary().get("Accept-CH"); + assertTrue(acceptCHValue.contains("Sec-CH-UA")); + assertTrue(acceptCHValue.contains("Sec-CH-UA-Model,Sec-CH-UA-Mobile")); + assertTrue(acceptCHValue.matches("^Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+$")); + } } private void createPipeline( diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index cf699e55b..176a2a156 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -32,6 +32,7 @@ import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Disabled; @@ -185,6 +186,18 @@ public void Init() throws Exception { tracker = new TestTracker(); } + /** + * The element runs its own sending thread, which closing it stops. + * Closing an element a test has already closed does nothing more. + */ + @AfterEach + public void cleanUp() throws Exception { + if (shareUsageElement != null) { + shareUsageElement.close(); + } + pipeline.close(); + } + private void createShareUsage( double sharePercentage, int minimumEntriesPerMessage, @@ -250,40 +263,41 @@ public void ShareUsageElement_SingleEvent_ClientIPAndHeader() throws Exception { evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + FIFTYONE_COOKIE_PREFIX + "Profile", "123456"); evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + "RemoveMe", "123456"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, devices.getLength(), "One device expected"); + NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "One device expected"); - NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(2, headers.getLength(), "2 headers expected"); - NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("5.6.7.8",header1.item(0).getTextContent()); - NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("2001::",header2.item(0).getTextContent()); + NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(2, headers.getLength(), "2 headers expected"); + NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("5.6.7.8",header1.item(0).getTextContent()); + NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("2001::",header2.item(0).getTextContent()); - NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, cookies.getLength(), "1 cookie expected"); - assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); - assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); + NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, cookies.getLength(), "1 cookie expected"); + assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); + assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); - NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, clientIp.getLength(), "1 client IP expected"); - assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, clientIp.getLength(), "1 client IP expected"); + assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + } } /** @@ -321,43 +335,44 @@ public void ShareUsageElement_SingleEvent_ClientIPAndHeader_HttpClient() throws evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + FIFTYONE_COOKIE_PREFIX + "Profile", "123456"); evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + "RemoveMe", "123456"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] zip = ((TestHttpClient)shareUsageElement.httpClient).getBuffer(); - // unzip the buffer - copy(new GZIPInputStream(new ByteArrayInputStream(zip)), baos); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] zip = ((TestHttpClient)shareUsageElement.httpClient).getBuffer(); + // unzip the buffer + copy(new GZIPInputStream(new ByteArrayInputStream(zip)), baos); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, devices.getLength(), "One device expected"); + NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "One device expected"); - NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(2, headers.getLength(), "2 headers expected"); - NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("5.6.7.8",header1.item(0).getTextContent()); - NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("2001::",header2.item(0).getTextContent()); + NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(2, headers.getLength(), "2 headers expected"); + NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("5.6.7.8",header1.item(0).getTextContent()); + NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("2001::",header2.item(0).getTextContent()); - NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, cookies.getLength(), "1 cookie expected"); - assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); - assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); + NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, cookies.getLength(), "1 cookie expected"); + assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); + assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); - NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, clientIp.getLength(), "1 client IP expected"); - assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, clientIp.getLength(), "1 client IP expected"); + assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + } } @Test @@ -373,14 +388,15 @@ public void ShareUsageElement_TwoEvents_FirstEvent() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); + // Act + shareUsageElement.process(data); - // sending of data never gets triggered - assertNull(shareUsageElement.getSendDataFuture()); + // sending of data never gets triggered + assertNull(shareUsageElement.getSendDataFuture()); + } } @Test @@ -396,30 +412,31 @@ public void ShareUsageElement_TwoEvents_SecondEvent() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - shareUsageElement.process(data); + // Act + shareUsageElement.process(data); + shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - // Verify that there is only one devices element - NodeList devices = (NodeList) xpath.compile("//Devices").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, devices.getLength(), "1 devices node expected"); + // Verify that there is only one devices element + NodeList devices = (NodeList) xpath.compile("//Devices").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "1 devices node expected"); - // Make sure there are 2 'Device' nodes - NodeList device = (NodeList) xpath.compile("//Device").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(2, device.getLength(), "2 device nodes expected"); + // Make sure there are 2 'Device' nodes + NodeList device = (NodeList) xpath.compile("//Device").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(2, device.getLength(), "2 device nodes expected"); + } } @Test @@ -440,33 +457,34 @@ public void ShareUsageElement_RestrictedHeaders() throws Exception { evidenceData.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "forwarded-for", "2001::"); evidenceData.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "user-agent", useragent); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); - - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); - - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); - - // Verify that there is only one clientIP element - NodeList clientIP = (NodeList) xpath.compile("//ClientIP").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, clientIP.getLength(), "1 ClientIP node expected"); - - // Make sure there is a header node - NodeList device = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, device.getLength(), "1 user-agent header expected"); - // Make sure no x-forwarded-for - NodeList xf = (NodeList) xpath.compile("//header[@Name='x-forwarded-for']").evaluate(doc, XPathConstants.NODESET); - assertEquals(0, xf.getLength(), "0 x-forwarded-for header expected"); - // Make sure no forwarded-for - NodeList f = (NodeList) xpath.compile("//header[@Name='forwarded-for']").evaluate(doc, XPathConstants.NODESET); - assertEquals(0, f.getLength(), "0 forwarded-for header expected"); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); + + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); + + // Verify that there is only one clientIP element + NodeList clientIP = (NodeList) xpath.compile("//ClientIP").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, clientIP.getLength(), "1 ClientIP node expected"); + + // Make sure there is a header node + NodeList device = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, device.getLength(), "1 user-agent header expected"); + // Make sure no x-forwarded-for + NodeList xf = (NodeList) xpath.compile("//header[@Name='x-forwarded-for']").evaluate(doc, XPathConstants.NODESET); + assertEquals(0, xf.getLength(), "0 x-forwarded-for header expected"); + // Make sure no forwarded-for + NodeList f = (NodeList) xpath.compile("//header[@Name='forwarded-for']").evaluate(doc, XPathConstants.NODESET); + assertEquals(0, f.getLength(), "0 forwarded-for header expected"); + } } @Test @@ -483,32 +501,33 @@ public void ShareUsageElement_LowPercentage() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - int requiredEvents = 0; - while (connector.getBaos().size()==0 && requiredEvents <= 1000000) { - shareUsageElement.process(data); - requiredEvents++; + // Act + int requiredEvents = 0; + while (connector.getBaos().size()==0 && requiredEvents <= 1000000) { + shareUsageElement.process(data); + requiredEvents++; + } + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + logger.info("Required events was {}", requiredEvents); + + // Assert + // On average, the number of required events should be around + // 100,000. However, as it's chance based it can vary + // significantly. We only want to catch any gross errors so just + // make sure the value is of the expected order of magnitude. + assertTrue(requiredEvents > 10000, + "Expected the number of required events to be at least " + + "10,000, but was actually '" + requiredEvents + "'"); + assertTrue(requiredEvents < 1000000, + "Expected the number of required events to be less than " + + "1,000,000, but was actually '" + requiredEvents + "'"); } - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); - - logger.info("Required events was {}", requiredEvents); - - // Assert - // On average, the number of required events should be around - // 100,000. However, as it's chance based it can vary - // significantly. We only want to catch any gross errors so just - // make sure the value is of the expected order of magnitude. - assertTrue(requiredEvents > 10000, - "Expected the number of required events to be at least " + - "10,000, but was actually '" + requiredEvents + "'"); - assertTrue(requiredEvents < 1000000, - "Expected the number of required events to be less than " + - "1,000,000, but was actually '" + requiredEvents + "'"); } @Test public void ShareUsageElement_SendOnCleanup() throws Exception { @@ -524,33 +543,34 @@ public void ShareUsageElement_SendOnCleanup() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); + // Act + shareUsageElement.process(data); - // No data should be being sending yet. - assertNull(shareUsageElement.getSendDataFuture()); + // No data should be being sending yet. + assertNull(shareUsageElement.getSendDataFuture()); - // Dispose of the element. - shareUsageElement.close(); + // Dispose of the element. + shareUsageElement.close(); - // Assert - // data should have been sent - assertNotNull(shareUsageElement.getSendDataFuture()); - assertTrue(shareUsageElement.isClosed()); + // Assert + // data should have been sent + assertNotNull(shareUsageElement.getSendDataFuture()); + assertTrue(shareUsageElement.isClosed()); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - // Check that the expected values are populated. - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "1 device expected"); + // Check that the expected values are populated. + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "1 device expected"); + } } @Test public void ShareUsageElement_CancelOnServerError() throws Exception { @@ -568,17 +588,18 @@ public void ShareUsageElement_CancelOnServerError() throws Exception { ((TestConnector)shareUsageElement.dataUploader).setResponseCode(500); Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - // Assert - assertFalse(shareUsageElement.isCanceled()); - assertEquals(0, shareUsageElement.evidenceCollection.size()); + // Assert + assertFalse(shareUsageElement.isCanceled()); + assertEquals(0, shareUsageElement.evidenceCollection.size()); + } } @Test @@ -598,15 +619,16 @@ public void ShareUsageElement_IgnoreOnEvidence() throws Exception { evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); evidenceData.put("header.User-Agent", "Azure Traffic Manager Endpoint Monitor"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Check that the consumer task did not start. - assertNull(shareUsageElement.getSendDataFuture()); + // Act + shareUsageElement.process(data); + // Check that the consumer task did not start. + assertNull(shareUsageElement.getSendDataFuture()); - assertEquals(0, connector.getBaos().size()); + assertEquals(0, connector.getBaos().size()); + } } @Test @Disabled // we now throw exceptions for invalid data @@ -672,28 +694,29 @@ public void ShareUsageElement_BadSchema() throws Exception { // (0x0018) - Cancel control character evidenceData.put(EVIDENCE_HEADER_USERAGENT_KEY, ""); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); - - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); - - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); - - // Check that the expected values are populated. - NodeList f = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "1 user-agent header expected"); - assertEquals("", f.item(0).getTextContent()); - assertEquals("true", f.item(0).getAttributes().getNamedItem("replaced").getTextContent()); - NodeList b = (NodeList) xpath.compile("//BadSchema").evaluate(doc, XPathConstants.NODESET); - assertEquals(0, b.getLength()); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); + + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); + + // Check that the expected values are populated. + NodeList f = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "1 user-agent header expected"); + assertEquals("", f.item(0).getTextContent()); + assertEquals("true", f.item(0).getAttributes().getNamedItem("replaced").getTextContent()); + NodeList b = (NodeList) xpath.compile("//BadSchema").evaluate(doc, XPathConstants.NODESET); + assertEquals(0, b.getLength()); + } } /** @@ -715,27 +738,28 @@ public void ShareUsageElement_SessionIdAndSequence_None() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - sequenceElement.process(data); - shareUsageElement.process(data); + // Act + sequenceElement.process(data); + shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "Expecting 1 session ID"); - NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); - assertEquals("1", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "Expecting 1 session ID"); + NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); + assertEquals("1", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + } } /** @@ -760,28 +784,29 @@ public void ShareUsageElement_SessionIdAndSequence_Existing() throws Exception { evidenceData.put(Constants.EVIDENCE_SESSIONID, "abcdefg-hijklmn-opqrst-uvwxyz"); evidenceData.put(Constants.EVIDENCE_SEQUENCE, 2); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - sequenceElement.process(data); - shareUsageElement.process(data); + // Act + sequenceElement.process(data); + shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "Expecting 1 session ID"); - assertEquals("abcdefg-hijklmn-opqrst-uvwxyz", f.item(0).getTextContent(), "Expecting session ID to be ..."); - NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); - assertEquals("3", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "Expecting 1 session ID"); + assertEquals("abcdefg-hijklmn-opqrst-uvwxyz", f.item(0).getTextContent(), "Expecting session ID to be ..."); + NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); + assertEquals("3", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + } } /** @@ -808,28 +833,29 @@ public void CheckForOffByOneError() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // collect multiple documents - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // Act - for (int i=0; i < count; i++) { - shareUsageElement.process(data); - } - // Wait for the consumer task to finish. - shareUsageElement.close(); + // collect multiple documents + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // Act + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + // Wait for the consumer task to finish. + shareUsageElement.close(); - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(count, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(count, f.getLength(), "Expecting devices"); + } } /** * A consumer that has finished its drain loop with fewer entries left than @@ -890,54 +916,55 @@ protected void sendUsageData() { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // collect multiple documents - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // collect multiple documents + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // close in a finally as well, so that an assertion failure before the - // Act step below still releases the lingering consumer. - try { - // Send one full batch and wait for the consumer to drain it. - for (int i = 0; i < minimumEntriesPerMessage; i++) { - shareUsageElement.process(data); - } - assertTrue(consumerHasDrainedQueue.await(LINGER_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), - "The consumer should have drained the first batch"); + // close in a finally as well, so that an assertion failure before the + // Act step below still releases the lingering consumer. + try { + // Send one full batch and wait for the consumer to drain it. + for (int i = 0; i < minimumEntriesPerMessage; i++) { + shareUsageElement.process(data); + } + assertTrue(consumerHasDrainedQueue.await(LINGER_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "The consumer should have drained the first batch"); - // Queue too few entries to start a consumer of their own, while the - // previous consumer's task has not completed yet. - for (int i = 0; i < remainder; i++) { - shareUsageElement.process(data); - } - assertEquals(remainder, shareUsageElement.evidenceCollection.size()); - assertTrue(shareUsageElement.isRunning(), - "The first consumer should still be reported as running"); + // Queue too few entries to start a consumer of their own, while the + // previous consumer's task has not completed yet. + for (int i = 0; i < remainder; i++) { + shareUsageElement.process(data); + } + assertEquals(remainder, shareUsageElement.evidenceCollection.size()); + assertTrue(shareUsageElement.isRunning(), + "The first consumer should still be reported as running"); - // Act - close must flush the remainder. - shareUsageElement.close(); - assertEquals(0, shareUsageElement.evidenceCollection.size(), - "close should leave nothing queued"); - } finally { - // Swallowed so that a failure above is still the reported one - the - // close in the Act step is the one under test. - try { + // Act - close must flush the remainder. shareUsageElement.close(); - } catch (Exception exception) { - logger.warn("Exception closing the element under test", exception); + assertEquals(0, shareUsageElement.evidenceCollection.size(), + "close should leave nothing queued"); + } finally { + // Swallowed so that a failure above is still the reported one - the + // close in the Act step is the one under test. + try { + shareUsageElement.close(); + } catch (Exception exception) { + logger.warn("Exception closing the element under test", exception); + } } - } - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(minimumEntriesPerMessage + remainder, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(minimumEntriesPerMessage + remainder, f.getLength(), "Expecting devices"); + } } @Test @@ -962,29 +989,30 @@ public void CheckForDrainQueueEvenIfError() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // collect multiple documents - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // Act - for (int i=0; i < count; i++) { - shareUsageElement.process(data); - } - // Wait for the consumer task to finish. - shareUsageElement.close(); + // collect multiple documents + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // Act + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + // Wait for the consumer task to finish. + shareUsageElement.close(); - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - // this shows that sending is ignoring the failures - assertEquals(count, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + // this shows that sending is ignoring the failures + assertEquals(count, f.getLength(), "Expecting devices"); + } } @Test public void CheckForContinueAndResumeWhenQueueFull() throws Exception { @@ -1003,48 +1031,49 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // the output document will collect multiple documents, - // so we need to wrap them in a root element - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // the output document will collect multiple documents, + // so we need to wrap them in a root element + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // Overwhelm the queue - int count = 55; - for (int i=0; i < count; i++) { - shareUsageElement.process(data); - } - long amountOfLostData; - assertTrue((amountOfLostData = shareUsageElement.lostData) > 0); - // wait for queue to drain - logger.info("Sleeping"); - Thread.sleep(5000); - - // send a new element which should reset the flag - shareUsageElement.process(data); - assertEquals(amountOfLostData, shareUsageElement.lostData); - - // overwhelm the queue again - for (int i=0; i < count; i++) { + // Overwhelm the queue + int count = 55; + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + long amountOfLostData; + assertTrue((amountOfLostData = shareUsageElement.lostData) > 0); + // wait for queue to drain + logger.info("Sleeping"); + Thread.sleep(5000); + + // send a new element which should reset the flag shareUsageElement.process(data); - } - assertTrue(amountOfLostData < shareUsageElement.lostData); + assertEquals(amountOfLostData, shareUsageElement.lostData); + + // overwhelm the queue again + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + assertTrue(amountOfLostData < shareUsageElement.lostData); - // Wait for the consumer task to finish. - shareUsageElement.close(); + // Wait for the consumer task to finish. + shareUsageElement.close(); - // get the data output - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + // get the data output + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - // the count is what we put in minus what we have recorded as lost - assertEquals(2 * count + 1 - shareUsageElement.lostData, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + // the count is what we put in minus what we have recorded as lost + assertEquals(2 * count + 1 - shareUsageElement.lostData, f.getLength(), "Expecting devices"); + } } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java index be67bd7e4..9583850a7 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java @@ -30,10 +30,12 @@ import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.services.DataUploader; import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; import org.json.JSONObject; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -95,43 +97,50 @@ public void Initialise() throws Exception { .setProcessCost(0) .build(); + // Closing the element sends what is still queued, so it is given an + // uploader that sends nothing anywhere. ShareUsageElement shareUsage = - new ShareUsageBuilder(loggerFactory) + new ShareUsageBuilder(loggerFactory, mock(DataUploader.class)) .build(); pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .addFlowElement(shareUsage) + .setAutoCloseElements(true) .build(); } + @AfterEach + public void cleanUp() throws Exception { + pipeline.close(); + } + + // The FlowData are all processed before any is closed, so that closing + // them is not part of the time measured. They are closed together in a + // finally block rather than one by one with try-with-resources. @Test - public void ShareUsageOverhead_SingleEvidence() { + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void ShareUsageOverhead_SingleEvidence() throws Exception { int iterations = 10_000; double maxOverheadPerCall = 0.1; List data = new ArrayList<>(); - for (int i = 0; i < iterations; i++) { - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); - data.add(flowData); - } + long start; + long end; + try { + for (int i = 0; i < iterations; i++) { + FlowData flowData = pipeline.createFlowData(); + data.add(flowData); + flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); + } - long start = System.currentTimeMillis(); - for (FlowData entry : data) { - entry.process(); - } - long end = System.currentTimeMillis(); - - // It is not efficient to use try-with-resources - // in this scenario, so loop through and perform - // manual close on each FlowData - data.forEach((f) -> { - try { - f.close(); - } catch (Exception e) { - e.printStackTrace(); + start = System.currentTimeMillis(); + for (FlowData entry : data) { + entry.process(); } - }); + end = System.currentTimeMillis(); + } finally { + closeAll(data); + } double msOverheadPerCall = (double)(end - start) / iterations; overheadMillis.put("ShareUsageOverhead_SingleEvidence_ms", msOverheadPerCall); @@ -142,36 +151,34 @@ public void ShareUsageOverhead_SingleEvidence() { msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCall); } + // See ShareUsageOverhead_SingleEvidence for why the FlowData are closed + // in a finally block. @Test - public void ShareUsageOverhead_ThousandEvidence() { + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void ShareUsageOverhead_ThousandEvidence() throws Exception { int iterations = 1_000; int evidenceCount = 1_000; int maxOverheadPerCallMillis = 10; List data = new ArrayList<>(); - for (int i = 0; i < iterations; i++) { - FlowData flowData = pipeline.createFlowData(); - for (int j = 0; j < evidenceCount; j++) { - flowData.addEvidence("header." + j, j); + long start; + long end; + try { + for (int i = 0; i < iterations; i++) { + FlowData flowData = pipeline.createFlowData(); + data.add(flowData); + for (int j = 0; j < evidenceCount; j++) { + flowData.addEvidence("header." + j, j); + } + flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); } - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); - data.add(flowData); - } - long start = System.currentTimeMillis(); - for (FlowData entry : data) { - entry.process(); - } - long end = System.currentTimeMillis(); - - // It is not efficient to use try-with-resources - // in this scenario, so loop through and perform - // manual close on each FlowData - data.forEach((f) -> { - try { - f.close(); - } catch (Exception e) { - e.printStackTrace(); + start = System.currentTimeMillis(); + for (FlowData entry : data) { + entry.process(); } - }); + end = System.currentTimeMillis(); + } finally { + closeAll(data); + } double msOverheadPerCall = (double)(end - start) / iterations; overheadMillis.put("ShareUsageOverhead_HundredEvidence_ms", msOverheadPerCall); @@ -181,4 +188,10 @@ public void ShareUsageOverhead_ThousandEvidence() { "Pipeline with share usage overhead per Process call was " + msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCallMillis); } + + private static void closeAll(List data) throws Exception { + for (FlowData flowData : data) { + flowData.close(); + } + } } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java index be3bbaada..5aed919b3 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java @@ -138,16 +138,15 @@ public void TestJavaScriptCookies( JavaScriptBuilderElement jsElement = new JavaScriptBuilderElementBuilder(loggerFactory) .setEnableCookies(enableInConfig) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) - .addFlowElement(cookieElement) - .addFlowElement(sequenceElement) - .addFlowElement(jsonElement) - .addFlowElement(jsElement) - .build(); - // Act String javaScript = null; - try (FlowData flowData = pipeline.createFlowData()) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(cookieElement) + .addFlowElement(sequenceElement) + .addFlowElement(jsonElement) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence( Constants.EVIDENCE_ENABLE_COOKIES, diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index a94604104..2178a5f66 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -72,12 +72,12 @@ private String render(String contextRoot) throws Exception { builder.setContextRoot(contextRoot); } JavaScriptBuilderElement jsElement = builder.build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) .addFlowElement(jsElement) .build(); - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence("query.user-agent", "iPhone"); flowData.addEvidence("query.id.usage", "non-marketing"); flowData.process(); @@ -164,18 +164,19 @@ private List renderWithHosts(String... hosts) throws Exception { new JavaScriptBuilderElementBuilder(factory) .setEndpoint("/json") .build(); - Pipeline pipeline = new PipelineBuilder(factory) + List scripts = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(factory) .addFlowElement(new SequenceElementBuilder(factory).build()) .addFlowElement(new JsonBuilderElementBuilder(factory).build()) .addFlowElement(jsElement) - .build(); - List scripts = new ArrayList<>(); - for (String host : hosts) { - try (FlowData flowData = pipeline.createFlowData()) { - flowData.addEvidence(Constants.EVIDENCE_HOST_KEY, host); - flowData.process(); - scripts.add( - flowData.getFromElement(jsElement).getJavaScript()); + .build()) { + for (String host : hosts) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(Constants.EVIDENCE_HOST_KEY, host); + flowData.process(); + scripts.add( + flowData.getFromElement(jsElement).getJavaScript()); + } } } return scripts; diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index 9c9d9f935..965526329 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -460,13 +460,15 @@ public void JsonBuilder_LazyLoading() throws Exception { .build(); SequenceElement sequenceElement = new SequenceElementBuilder(loggerFactory) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the elements, the lazy loading engine's + // threads included. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(sequenceElement) .addFlowElement(engine) .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.process(); diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java index 351b3b5da..309b7669b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java @@ -127,15 +127,24 @@ public boolean preHandle( HttpServletResponse response, Object handler) throws Exception { resultService.process(request); - - // Set the Client Hints request headers - uachService.setResponseHeaders(request, response); - - if (fiftyOneJsService.serveJS(request, response) == false && - fiftyOneJsService.serveJson(request, response) == false) { - return super.preHandle(request, response, handler); + + boolean proceed = false; + try { + // Set the Client Hints request headers + uachService.setResponseHeaders(request, response); + + proceed = fiftyOneJsService.serveJS(request, response) == false && + fiftyOneJsService.serveJson(request, response) == false && + super.preHandle(request, response, handler); + return proceed; + } finally { + // Spring calls afterCompletion, which closes the FlowData, only + // when preHandle returns true. In every other case, serving the + // JavaScript or JSON included, the FlowData is closed here. + if (proceed == false) { + flowDataProvider.getFlowData(request).close(); + } } - return false; } @Override diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java index 2b9562ada..077a88b46 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.web.mvc; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -123,4 +124,18 @@ public void FiftyOneInterceptor_AfterCompletion() throws Exception { verify(flowData, times(1)) .close(); } + + /** + * Check that the FlowData is closed when preHandle serves the JavaScript + * itself, as Spring does not call afterCompletion when preHandle returns + * false. + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() + throws Exception { + assertFalse(interceptor.preHandle(request, response, handler)); + verify(flowData, times(1)) + .close(); + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java index 1834f5a56..098da0926 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java @@ -122,27 +122,30 @@ public void doFilter( // Populate the request properties and store against the // HttpContext. resultService.process((HttpServletRequest)request); - - // Get FlowData from the request. - FlowData flowData = flowDataProviderCore.getFlowData((HttpServletRequest)request); - - // Set UACH response headers. - uachServiceCore.setResponseHeaders( - flowData, (HttpServletResponse)response); - - // If 51Degrees JavaScript or JSON is being requested then serve it. - // Otherwise continue down the filter Pipeline. - if (jsService.serveJS((HttpServletRequest)request, (HttpServletResponse) response) == false && - jsService.serveJson((HttpServletRequest)request, (HttpServletResponse) response) == false) { - chain.doFilter(request, response);//sends request to next resource - } - // The rest of the filters in the chain have now been called. So it is - // time to dispose of the FlowData instance. try { - flowDataProviderCore.getFlowData((HttpServletRequest) request).close(); - } catch (Exception e) { - throw new ServletException("FlowData could not be disposed of.", e); + // Get FlowData from the request. + FlowData flowData = flowDataProviderCore.getFlowData((HttpServletRequest)request); + + // Set UACH response headers. + uachServiceCore.setResponseHeaders( + flowData, (HttpServletResponse)response); + + // If 51Degrees JavaScript or JSON is being requested then serve it. + // Otherwise continue down the filter Pipeline. + if (jsService.serveJS((HttpServletRequest)request, (HttpServletResponse) response) == false && + jsService.serveJson((HttpServletRequest)request, (HttpServletResponse) response) == false) { + chain.doFilter(request, response);//sends request to next resource + } + } finally { + // The rest of the filters in the chain have now been called, or + // one of them failed. Either way it is time to dispose of the + // FlowData instance. + try { + flowDataProviderCore.getFlowData((HttpServletRequest) request).close(); + } catch (Exception e) { + throw new ServletException("FlowData could not be disposed of.", e); + } } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java index 9bc21609a..289c90ff9 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java @@ -71,14 +71,29 @@ public Default( this.pipeline = pipeline; } + /** + * The FlowData is kept in the request, and whatever handles the end + * of the request closes it. If processing fails it never reaches the + * request, so it is closed here instead. + */ @Override + @SuppressWarnings("PMD.FlowDataInTryWithResources") public void process(HttpServletRequest request) { // Create the flowData FlowData flowData = pipeline.createFlowData(); - // Extract the required pieces of evidence from the request - evidenceService.addEvidenceFromRequest(flowData, request); - // Start processing the data - flowData.process(); + try { + // Extract the required pieces of evidence from the request + evidenceService.addEvidenceFromRequest(flowData, request); + // Start processing the data + flowData.process(); + } catch (RuntimeException | Error e) { + try { + flowData.close(); + } catch (Exception closeException) { + e.addSuppressed(closeException); + } + throw e; + } // Store the FlowData in the request request.setAttribute(HTTPCONTEXT_FLOWDATA, flowData); } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java index e4362e3ae..e0f7b7508 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java @@ -25,6 +25,7 @@ import fiftyone.pipeline.core.data.ElementData; import fiftyone.pipeline.core.data.FlowData; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,7 +42,9 @@ import static fiftyone.pipeline.engines.fiftyone.data.SetHeadersData.*; import static fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement.*; import static fiftyone.pipeline.web.Constants.*; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -54,6 +57,7 @@ public class PipelineFilterTests { private FilterChain chain; private FlowData flowData; private ElementData elementData; + private PipelineFilter filter; @Before public void init() throws IOException, ServletException { @@ -102,13 +106,23 @@ public void init() throws IOException, ServletException { }).when(request).setAttribute(eq(HTTPCONTEXT_FLOWDATA), any(Object.class)); } + /** + * Close the pipeline each filter built. + */ + @After + public void cleanUp() { + if (filter != null) { + filter.destroy(); + } + } + /** * Check that the filter can be initialised. * @throws ServletException */ @Test public void PipelineFilter_Initialise() throws ServletException { - PipelineFilter filter = new PipelineFilter(); + filter = new PipelineFilter(); filter.init(config); } @@ -121,7 +135,7 @@ public void PipelineFilter_Initialise() throws ServletException { */ @Test public void PipelineFilter_Process() throws ServletException, IOException { - PipelineFilter filter = new PipelineFilter(); + filter = new PipelineFilter(); filter.init(config); filter.doFilter(request, response, chain); @@ -142,10 +156,31 @@ public void PipelineFilter_Process() throws ServletException, IOException { */ @Test public void PipelineFilter_CloseFlowData() throws Exception { - PipelineFilter filter = new PipelineFilter(); + filter = new PipelineFilter(); filter.init(config); filter.doFilter(request, response, chain); verify(flowData, times(1)).close(); } + + /** + * Check that the filter disposes of the FlowData when a later filter in + * the chain fails. + * @throws Exception + */ + @Test + public void PipelineFilter_CloseFlowData_ChainFails() throws Exception { + doThrow(new ServletException("chain failed")) + .when(chain).doFilter(any(ServletRequest.class), any(ServletResponse.class)); + filter = new PipelineFilter(); + + filter.init(config); + try { + filter.doFilter(request, response, chain); + fail("Expected the exception from the chain"); + } catch (ServletException e) { + assertEquals("chain failed", e.getMessage()); + } + verify(flowData, times(1)).close(); + } } diff --git a/pipeline/pom.xml b/pipeline/pom.xml index a3c3b5afc..abcd6d8f6 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -95,6 +95,7 @@ 2.6 3.7 0.7.0 + 3.26.0 true false @@ -351,6 +352,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + From 4d36bacd6896108bb6b57d838e5f2fdcf2a29df0 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:52:07 +0100 Subject: [PATCH 15/95] Merged Pull Request '#46 update-packages/rc->rc: Update packages' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- examples/device-detection/pom.xml | 2 +- examples/ip-intelligence/pom.xml | 4 ++-- examples/ip-intelligence/web/getting-started.api/pom.xml | 2 +- location/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index fa55e5534..8e9dd78bc 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -50,7 +50,7 @@ 4.13.2 true 1.30 - 4.5.128 + 4.5.130 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 2e3624a91..8d7ff14cf 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -50,8 +50,8 @@ 4.13.2 true 1.30 - 4.5.128 - 4.5.128 + 4.5.130 + 4.5.130 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/web/getting-started.api/pom.xml b/examples/ip-intelligence/web/getting-started.api/pom.xml index 5cb1db7d1..ef29feee4 100644 --- a/examples/ip-intelligence/web/getting-started.api/pom.xml +++ b/examples/ip-intelligence/web/getting-started.api/pom.xml @@ -37,7 +37,7 @@ 2.0.20 1.3.16 - 4.5.128 + 4.5.130 20231013 diff --git a/location/pom.xml b/location/pom.xml index 930a853f5..9974783aa 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -60,7 +60,7 @@ 1.8 ${project.version} - 4.5.128 + 4.5.130 4.13.2 1.7.36 From 9b859abf253053b55c66287a8a3b8931b1c28f23 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Thu, 5 Nov 2026 06:52:12 +0100 Subject: [PATCH 16/95] Merged Pull Request '#38 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- .../resources/fiftyone/pipeline/javascriptbuilder/templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates b/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates index d3f96049b..c773ce221 160000 --- a/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates +++ b/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates @@ -1 +1 @@ -Subproject commit d3f96049bde389951c79638047b8ea80dfdd5cdf +Subproject commit c773ce221ccec6fcb67fbdd5ad2175a66fab11d6 From 66cc2dc08ce1f7056ddfe204994b56b898db5d76 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 5 Nov 2026 08:18:27 +0000 Subject: [PATCH 17/95] Allow only http and https in the callback URL, and test the zone and port rules (#47) --- .../flowelements/CallbackUrl.java | 58 ++++++++--- .../JavaScriptBuilderElementBuilder.java | 11 ++- .../javascriptbuilder/JavaScriptUrlTests.java | 63 ++++++++++++ .../flowelements/CallbackUrlTests.java | 97 ++++++++++++++++++- 4 files changed, 211 insertions(+), 18 deletions(-) diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java index 4118f86f9..c6e76aa65 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java @@ -30,10 +30,10 @@ * The callback URL the client script posts to for updated JSON. The protocol * and host usually come from the request, and the script holds the URL in a * single-quoted string, so the URL is checked before it is written into the - * script. A URL that is not valid is refused. In the user information, in - * anything after the host and port, and in the endpoint, the characters a - * URL cannot hold or that could end the string or the script are - * percent-encoded. + * script. A URL that is not valid, including one whose protocol is not http + * or https, is refused. In the user information, in anything after the host + * and port, and in the endpoint, the characters a URL cannot hold or that + * could end the string or the script are percent-encoded. * @see Specification */ final class CallbackUrl { @@ -71,11 +71,11 @@ final class CallbackUrl { "\\u2028\\u2029\\u202F\\u205F\\u3000]*"); /** - * A scheme as RFC 3986 section 3.1 defines one, being a letter followed - * by letters, digits, '+', '-' and '.'. + * The schemes the script can send its request with. It uses the URL only + * with fetch and XMLHttpRequest, which reach a server with no other + * scheme. */ - private static final Pattern SCHEME = - Pattern.compile("[A-Za-z][A-Za-z0-9+.-]*"); + private static final String[] SCHEMES = {"http", "https"}; /** * A label of a host name, being ASCII letters, digits, '-' and '_'. @@ -128,7 +128,7 @@ private CallbackUrl(boolean valid, String url) { * separates the host and the endpoint, being added when neither has one * and removed when both do. The protocol and the host are written in * lower case and the rest as given, apart from the percent-encoding. - * @param protocol the protocol, such as https + * @param protocol the protocol, http or https in any case * @param host the host, which can start with user information and end * with a port, a path, a query or a fragment * @param endpoint the endpoint @@ -147,7 +147,7 @@ static CallbackUrl of(String protocol, String host, String endpoint) { endpoint = endpoint.substring(1); } String built = protocol + "://" + host + endpoint; - if (SCHEME.matcher(protocol).matches() == false) { + if (isScheme(protocol) == false) { return new CallbackUrl(false, built); } int end = authorityEnd(host); @@ -157,10 +157,46 @@ static CallbackUrl of(String protocol, String host, String endpoint) { } return new CallbackUrl( true, - protocol.toLowerCase(Locale.ROOT) + "://" + authority + + asciiLowerCase(protocol) + "://" + authority + encoded(host.substring(end)) + encoded(endpoint)); } + /** + * Whether the protocol is http or https, ignoring the case of its ASCII + * letters only. String.equalsIgnoreCase also matches a letter outside + * ASCII to an ASCII one, such as the long s (U+017F) to 's', so it is + * not used. + * @param protocol the protocol + * @return true if the protocol is http or https + */ + static boolean isScheme(String protocol) { + if (protocol == null) { + return false; + } + String lower = asciiLowerCase(protocol); + for (String scheme : SCHEMES) { + if (scheme.equals(lower)) { + return true; + } + } + return false; + } + + /** + * The text with the ASCII letters A to Z in lower case and every other + * character as it is. + * @param text the text + * @return the text with A to Z in lower case + */ + private static String asciiLowerCase(String text) { + StringBuilder lower = new StringBuilder(text.length()); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + lower.append(c >= 'A' && c <= 'Z' ? (char) (c + ('a' - 'A')) : c); + } + return lower.toString(); + } + /** * Whether a part of the URL is not set, being null, empty or only * whitespace. diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java index 28e77260c..47a449373 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java @@ -105,20 +105,21 @@ public JavaScriptBuilderElementBuilder setContextRoot(String contextRoot) { *

* By default, the protocol from the request will be * used. - * @param protocol The protocol to use (http / https) + * @param protocol The protocol to use, http or https in any case of its + * ASCII letters * @return this builder + * @throws PipelineConfigurationException if the protocol is neither http + * nor https */ @DefaultValue("The protocol from the request") public JavaScriptBuilderElementBuilder setProtocol(String protocol) { - if (protocol.equalsIgnoreCase("http") || - protocol.equalsIgnoreCase("https")) { + if (CallbackUrl.isScheme(protocol)) { this.protocol = protocol; } else { throw new PipelineConfigurationException( "Invalid protocol in configuration (" + protocol + - "), must be 'http' or https'"); - + "), must be 'http' or 'https'"); } return this; } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index 2178a5f66..67006a697 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -25,6 +25,7 @@ import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; @@ -32,6 +33,8 @@ import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.ILoggerFactory; import org.slf4j.Logger; @@ -43,6 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -226,6 +230,65 @@ public void Url_RefusedHost_NoCallbackAndLoggedOnce() throws Exception { errors.get(0)); } + /** + * A protocol from the request that is neither http nor https gives a + * script with no callback, and is logged once like any other URL that + * is not valid. + */ + @Test + public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { + TestLoggerFactory factory = hostLoggerFactory; + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(factory) + .setEndpoint("/json") + .build(); + try (Pipeline pipeline = new PipelineBuilder(factory) + .addFlowElement(new SequenceElementBuilder(factory).build()) + .addFlowElement(new JsonBuilderElementBuilder(factory).build()) + .addFlowElement(jsElement) + .build()) { + for (int i = 0; i < 2; i++) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence( + Constants.EVIDENCE_HOST_KEY, "example.com"); + flowData.addEvidence( + fiftyone.pipeline.core.Constants.EVIDENCE_PROTOCOL, + "ftp"); + flowData.process(); + assertTrue(requestUrls(flowData.getFromElement(jsElement) + .getJavaScript()).isEmpty(), + "A refused protocol reached the script."); + } + } + } + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + assertTrue(errors.get(0).contains("\"ftp://example.com/json\""), + errors.get(0)); + } + + /** + * A protocol configured on the builder is used for every request, so one + * the script cannot send its request with is refused. The long s is + * refused too, which String.equalsIgnoreCase would match to 's'. + */ + @ParameterizedTest + @ValueSource(strings = {"ftp", "ws", "javascript", "https ", "http\u017f"}) + public void Protocol_ConfiguredOtherThanHttpOrHttps_IsRefused( + String protocol) { + JavaScriptBuilderElementBuilder builder = + new JavaScriptBuilderElementBuilder(loggerFactory); + assertThrows(PipelineConfigurationException.class, + () -> builder.setProtocol(protocol)); + } + + @ParameterizedTest + @ValueSource(strings = {"http", "https", "HTTPS"}) + public void Protocol_ConfiguredHttpOrHttps_IsAccepted(String protocol) { + assertNotNull(new JavaScriptBuilderElementBuilder(loggerFactory) + .setProtocol(protocol)); + } + /** * The URL is quoted and escaped in the log, so a line break in it cannot * start a new log line. diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java index fb7f6d91d..b4be236ef 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java @@ -192,6 +192,92 @@ public void CallbackUrl_RefusesHost(String host) { assertInvalid(url("https", host), "host " + host); } + static Stream refusedZonesAndTextAfterTheBracket() { + return Stream.of( + "[fe80::1%25]", + "[fe80::1%25'+alert(1)+']", + "[fe80::1%eth0]", + "[fe80::1%25eth%0]", + "[::1]x", + "[::1]80", + "[::1]:80x", + "[::1]]"); + } + + /** + * Inside the brackets only an IPv6 address, optionally followed by '%25' + * and a zone of unreserved characters, and after them only a port. The + * zone check is what keeps a quote out of the script for a bracketed + * host. + */ + @ParameterizedTest(name = "[{index}]") + @MethodSource("refusedZonesAndTextAfterTheBracket") + public void CallbackUrl_RefusesZoneOrTextAfterTheBracket(String host) { + assertInvalid(url("https", host), "host " + host); + } + + @Test + public void CallbackUrl_AcceptsZoneFollowedByPort() { + assertValid( + "https://[fe80::1%25eth0]:8080/51dpipeline/json", + url("https", "[fe80::1%25eth0]:8080"), + "host [fe80::1%25eth0]:8080"); + } + + /** + * A port goes by its value, so leading zeros are allowed, and a port too + * long to read is refused rather than read as small enough. + */ + @Test + public void CallbackUrl_ChecksThePortByItsValue() { + assertValid( + "https://example.com:00080/51dpipeline/json", + url("https", "example.com:00080"), + "host example.com:00080"); + assertValid( + "https://example.com:0065535/51dpipeline/json", + url("https", "example.com:0065535"), + "host example.com:0065535"); + for (String host : new String[] { + "example.com:0065536", + "example.com:123456", + "example.com:4294967296", + "example.com:99999999999999999999"}) { + assertInvalid(url("https", host), "host " + host); + } + } + + /** + * Each character rule 8 lists, in the user information, after the host + * and in the endpoint, and a '%' without two hexadecimal digits after + * it, even at the end. + */ + @Test + public void CallbackUrl_EncodesTheCharactersRule8Lists() { + assertValid( + "https://u%22%5E%60%7B%7C%7D@example.com/%22%5E%60%7B%7C%7D" + + "/j%22%5E%60%7B%7C%7D", + CallbackUrl.of( + "https", "u\"^`{|}@example.com/\"^`{|}", "/j\"^`{|}"), + "rule 8 characters"); + assertValid( + "https://example.com/a%25/51dpipeline/json", + url("https", "example.com/a%"), + "host example.com/a%"); + assertValid( + "https://example.com/a%254/51dpipeline/json", + url("https", "example.com/a%4"), + "host example.com/a%4"); + assertValid( + "https://example.com/a%41/51dpipeline/json", + url("https", "example.com/a%41"), + "host example.com/a%41"); + assertValid( + "https://example.com/json%25", + CallbackUrl.of("https", "example.com", "/json%"), + "endpoint /json%"); + } + @Test public void CallbackUrl_InvalidUrlIsTheOneBuilt() { CallbackUrl url = url("https", "exa mple.com"); @@ -200,7 +286,7 @@ public void CallbackUrl_InvalidUrlIsTheOneBuilt() { } @ParameterizedTest - @ValueSource(strings = {"https", "http", "ftp", "h+t-p.s"}) + @ValueSource(strings = {"https", "http", "HtTp"}) public void CallbackUrl_AcceptsScheme(String protocol) { CallbackUrl url = url(protocol, "example.com"); assertNotNull(url); @@ -215,8 +301,15 @@ public void CallbackUrl_WritesTheSchemeInLowerCase() { "protocol HTTPS"); } + /** + * Only http and https, ignoring the case of ASCII letters only, so a + * letter outside ASCII that a case-insensitive comparison can match to an + * ASCII one, such as the long s, is refused. + */ @ParameterizedTest - @ValueSource(strings = {"1http", "h t", "x');alert(1);//", "a://b"}) + @ValueSource(strings = {"ftp", "h+t-p.s", "ws", "javascript", "httpx", + "https ", "1http", "h t", "x');alert(1);//", "a://b", "http\u017f", + "HTTP\u017f"}) public void CallbackUrl_RefusesScheme(String protocol) { assertInvalid(url(protocol, "example.com"), "protocol " + protocol); } From 1aaaf7093bd0546bfacab5302ac44fc6a2216adf Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 00:22:48 +0000 Subject: [PATCH 18/95] Share only the evidence usage sharing accepts, as .NET does (#51) --- .../data/EvidenceKeyFilterShareUsage.java | 67 ++++-- .../fiftyone/flowelements/ShareUsageBase.java | 3 +- .../flowelements/ShareUsageBuilderBase.java | 15 +- .../EvidenceKeyFilterShareUsageTests.java | 209 ++++++++++++++++++ .../flowelements/ShareUsageElementTests.java | 138 ++++++++++++ 5 files changed, 410 insertions(+), 22 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 3df2733df..996694614 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -36,12 +36,18 @@ /** * This filter is used by the {@link ShareUsageElement}. * It will include anything that is: - * 1) An HTTP header that is not blocked by the constructor parameter. + * 1) An HTTP header that is not blocked by the constructor parameter, other + * than Cookie. * 2) A cookie that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX} or is * the session cookie (if configured in the constructor). - * 3) An query string parameters that have been configured to be shared - * using the constructor parameter. - * 4) Not a header, cookie or query string parameter. + * 3) A session value that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX}. + * 4) A query string parameter that starts with + * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be + * shared using the constructor parameter. + * 5) Not a header, cookie, session value or query string parameter. + * The prefix of a key is everything before its first '.', and the field + * everything after it. An email address, the salt for hashed email + * identifiers, the query string and the path are never included. * * As this filter is generally inclusive, it will often cause far more * evidence to be passed into a pipeline than the engine-specific @@ -75,6 +81,21 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final Set includedQueryStringParams = new HashSet<>(); + /** + * Evidence keys that are never shared, matched without regard to case as + * the whole key or after a '.'. 'id.email' and 'id.salt' are a raw email + * address and the salt used to make hashed email identifiers. + * 'request-query' is the whole query string, which would undo the rules + * for its parameters. 'request-path' would make each page a visitor opens + * look like a new session to the tracker. + */ + private static final String[] NEVER_SHARED_EVIDENCE_KEYS = { + "id.email", + "id.salt", + "request-query", + "request-path" + }; + /** * Constructor. * @param blockedHttpHeaders a list of the names of the HTTP headers that @@ -92,9 +113,13 @@ public EvidenceKeyFilterShareUsage( List includedQueryStringParams, boolean includeSession, String sessionCookieName) { + // The Cookie header carries every cookie, including the ones the + // cookie rule leaves out. this.blockedHttpHeaders.add("cookies"); + this.blockedHttpHeaders.add("cookie"); this.includeSession = includeSession; - this.sessionCookieName = sessionCookieName; + this.sessionCookieName = sessionCookieName == null ? + null : sessionCookieName.toLowerCase(); for (String header : blockedHttpHeaders) { String lowerHeader = header.toLowerCase(); if (!this.blockedHttpHeaders.contains(lowerHeader)) { @@ -111,30 +136,42 @@ public EvidenceKeyFilterShareUsage( @Override public boolean include(String key) { + String lowerKey = key.toLowerCase(); + for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) { + if (lowerKey.equals(neverShared) || + lowerKey.endsWith(EVIDENCE_SEPERATOR + neverShared)) { + return false; + } + } + boolean result; - String[] parts = key.toLowerCase().split("\\."); - if (parts.length == 2) { - switch (parts[0]) { + int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); + if (separator > 0) { + String prefix = lowerKey.substring(0, separator); + String field = lowerKey.substring(separator + 1); + switch (prefix) { case EVIDENCE_HTTPHEADER_PREFIX: // Add the header to the list if the header name does not // appear in the list of blocked headers. - result = blockedHttpHeaders.contains(parts[1]) == false; + result = blockedHttpHeaders.contains(field) == false; break; case EVIDENCE_COOKIE_PREFIX: // Only add cookies that start with the 51Degrees cookie // prefix. - result = parts[1].startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - (includeSession && parts[1].equals(sessionCookieName)); + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (includeSession && field.equals(sessionCookieName)); break; case EVIDENCE_SESSION_PREFIX: // Only add session values that start with the 51Degrees cookie // prefix. - result = parts[1].startsWith(Constants.FIFTYONE_COOKIE_PREFIX); + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX); break; case EVIDENCE_QUERY_PREFIX: - // Only include query string parameters that have been - // specified in the constructor. - result = includedQueryStringParams.contains(parts[1]); + // Only include query string parameters that start with the + // 51Degrees prefix or have been specified in the + // constructor. + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + includedQueryStringParams.contains(field); break; default: // Add anything that is not a cookie or a header. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 58a46d731..ddb8809ba 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -29,6 +29,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.configuration.CacheConfiguration; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsageTracker; import fiftyone.pipeline.engines.fiftyone.exceptions.HttpException; import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; import fiftyone.pipeline.engines.trackers.Tracker; @@ -440,7 +441,7 @@ protected ShareUsageBase( this.tracker = new ShareUsageTracker( cacheConfiguration, intervalMillis, - new EvidenceKeyFilterShareUsage( + new EvidenceKeyFilterShareUsageTracker( blockedHttpHeaders, includedQueryStringParameters, trackSession, sessionCookieName)); } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index bad9c1bbd..e376660cf 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -83,7 +83,8 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

* This setter replaces the current list, if any *

@@ -95,7 +96,7 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameters( List queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -104,7 +105,8 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

* This setter replaces the current list, if any *

@@ -115,7 +117,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameters to include * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameters( String queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -125,7 +127,8 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

* This setting adds the name of a parameter to share with 51Degrees * if needed. @@ -133,7 +136,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameter to include * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameter( String queryStringParameterName) { includedQueryStringParameters.add(queryStringParameterName.trim()); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java new file mode 100644 index 000000000..6e6e17147 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -0,0 +1,209 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests which evidence {@link EvidenceKeyFilterShareUsage} lets usage sharing + * keep and send. + */ +public class EvidenceKeyFilterShareUsageTests { + + private static EvidenceKeyFilterShareUsage filter( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession) { + return new EvidenceKeyFilterShareUsage( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + DEFAULT_SESSION_COOKIE_NAME); + } + + private static EvidenceKeyFilterShareUsage defaults() { + return filter( + Collections.emptyList(), + Collections.emptyList(), + false); + } + + /** + * By default headers other than Cookie are shared, cookies, session + * values and query string parameters only when their names start with + * 51D_, and any other evidence. + */ + @Test + public void defaultRules() { + EvidenceKeyFilterShareUsage filter = defaults(); + + assertTrue(filter.include("header.user-agent")); + assertFalse(filter.include("header.cookie")); + assertTrue(filter.include("cookie.51d_screenpixelswidth")); + assertTrue(filter.include("cookie.51D_ScreenPixelsWidth")); + assertFalse(filter.include("cookie.sessionid")); + assertFalse(filter.include("cookie.a.b")); + assertTrue(filter.include("session.51d_value")); + assertFalse(filter.include("session.org.example.attribute")); + assertTrue(filter.include("query.51d_gethighentropyvalues")); + assertTrue(filter.include("query.51D_Test")); + assertFalse(filter.include("query.password")); + assertFalse(filter.include("query.id.email")); + assertTrue(filter.include("server.client-ip")); + assertTrue(filter.include("nodot")); + } + + /** + * A name containing a dot is matched as a whole against the parameters to + * share. + */ + @Test + public void fieldIsEverythingAfterTheFirstDot() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("valid.email"), + false); + + assertTrue(filter.include("query.valid.email")); + assertFalse(filter.include("query.valid")); + assertFalse(filter.include("query.other.name")); + } + + /** + * The headers not to share and the parameters to share are matched + * without regard to case. + */ + @Test + public void listsIgnoreCase() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.singletonList("X-Secret"), + Collections.singletonList("Campaign"), + false); + + assertFalse(filter.include("header.x-secret")); + assertTrue(filter.include("query.campaign")); + assertTrue(filter.include("header.user-agent")); + } + + /** + * The session cookie is included only when asked for, whatever the case of + * its name, and no other ordinary cookie is. + */ + @Test + public void sessionCookie() { + EvidenceKeyFilterShareUsage kept = filter( + Collections.emptyList(), + Collections.emptyList(), + true); + EvidenceKeyFilterShareUsage sent = defaults(); + + assertTrue(kept.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertTrue(kept.include("cookie.jsessionid")); + assertFalse(sent.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(kept.include("cookie._ga")); + assertFalse(sent.include("cookie._ga")); + } + + /** + * The email address, the salt for hashed email identifiers, the query + * string and the path are never shared, whether bare or after a prefix, + * and without regard to case, even when the rules for their prefixes + * would share them. + */ + @Test + public void neverShared() { + List filters = Arrays.asList( + defaults(), + filter( + Collections.emptyList(), + Arrays.asList("id.email", "id.salt"), + true)); + List keys = Arrays.asList( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.email", "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"); + for (EvidenceKeyFilterShareUsage filter : filters) { + for (String key : keys) { + assertFalse(filter.include(key), key); + } + } + } + + /** + * A key that only ends with the same letters as a key never shared is + * treated as an ordinary key. + */ + @Test + public void neverSharedKeysMatchWholeParts() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("valid.email"), + false); + + assertTrue(filter.include("query.valid.email")); + } + + /** + * Sharing one parameter does not share the whole query string, which + * carries every parameter. + */ + @Test + public void namingAParameterDoesNotShareTheQueryString() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("utm_source"), + false); + + assertTrue(filter.include("query.utm_source")); + assertFalse(filter.include("server.request-query")); + } + + /** + * The tracker leaves out the session id and sequence number, which differ + * on every request, and follows the same rules otherwise. + */ + @Test + public void trackerLeavesOutSessionIdAndSequence() { + EvidenceKeyFilterShareUsageTracker tracker = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Arrays.asList("session-id", "sequence"), + false, + DEFAULT_SESSION_COOKIE_NAME); + + assertFalse(tracker.include("query.session-id")); + assertFalse(tracker.include("query.sequence")); + assertTrue(tracker.include("header.user-agent")); + assertFalse(tracker.include("query.id.email")); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 176a2a156..260118fe6 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -1076,4 +1076,142 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { } } + + /** + * Default settings share headers other than Cookie, the 51Degrees cookies + * and query string parameters, and other evidence, and leave out ordinary + * cookies, other parameters, session values and the evidence that is + * never shared. + */ + @Test + public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Exception { + // Arrange + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "some user agent"); + evidenceData.put("cookie.51d_screenpixelswidth", "screen width"); + evidenceData.put("query.51D_Test", "fiftyone parameter"); + evidenceData.put("cookie.sessionid", "sessionid cookie"); + evidenceData.put("header.cookie", "cookie header"); + evidenceData.put("query.password", "password parameter"); + evidenceData.put("query.id.email", "email parameter"); + evidenceData.put("header.id.email", "email header"); + evidenceData.put("id.salt", "salt value"); + evidenceData.put("server.request-query", "whole query string"); + evidenceData.put("server.request-path", "request path"); + evidenceData.put("session.org.example.attribute", "session attribute"); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + + // Act + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + } + + // Assert + String output = new String( + connector.getBaos().toByteArray(), StandardCharsets.UTF_8); + for (String shared : Arrays.asList( + "some user agent", "screen width", "fiftyone parameter")) { + assertTrue(output.contains(shared), shared); + } + for (String withheld : Arrays.asList( + "sessionid cookie", "cookie header", "password parameter", + "email parameter", "email header", "salt value", + "whole query string", "request path", "session attribute")) { + assertFalse(output.contains(withheld), withheld); + } + } + + /** + * The default tracker does not share again a request that differs from + * one already shared only in its session id and sequence number. + */ + @Test + public void ShareUsageElement_RepeatWithNewSequence_NotShared() throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + + // Act + for (int sequence = 1; sequence <= 2; sequence++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "repeated user agent"); + evidenceData.put(Constants.EVIDENCE_SESSIONID, "session " + sequence); + evidenceData.put(Constants.EVIDENCE_SEQUENCE, sequence); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "1 device expected"); + } + + /** + * The session cookie the element names is kept in evidence for the + * tracker, whatever the case of its name, and never sent. With sessions + * tracked, the same evidence from a new session is shared again. + */ + @Test + public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + String sessionCookie = EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + + DEFAULT_SESSION_COOKIE_NAME; + + // Act + for (int session = 1; session <= 2; session++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "repeated user agent"); + evidenceData.put(sessionCookie, "session cookie " + session); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + assertTrue(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); + byte[] output = connector.getBaos().toByteArray(); + assertFalse(new String(output, StandardCharsets.UTF_8).contains( + "session cookie"), "the session cookie is never sent"); + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(output)); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(2, devices.getLength(), "1 device expected for each session"); + } } From 7b5c70fcadf02052b9bb3a5b3bbc298b5571af0a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 00:23:12 +0000 Subject: [PATCH 19/95] Run every example on the free data files, and show what needs the Enterprise data file (#48) --- examples/device-detection/README.md | 9 +- .../console/GettingStartedOnPrem.java | 64 +++- .../examples/console/MatchMetrics.java | 21 +- .../examples/console/MetadataOnPrem.java | 11 +- .../examples/console/OfflineProcessing.java | 56 +++- .../console/PerformanceBenchmark.java | 22 +- .../examples/console/UpdateDataFile.java | 63 +++- .../console/ExamplesWithoutSettingsTest.java | 129 ++++++++ .../examples/shared/DataFileHelper.java | 211 ++++++++++--- .../examples/web/HtmlContentHelper.java | 39 ++- examples/ip-intelligence/README.md | 54 ++-- .../examples/console/CompareOnPrem.java | 54 +++- .../examples/console/GettingStartedMixed.java | 157 +++++++--- .../console/GettingStartedOnPrem.java | 119 +++++--- .../examples/console/MetadataOnPrem.java | 25 +- .../examples/console/MetricsOnPrem.java | 26 +- .../examples/console/OfflineProcessing.java | 50 ++-- .../console/PerformanceBenchmark.java | 49 ++- .../examples/console/SuspiciousOnPrem.java | 44 ++- .../examples/console/UpdateDataFile.java | 89 +++--- .../console/ExamplesWithoutSettingsTest.java | 164 +++++++++++ .../examples/shared/DataFileHelper.java | 278 ++++++++++++++++-- .../examples/web/GettingStartedApi.java | 4 +- .../examples/web/GettingStartedWebMixed.java | 144 +++++---- .../examples/web/GettingStartedWebOnPrem.java | 88 +++--- .../examples/web/HtmlContentHelper.java | 9 +- 26 files changed, 1582 insertions(+), 397 deletions(-) create mode 100644 examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java create mode 100644 examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 4c80193e5..d908ca138 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -27,7 +27,14 @@ The on-premise examples use the free Lite data file and the evidence files in `cxx/data/device-detection`, a folder of the [data](https://github.com/51Degrees/data) repository. An example that cannot find one of these files runs the data repository's download script for it, -and logs the command it runs. +and logs the command it runs. The examples find `cxx/data` whether they are +run from the root of this repository or from this folder. + +Each example shows the properties that only the Enterprise data file has, +such as HardwareVendor, HardwareName and DeviceType, as needing it, and gives +a link to ask for a copy. [Contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=enterprise-data-file) +for a copy of the Enterprise data file, and pass its path as the first +argument to use it. ## Cloud resource keys diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 44b7227fb..57c898e34 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -30,6 +30,7 @@ import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.fiftyone.flowelements.FiftyOnePipelineBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,7 @@ import java.io.PrintWriter; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.devicedetection.examples.shared.PropertyHelper.asString; @@ -63,10 +65,17 @@ public class GettingStartedOnPrem { parameter. Note that the Lite data file is only used for illustration, and has limited accuracy and - capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ + capabilities. The example shows the properties only the Enterprise data file has as needing + it. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static String ENTERPRISE_HASH = "TAC-HashV41.hash"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -115,15 +124,20 @@ public static void run(String dataFile, .buildFromConfiguration(pipelineOptions)) { - // carry out some sample detections - for (Map evidence : evidenceList) { - analyzeEvidence(evidence, pipeline, outputStream); - } - /* Get the 'engine' element within the pipeline that performs device detection. We can use this to get details about the data file as well as meta-data describing things such as the available properties. */ DeviceDetectionHashEngine engine = pipeline.getElement(DeviceDetectionHashEngine.class); + + // carry out some sample detections + boolean needsEnterprise = false; + for (Map evidence : evidenceList) { + needsEnterprise |= analyzeEvidence(evidence, pipeline, engine, outputStream); + } + if (needsEnterprise) { + DataFileHelper.writeEnterpriseNote(new PrintWriter(outputStream), CONTACT_US); + } + DataFileHelper.logDataFileInfo(engine); logger.info("All done"); @@ -135,11 +149,15 @@ public static void run(String dataFile, * supplied and output the device detection results to the output stream. * @param evidence a map representing HTTP headers * @param pipeline a pipeline set up to process the evidence + * @param engine the engine, which says which properties its data file has * @param out somewhere to send the detection results + * @return true if a property needed the Enterprise data file */ - private static void analyzeEvidence(Map evidence, - Pipeline pipeline, - OutputStream out) throws Exception { + private static boolean analyzeEvidence(Map evidence, + Pipeline pipeline, + DeviceDetectionHashEngine engine, + OutputStream out) throws Exception { + boolean needsEnterprise = false; PrintWriter writer = new PrintWriter(out); /* FlowData is a data structure that is used to convey information required for detection and the results of the detection through the pipeline. Information required for @@ -177,9 +195,37 @@ private static void analyzeEvidence(Map evidence, writer.println("\tPlatform Version: " + asString(device.getPlatformVersion())); writer.println("\tBrowser Name: " + asString(device.getBrowserName())); writer.println("\tBrowser Version: " + asString(device.getBrowserVersion())); + // Only the Enterprise data file has these properties. The free + // Lite data file has the ones above. + needsEnterprise |= writeEnterpriseValue(writer, "Hardware Vendor", + engine, "HardwareVendor", device::getHardwareVendor); + needsEnterprise |= writeEnterpriseValue(writer, "Hardware Name", + engine, "HardwareName", device::getHardwareName); + needsEnterprise |= writeEnterpriseValue(writer, "Device Type", + engine, "DeviceType", device::getDeviceType); } writer.println(); writer.flush(); + return needsEnterprise; + } + + /** + * Writes a property that only the Enterprise data file has, or that it + * needs the Enterprise data file when the data file in use does not + * have it. The value is only read when the data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean writeEnterpriseValue(PrintWriter writer, + String name, + DeviceDetectionHashEngine engine, + String property, + Supplier> value) { + if (DataFileHelper.hasProperty(engine, property)) { + writer.println("\t" + name + ": " + asString(value.get())); + return false; + } + writer.println("\t" + name + ": " + DataFileHelper.NEEDS_ENTERPRISE); + return true; } } /*! diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index 4c0aa55a1..ddabcaed4 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -45,6 +45,7 @@ import java.util.Objects; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.util.stream.Collectors.groupingBy; @@ -62,9 +63,18 @@ public class MatchMetrics { private static final Logger logger = LoggerFactory.getLogger(MatchMetrics.class); + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String dataFilename = args.length > 0 ? args[0] : null; + // Use the supplied data file, or the free Lite file, which is + // downloaded if it is missing + String dataFilename = DataFileHelper.getDataFileLocation( + args.length > 0 ? args[0] : LITE_HASH_DATA_FILE_PATH); run(dataFilename, EvidenceHelper.setUpEvidence(), false, System.out); } @@ -106,8 +116,9 @@ static void run(String dataFile, List> evidenceList, // Uncomment BrowserName to include Browser component profile ID // in the device ID value. //.setProperty("BrowserName") - // If using the full on-premise data file this property will be - // present in the data file. See https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java&utm_term=run + // Only the Enterprise data file has this property, and the + // free Lite data file leaves it out. Contact us for a copy at + // https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java&utm_term=run .setProperty("HardwareName") // Only use the predictive graph to better handle variances // between the training data and the target User-Agent string. @@ -215,6 +226,10 @@ static void run(String dataFile, List> evidenceList, }); writer.println(); } + if (DataFileHelper.isLiteDataFile( + pipeline.getElement(DeviceDetectionHashEngine.class))) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } } logger.info("Finished Match Metrics Example"); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 350172405..8d8a06bcf 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -52,9 +52,15 @@ public class MetadataOnPrem { parameter. Note that the Lite data file is only used for illustration, and has limited accuracy and - capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ + capabilities. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); String dataFile = args.length > 0 ? args[0] : LITE_V_4_1_HASH; @@ -108,6 +114,9 @@ public static void run(String dataFile, OutputStream output) throws Exception { writer.flush(); DataFileHelper.logDataFileInfo(ddEngine); + if (DataFileHelper.isLiteDataFile(ddEngine)) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } } } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index 8fc928a56..cba0c7e89 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -40,6 +40,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; @@ -71,10 +72,16 @@ public class OfflineProcessing { // cxx/data, and is downloaded if it is missing // // Note that the Lite data file is only used for illustration, and has - // limited accuracy and capabilities. Find out about the Enterprise data - // file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash + // limited accuracy and capabilities. Contact us for a copy of the + // Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash public static final String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=enterprise-data-file"; // This 51degrees file of 20,000 examples is used from the same folder, // and is downloaded if it is missing public static final String HEADER_EVIDENCE_YML = @@ -168,6 +175,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws try (Writer writer = new OutputStreamWriter(os)) { // read a batch of device data from the stream int count = 0; + boolean needsEnterprise = false; while (evidenceIterator.hasNext() && count < 20) { // Flow data is the container for inputs and outputs that // flow through the pipeline a flowdata instance is @@ -199,6 +207,17 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws resultMap.put("device.ismobile", asString(device.getIsMobile())); resultMap.put("device.platformname", asString(device.getPlatformName())); resultMap.put("device.platformversion", asString(device.getPlatformVersion())); + // Only the Enterprise data file has these properties. + // The free Lite data file has the ones above. + needsEnterprise |= putEnterpriseValue(resultMap, + "device.hardwarevendor", engine, "HardwareVendor", + device::getHardwareVendor); + needsEnterprise |= putEnterpriseValue(resultMap, + "device.hardwarename", engine, "HardwareName", + device::getHardwareName); + needsEnterprise |= putEnterpriseValue(resultMap, + "device.devicetype", engine, "DeviceType", + device::getDeviceType); // to look at all device detection properties use the following: // resultMap.putAll(getPopulatedProperties(device, "device.")); @@ -211,21 +230,38 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws } count++; } + if (needsEnterprise) { + // A comment keeps the output a valid YAML stream. + writer.write("# Some of these properties need the " + + "Enterprise data file. Contact us for a copy at " + + CONTACT_US + "\n"); + } // finish the last YAML document writer.write("...\n"); writer.flush(); logger.info("Finished processing {} records", count); - - if (engine.getDataSourceTier().equals("Lite")) { - logger.warn("You have used a Lite data file which has " + - "limited properties and is of limited accuracy"); - logger.info("The example requires an Enterprise data file " + - "to work fully. Find out about the Enterprise " + - "data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite-data-file"); - } } } } + + /** + * Records a property that only the Enterprise data file has, or that it + * needs the Enterprise data file when the data file in use does not + * have it. The value is only read when the data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean putEnterpriseValue(Map resultMap, + String key, + DeviceDetectionHashEngine engine, + String property, + Supplier> value) { + if (DataFileHelper.hasProperty(engine, property)) { + resultMap.put(key, asString(value.get())); + return false; + } + resultMap.put(key, DataFileHelper.NEEDS_ENTERPRISE); + return true; + } /** * Filter entries that are not keyed on the required prefix diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index 8c6f2b0fe..163e9f8da 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -46,6 +46,7 @@ import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDataFileLocation; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getEvidenceFile; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; /** @@ -64,6 +65,15 @@ public class PerformanceBenchmark { public static final Logger logger = LoggerFactory.getLogger(PerformanceBenchmark.class); + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-performancebenchmark.java&utm_term=enterprise-data-file"; + + // true when the benchmarks ran on the free Lite data file + private boolean usedLiteDataFile; + // where the results of the tests are gathered private List> resultList; private int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; @@ -82,7 +92,9 @@ public class PerformanceBenchmark { public static void main(String[] args) throws Exception { LogbackHelper.configureLogback(FileFinder.getFilePath("logback.xml")); - String dataFilename = args.length > 0 ? args[0] : null; + // Use the supplied data file, or the free Lite file, which is + // downloaded if it is missing + String dataFilename = args.length > 0 ? args[0] : LITE_HASH_DATA_FILE_PATH; String evidenceFilename = args.length > 1 ? args[1] : null; int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; if (args.length > 2) { @@ -132,6 +144,9 @@ protected void runBenchmarks(PerformanceConfiguration[] performanceConfiguration for (PerformanceConfiguration config: performanceConfigurations){ executeBenchmark(true, config); } + if (usedLiteDataFile) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } logger.info("Finished Performance example"); } @@ -166,7 +181,10 @@ private void executeBenchmark(boolean configureFromDisk, setPipelinePerformanceProperties(builder, config); pipeline = builder.build(); - DataFileHelper.logDataFileInfo(pipeline.getElement(DeviceDetectionHashEngine.class)); + DeviceDetectionHashEngine engine = + pipeline.getElement(DeviceDetectionHashEngine.class); + DataFileHelper.logDataFileInfo(engine); + usedLiteDataFile = DataFileHelper.isLiteDataFile(engine); } else { logger.info("Load memory from {}", dataFileLocation); byte[] fileContent = Files.readAllBytes(new File(dataFileLocation).toPath()); diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index d3babcd27..9748b15f5 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.PrintWriter; import java.nio.file.*; import java.text.SimpleDateFormat; import java.util.Objects; @@ -42,6 +43,7 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDatafileMetaData; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.ENTERPRISE_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /*! @@ -57,8 +59,9 @@ *

  • Daily auto-update
    • * * ## License Key - * In order to test this example you will need a 51Degrees Enterprise license which can be - * purchased from our [pricing page](https://51degrees.com/pricing/annual?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=updatedatafile). + * Updating the data file needs a 51Degrees licence key for the Enterprise data file. Without + * one, the example shows the free Lite data file and says where to ask for a key. Contact us for + * a key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=updatedatafile * # Data Files * You can find out more about data files, licenses etc. at our [FAQ page](https://51degrees.com/resources/faqs?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=updatedatafile) * ## Enterprise Data File @@ -179,9 +182,9 @@ *
  • Daily auto-update
    • * *

    - * To run this example you must obtain a license key purchased from our - * pricing page. Look for our "Bigger" or - * "Biggest" options. This license key must be supplied as a command line argument or by setting + * To update the data file you need a licence key, which you can ask for on our + * contact page. + * The key must be supplied as a command line argument or by setting * an environment variable or system property called {@link UpdateDataFile#UPDATE_EXAMPLE_LICENSE_KEY_NAME} */ public class UpdateDataFile { @@ -192,6 +195,12 @@ public class UpdateDataFile { private static final Logger logger = LoggerFactory.getLogger(UpdateDataFile.class); + /** + * Where to ask for a licence key and a copy of the Enterprise data file, + * tagged with this example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=licence-key"; + static final UpdateCompletionListener completionListener = new UpdateCompletionListener(); static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss Z"); @@ -221,12 +230,8 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr licenseKey = KeyUtils.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); } if (Objects.isNull(licenseKey) || KeyUtils.isInvalidKey(licenseKey)) { - logger.error("In order to test this example you will need a 51Degrees Enterprise " + - "license which can be obtained on a trial basis or purchased from our\n" + - "pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=license-key-required. You must supply the license " + - "key as an argument to this program, or as an environment or system variable " + - "named '{}'", UPDATE_EXAMPLE_LICENSE_KEY_NAME); - throw new IllegalArgumentException("No license key available"); + explainLicenseKey(dataFilename, new PrintWriter(System.out, true)); + return; } // work out where the downloaded file will be put, directory must exist @@ -304,7 +309,7 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // to notify when update complete .setDataUpdateService(dataUpdateService) // For automatic updates to work you will need to provide a license key. - // A license key can be obtained with a subscription from https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=run + // Contact us for a licence key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=run .setDataUpdateLicenseKey(licenseKey) // Enable update on startup, the auto update system // will be used to check for an update before the @@ -359,6 +364,40 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr } } + /** + * Without a licence key the example cannot download a data file, so it + * shows the data file it would start from, the free Lite file unless + * another is given, and says what an update needs. + * @param dataFilename the data file given, or null + * @param output where to write + * @throws Exception if the data file cannot be read + */ + static void explainLicenseKey(String dataFilename, PrintWriter output) throws Exception { + File dataFile = null; + try { + dataFile = Objects.isNull(dataFilename) ? + DataFileHelper.findDataFile(LITE_HASH_DATA_FILE_PATH) : + new File(dataFilename); + } catch (IllegalArgumentException e) { + logger.warn(e.getMessage()); + } + if (Objects.nonNull(dataFile) && dataFile.exists()) { + DataFileHelper.DatafileInfo metadata = + getDatafileMetaData(dataFile.getAbsolutePath()); + output.println("The data file in use is the '" + metadata.getTier() + + "' file published on " + simpleDateFormat.format( + metadata.getFileInfo().getDataPublishedDateTime()) + + ", at '" + metadata.getFileInfo().getDataFilePath() + "'."); + } + output.println(); + output.println("Updating the data file downloads the Enterprise data " + + "file, which needs a licence key. Give the key as the first " + + "command line argument, or in the environment variable or " + + "system property '" + UPDATE_EXAMPLE_LICENSE_KEY_NAME + "'. " + + "Contact us for a key and a copy at " + CONTACT_US); + output.flush(); + } + private static void logFileInfo(DataFileHelper.DatafileInfo metadata) { logger.info("Current data file is a {} tier data file, last updated at {}, at " + "location {}. Next update expected {}.", metadata.getTier(), diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java new file mode 100644 index 000000000..aea911082 --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java @@ -0,0 +1,129 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Runs each on-premise console example's main with no arguments and no + * licence key, as someone who has just cloned the repository runs it, and + * checks that it says what the Enterprise data file adds, with the link to + * the contact page. The examples default to the free Lite data file. CI + * gives the other tests the Enterprise data files, so nothing else tests + * what a fresh clone shows. + */ +public class ExamplesWithoutSettingsTest { + + private static final String CONTACT_US = "https://51degrees.com/contact-us?"; + + private static final String LICENSE_KEY = + UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME; + + /** + * An example's main method. + */ + private interface Main { + void main(String[] args) throws Exception; + } + + /** + * Runs the example's main with no arguments and the licence key system + * property removed, and returns what it wrote to standard output. + */ + private static String runWithoutSettings(Main main) throws Exception { + assumeTrue("Skipping, a licence key is set in the environment", + System.getenv(LICENSE_KEY) == null && + System.getenv(LICENSE_KEY.toUpperCase()) == null); + String licenseKey = System.clearProperty(LICENSE_KEY); + PrintStream out = System.out; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + System.setOut(new PrintStream(buffer, true, "UTF-8")); + try { + main.main(new String[0]); + } finally { + System.setOut(out); + if (licenseKey != null) { + System.setProperty(LICENSE_KEY, licenseKey); + } + } + String output = buffer.toString("UTF-8"); + out.println(output); + return output; + } + + private static void assertContains(String output, String expected) { + assertTrue("The output should contain '" + expected + "'", + output.contains(expected)); + } + + private static void assertOffer(String output) { + assertContains(output, "This example used the free Lite data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void gettingStarted() throws Exception { + String output = runWithoutSettings(GettingStartedOnPrem::main); + assertContains(output, "Hardware Vendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "Some of these properties need the Enterprise data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void matchMetrics() throws Exception { + assertOffer(runWithoutSettings(MatchMetrics::main)); + } + + @Test + public void metadata() throws Exception { + assertOffer(runWithoutSettings(MetadataOnPrem::main)); + } + + @Test + public void offlineProcessing() throws Exception { + String output = runWithoutSettings(OfflineProcessing::main); + assertContains(output, "device.hardwarevendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "# Some of these properties need the Enterprise " + + "data file. Contact us for a copy at " + CONTACT_US); + } + + @Test + public void performance() throws Exception { + assertOffer(runWithoutSettings(PerformanceBenchmark::main)); + } + + @Test + public void updateDataFile() throws Exception { + String output = runWithoutSettings(UpdateDataFile::main); + assertContains(output, "The data file in use is the 'Lite' file"); + assertContains(output, "which needs a licence key"); + assertContains(output, CONTACT_US); + } +} diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java index 337f6e165..55a9c44c1 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java @@ -26,6 +26,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.util.FileFinder; import org.slf4j.Logger; @@ -33,11 +34,11 @@ import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Locale; @@ -50,10 +51,30 @@ public class DataFileHelper { /** * Sent with each download, so the Distributor can count the downloads - * made by these examples. + * made by these examples. It is left out inside GitHub Actions, where + * the data repository's script names the build instead, so that test + * runs are not counted as people using the examples. */ private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + private static final boolean IN_GITHUB_ACTIONS = + "true".equals(System.getenv("GITHUB_ACTIONS")); + + private static final String WINDOWS_POWERSHELL = "powershell"; + + /** + * Where to ask about the Enterprise data file when the data file in use + * is old or cannot be found. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file"; + + /** + * Shown in place of a value for a property that the data file in use + * does not have. Every property the examples ask for is in the + * Enterprise data file. + */ + public static final String NEEDS_ENTERPRISE = "needs the Enterprise data file"; + public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; @@ -78,6 +99,13 @@ public static DatafileInfo getDatafileMetaData(String dataFileLocation) throws E } } + /** + * Logs the data file in use, and a warning if it is old. An example run + * on the free Lite data file says what the Enterprise data file adds in + * its own output, with {@link #writeEnterpriseNote} or + * {@link #writeEnterpriseOffer}. + * @param engine the engine whose data file is logged + */ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { // Lite or Enterprise String dataTier = engine.getDataSourceTier(); @@ -89,13 +117,6 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { String displayDate = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(fileDate); logger.info("Used a '{}' data file, created {}, {} days ago, from location '{}'", dataTier, displayDate, daysOld, dataFileLocation); - if (dataTier.equals("Lite")) { - logger.warn("This example is using the 'Lite ' data file. " + - "This is used for illustration, and has limited " + - "accuracy and capabilities. Find out about the " + - "Enterprise data file on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=lite-data-file"); - } if (daysOld > 28) { logger.warn("This example is using a data file that is more " + "than {} days old. A more recent data file " + @@ -104,10 +125,9 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { logger.info("Run get-data-files.ps1 -Update or " + "get-data-files.sh -u in cxx/data, which is the data " + "repository https://github.com/51Degrees/data, to get " + - "the latest 'Lite' data file. " + - "Find out about the Enterprise data file, which " + - "includes automatic daily updates, on our pricing " + - "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-age-warning"); + "the latest 'Lite' data file. Contact us about the " + + "Enterprise data file, which includes automatic daily " + + "updates, at " + CONTACT_US); } } @@ -117,10 +137,67 @@ public static void cantFindDataFile(String dataFile) { logger.info("The free 'Lite' data file and the evidence files " + "are downloaded by get-data-files.ps1 or get-data-files.sh " + "in cxx/data, which is the data repository " + - "https://github.com/51Degrees/data. " + - "Find out about the Enterprise data file, which includes " + - "automatic daily updates, on our pricing " + - "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-not-found"); + "https://github.com/51Degrees/data. Contact us about the " + + "Enterprise data file, which includes automatic daily " + + "updates, at " + CONTACT_US); + } + + /** + * True when the engine's data file has the property. The Lite data + * file leaves out the properties only the Enterprise data file has, + * such as HardwareVendor, HardwareName and DeviceType. + * @param engine the engine to check + * @param name the name of the property + * @return true if the data file has the property + */ + public static boolean hasProperty(DeviceDetectionHashEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + /** + * True when the engine uses the free Lite data file. + * @param engine the engine to check + * @return true for the Lite data file + */ + public static boolean isLiteDataFile(DeviceDetectionHashEngine engine) { + return "Lite".equals(engine.getDataSourceTier()); + } + + /** + * Writes, after an example has shown properties that the data file in + * use does not have, where to ask for a copy of the Enterprise data + * file. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseNote(PrintWriter output, String contactUs) { + output.println(); + output.println("Some of these properties need the Enterprise data " + + "file. Contact us for a copy at " + contactUs); + output.flush(); + } + + /** + * Writes, after an example that shows no properties has run on the + * free Lite data file, what the Enterprise data file adds and where to + * ask for a copy. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { + output.println(); + output.println("This example used the free Lite data file. The " + + "Enterprise data file adds many more properties, such as " + + "HardwareVendor, HardwareName and DeviceType. Contact us " + + "for a copy at " + contactUs); + output.flush(); } /** @@ -173,28 +250,31 @@ public static String getDataFileLocation(String dataFilename) throws Exception { /** * Finds a file below the project root, as - * {@link FileFinder#getFilePath(String)} does. A free data file that is - * missing has a {@code .link} in the data repository, so the data - * repository's script is run to download it, and the file is looked for - * again. + * {@link FileFinder#getFilePath(String)} does, and then in the data + * repository at the root of this repository, which is above the folder + * of the examples that their READMEs run them from. A free data file + * that is missing has a {@code .link} in the data repository, so + * the data repository's script is run to download it, and the file is + * looked for again. * @param file the name of the file, or the end of its path * @return the file * @throws IllegalArgumentException if the file was not found, naming the * command that downloads it when there is one */ public static File findDataFile(String file) { + File repository = findDataRepository(); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException notFound) { File link; try { - link = getFilePath(file + ".link"); + link = find(file + ".link", repository); } catch (IllegalArgumentException noLink) { throw notFound; } String command = runDownload(link); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException stillNotFound) { String message = "Could not download '" + file + "'. Run this command to download it: " + command; @@ -204,6 +284,37 @@ public static File findDataFile(String file) { } } + /** + * Finds a file below the project root, then below the data repository. + */ + private static File find(String file, File repository) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + if (repository == null) { + throw notFound; + } + return getFilePath(file, repository.getPath()); + } + } + + /** + * The data repository, cxx/data, in the nearest folder at or above the + * working directory that has one. + * @return the folder, or null if there is none + */ + private static File findDataRepository() { + File folder = new File(System.getProperty("user.dir")).getAbsoluteFile(); + while (folder != null) { + File data = new File(new File(folder, "cxx"), "data"); + if (new File(data, "get-data-files.sh").isFile()) { + return data; + } + folder = folder.getParentFile(); + } + return null; + } + /** * Runs the data repository's script for the file that a link is for, * and waits for it to finish. The data repository is the folder above @@ -215,17 +326,30 @@ private static String runDownload(File link) { String name = link.getName().substring( 0, link.getName().length() - ".link".length()); File data = link.getAbsoluteFile().getParentFile().getParentFile(); - List command; + List command = new ArrayList<>(); + String powerShell = null; if (System.getProperty("os.name", "") .toLowerCase(Locale.ROOT).startsWith("windows")) { - command = Arrays.asList("powershell", "-NoProfile", - "-ExecutionPolicy", "Bypass", "-File", - new File(data, "get-data-files.ps1").getPath(), name, - "-Source", DOWNLOAD_SOURCE); + powerShell = powerShell(); + command.add(powerShell); + command.add("-NoProfile"); + command.add("-ExecutionPolicy"); + command.add("Bypass"); + command.add("-File"); + command.add(new File(data, "get-data-files.ps1").getPath()); + command.add(name); + if (IN_GITHUB_ACTIONS == false) { + command.add("-Source"); + command.add(DOWNLOAD_SOURCE); + } } else { - command = Arrays.asList("sh", - new File(data, "get-data-files.sh").getPath(), - "-s", DOWNLOAD_SOURCE, name); + command.add("sh"); + command.add(new File(data, "get-data-files.sh").getPath()); + if (IN_GITHUB_ACTIONS == false) { + command.add("-s"); + command.add(DOWNLOAD_SOURCE); + } + command.add(name); } List typed = new ArrayList<>(); for (String argument : command) { @@ -236,7 +360,13 @@ private static String runDownload(File link) { logger.info("'{}' is missing, so it is being downloaded by running {}", name, line); try { - new ProcessBuilder(command).inheritIO().start().waitFor(); + ProcessBuilder builder = new ProcessBuilder(command).inheritIO(); + if (WINDOWS_POWERSHELL.equals(powerShell)) { + // Started from PowerShell 7, Windows PowerShell inherits its + // module path and then cannot load its own commands. + builder.environment().remove("PSModulePath"); + } + builder.start().waitFor(); } catch (IOException e) { logger.error("Could not run {}. {}", line, e.getMessage()); } catch (InterruptedException e) { @@ -244,4 +374,19 @@ private static String runDownload(File link) { } return line; } + + /** + * PowerShell 7 when it is on the path, otherwise Windows PowerShell. + */ + private static String powerShell() { + String path = System.getenv("PATH"); + if (path != null) { + for (String folder : path.split(File.pathSeparator)) { + if (new File(folder.replace("\"", ""), "pwsh.exe").isFile()) { + return "pwsh"; + } + } + } + return WINDOWS_POWERSHELL; + } } diff --git a/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java b/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java index bf4467038..2df9b35a9 100644 --- a/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java +++ b/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java @@ -22,11 +22,13 @@ package fiftyone.devicedetection.examples.web; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -40,6 +42,7 @@ import java.util.Date; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static fiftyone.devicedetection.examples.shared.PropertyHelper.asString; import static fiftyone.devicedetection.examples.shared.PropertyHelper.tryGet; @@ -172,6 +175,9 @@ public static void doDataFileAgeWarning(PrintWriter out, FlowData flowData) { public static void doDeviceData(PrintWriter out, DeviceData device, FlowData flowData, String dataFileLocation) { FlowElement engine = getFlowElement(flowData); + // The on-premise engine says which properties its data file has. + DeviceDetectionHashEngine onPremise = engine instanceof DeviceDetectionHashEngine ? + (DeviceDetectionHashEngine) engine : null; String content = ""; if (engine instanceof CloudRequestEngine) { content = "

    The following values were detected using the cloud " + @@ -200,20 +206,35 @@ public static void doDeviceData(PrintWriter out, DeviceData device, FlowData flo "Key" + "Value\n" + " \n" + - " Hardware Vendor" + asString(tryGet(device::getHardwareVendor)) + "\n" + - " Hardware Name" + asString(tryGet(device::getHardwareName)) + "\n" + - " Device Type" + asString(tryGet(device::getDeviceType)) + "\n" + - " Platform Vendor" + asString(tryGet(device::getPlatformVendor)) + "\n" + - " Platform Name" + asString(tryGet(device::getPlatformName)) + "\n" + - " Platform Version" + asString(tryGet(device::getPlatformVersion)) + "\n" + - " Browser Vendor" + asString(tryGet(device::getBrowserVendor)) + "\n" + - " Browser Name" + asString(tryGet((device::getBrowserName))) + "\n" + - " Browser Version" + asString(tryGet(device::getBrowserVersion)) + "\n" + + " Hardware Vendor" + value(onPremise, "HardwareVendor", device::getHardwareVendor) + "\n" + + " Hardware Name" + value(onPremise, "HardwareName", device::getHardwareName) + "\n" + + " Device Type" + value(onPremise, "DeviceType", device::getDeviceType) + "\n" + + " Platform Vendor" + value(onPremise, "PlatformVendor", device::getPlatformVendor) + "\n" + + " Platform Name" + value(onPremise, "PlatformName", device::getPlatformName) + "\n" + + " Platform Version" + value(onPremise, "PlatformVersion", device::getPlatformVersion) + "\n" + + " Browser Vendor" + value(onPremise, "BrowserVendor", device::getBrowserVendor) + "\n" + + " Browser Name" + value(onPremise, "BrowserName", device::getBrowserName) + "\n" + + " Browser Version" + value(onPremise, "BrowserVersion", device::getBrowserVersion) + "\n" + " \n" + " \n" + "

    \n"); } + /** + * The value of a property for the page. With the on-premise engine, a + * property its data file does not have needs the Enterprise data file, + * and the value is only read when the data file has the property. + * @param onPremise the on-premise engine, or null for the cloud + */ + private static String value(DeviceDetectionHashEngine onPremise, + String property, + Supplier> value) { + if (onPremise != null && DataFileHelper.hasProperty(onPremise, property) == false) { + return DataFileHelper.NEEDS_ENTERPRISE; + } + return asString(tryGet(value)); + } + public static void doUachInfo(PrintWriter out) { //language=html out.append( diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 9344df4e0..2d989f323 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -14,28 +14,37 @@ The data files for on-premise IP intelligence are kept in `cxx/data/ip-intellige folder of the [data](https://github.com/51Degrees/data) repository, which is a submodule of cxx. -**IMPORTANT**: The enterprise IP Intelligence data file must be placed in the -`cxx/data/ip-intelligence` directory of this repository. The expected filename is -`51Degrees-EnterpriseIpiV41.ipi`. - -Two free data files are also usable for many of the examples, the ASN file -(`51Degrees-IPIV4AsnIpiV41.ipi`) and the Lite file (`51Degrees-LiteV41.ipi`). Run -`cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download -them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. An example -that needs one of them and cannot find it runs the script for that file, and logs the -command it runs. When the enterprise file is -absent, the tests for examples that work with any data file fall back to the ASN file and -then the Lite file. The OfflineProcessing and PerformanceBenchmark examples need the -RegisteredName property, which the ASN file does not contain, so their tests fall back to -the Lite file only. Tests for examples needing enterprise-only properties (Metrics, Compare -and Suspicious) are skipped instead. +The examples use the free data files by default, so they run from a fresh clone with no +settings. The examples that show location and network properties use the Lite file +(`51Degrees-LiteV41.ipi`), which has the registration properties, and the performance and +update examples use the much smaller ASN file (`51Degrees-IPIV4AsnIpiV41.ipi`). An example +that cannot find its file runs the data repository's script to download it, and logs the +command it runs. To download both first, run `cxx/data/get-data-files.sh ip-intelligence` +from the root of this repository, or `cxx\data\get-data-files.ps1 ip-intelligence` in +PowerShell on Windows. + +Each example shows the properties that only the Enterprise data file has as needing it, and +gives a link to ask for a copy. The Compare, Metrics and Suspicious examples need those +properties, so with a free data file they say which they need and stop. +[Contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=enterprise-data-file) for a copy of +the Enterprise data file. To use it, place it in `cxx/data/ip-intelligence` and pass its path +as the first argument, or set the `51DEGREES_IPI_PATH` environment variable or system +property to its path. + +The `ExamplesWithoutSettingsTest` tests run each console example as a fresh clone does. The +other tests use the Enterprise data file, `51Degrees-EnterpriseIpiV41.ipi`, when it is in +`cxx/data/ip-intelligence`. When it is absent, the tests for examples that work with any data +file fall back to the ASN file and then the Lite file. The OfflineProcessing and +PerformanceBenchmark tests fall back to the Lite file only, and the tests of the Metrics, +Compare and Suspicious examples with the Enterprise data file are skipped. ``` 51degrees-java/ ├── cxx/ │ └── data/ │ └── ip-intelligence/ -│ └── 51Degrees-EnterpriseIpiV41.ipi +│ ├── 51Degrees-LiteV41.ipi +│ └── 51Degrees-IPIV4AsnIpiV41.ipi └── examples/ └── ip-intelligence/ ├── console/ @@ -46,11 +55,14 @@ and Suspicious) are skipped instead. The examples locate the data file in the following order: 1. The `51DEGREES_IPI_PATH` environment variable or system property, which can - be set to an explicit path to the data file. -2. A search of the project folder hierarchy for the expected file name. -3. The expected location `cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi` - (the configuration files for the getting started examples fall back to the - free Lite file `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi`). + be set to an explicit path to the data file. It is used in place of any + file named without a full path. +2. The path given as the first argument, or the free file the example uses + by default, searched for below the folder the example is run from and then + below `cxx/data`, so that the examples find it whether they are run from + the root of this repository or from this folder. +3. A free file that is still missing is downloaded by the data repository's + script. This project contains sub-modules - **console**, giving examples that are intended to be run from the command line/console and **web**, illustrating use diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index 4b994e3b9..dfd8ccb11 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -56,8 +56,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=contact-us). + * This example needs the location properties only the Enterprise IP Intelligence data file has. + * With the free Lite data file, which it uses by default, it says which and stops. To obtain an + * Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=contact-us). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -67,6 +68,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.IPIntelligenceOnPremisePipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.examples.console.areas.Calculations; @@ -87,6 +89,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -95,7 +98,7 @@ import java.util.concurrent.Future; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.firstValue; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -122,6 +125,22 @@ public class CompareOnPrem { */ public static final String DEFAULT_OUTPUT_FILE = "compare-output.csv"; + /** + * The properties the comparison asks the engine for, which only the + * Enterprise data file has. + */ + public static final List LOCATION_PROPERTIES = Arrays.asList( + "Latitude", + "Longitude", + "LocationConfidence", + "Areas"); + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=enterprise-data-file"; + /** * Mean radius of the earth in kilometers, used in the haversine distance * calculation. @@ -159,12 +178,22 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = DataFileHelper.getDataFileLocation( + args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH); // Use the supplied path for the truth CSV file String csvTruthFile = args.length > 1 ? args[1] : GEOIP_COMPARISON_EVIDENCE_REL_PATH; // Get the location for the output file String outputFile = args.length > 2 ? args[2] : DEFAULT_OUTPUT_FILE; + // The free data files have none of the location properties. + if (DataFileHelper.lacksEnterpriseProperties(dataFile, + "Comparing known locations with IP Intelligence", + LOCATION_PROPERTIES, + new PrintWriter(System.out), + CONTACT_US)) { + return; + } + try (PrintWriter writer = new PrintWriter(new FileWriter(outputFile))) { run(dataFile, csvTruthFile, writer); } @@ -191,13 +220,13 @@ public static void run(String dataFile, throw e; } - File truthFile = getFilePath(csvTruthFile); + File truthFile = DataFileHelper.findDataFile(csvTruthFile); int processors = Runtime.getRuntime().availableProcessors(); // Build a new pipeline with an on-premise IP Intelligence engine using // the max performance profile and restricted to the properties needed. - try (Pipeline pipeline = new IPIntelligencePipelineBuilder() + IPIntelligenceOnPremisePipelineBuilder builder = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) // We use the max performance profile for optimal lookup // speed in this example. See the documentation for more detail @@ -210,14 +239,13 @@ public static void run(String dataFile, .setAutoUpdate(false) .setDataUpdateOnStartup(false) .setDataFileSystemWatcher(false) - // Set to only return from processing the properties needed. - .setProperty("Latitude") - .setProperty("Longitude") - .setProperty("LocationConfidence") - .setProperty("Areas") // Optimize for the expected parallel workload. - .setConcurrency(processors) - .build()) { + .setConcurrency(processors); + // Set to only return from processing the properties needed. + for (String property : LOCATION_PROPERTIES) { + builder.setProperty(property); + } + try (Pipeline pipeline = builder.build()) { // Read the truth records, skipping duplicate IP addresses. List truths = readTruths(truthFile); diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 07f33c130..7c0940afe 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -35,9 +35,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java). * - * This example requires: - * - A Device Detection data file (.hash format) - * - An IP Intelligence data file (.ipi format) + * By default the example uses the free Lite Device Detection and IP Intelligence data files, + * which it downloads when they are missing, and shows each property only the Enterprise data + * files have as needing them. To obtain the Enterprise data files, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedmixed.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -55,6 +56,8 @@ import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.flowelements.FiftyOnePipelineBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +70,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.*; @@ -84,7 +88,13 @@ public class GettingStartedMixed { private static final Logger logger = LoggerFactory.getLogger(GettingStartedMixed.class); public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; - public static final String IPI_DATA_FILE_DEFAULT = DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; + public static final String IPI_DATA_FILE_DEFAULT = DataFileHelper.LITE_DATA_FILE_REL_PATH; + + /** + * Where to ask for a copy of the Enterprise data files, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedmixed.java&utm_term=enterprise-data-file"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -139,20 +149,27 @@ public static void run(String ddDataFile, try (Pipeline pipeline = new FiftyOnePipelineBuilder() .buildFromConfiguration(pipelineOptions)) { + // The engines say which properties their data files have. + DeviceDetectionHashEngine ddEngine = pipeline.getElement(DeviceDetectionHashEngine.class); + IPIntelligenceOnPremiseEngine ipiEngine = pipeline.getElement(IPIntelligenceOnPremiseEngine.class); + // Process each set of evidence + boolean needsEnterprise = false; for (Map evidence : evidenceList) { - analyzeEvidence(evidence, pipeline, outputStream); + needsEnterprise |= analyzeEvidence( + evidence, pipeline, ddEngine, ipiEngine, outputStream); + } + if (needsEnterprise) { + DataFileHelper.writeEnterpriseNote(new PrintWriter(outputStream), CONTACT_US); } // Log data file info for both engines - DeviceDetectionHashEngine ddEngine = pipeline.getElement(DeviceDetectionHashEngine.class); if (ddEngine != null) { logger.info("Device Detection data file: '{}', tier: '{}'", ddEngine.getDataFileMetaData().getDataFilePath(), ddEngine.getDataSourceTier()); } - IPIntelligenceOnPremiseEngine ipiEngine = pipeline.getElement(IPIntelligenceOnPremiseEngine.class); if (ipiEngine != null) { DataFileHelper.logDataFileInfo(ipiEngine); } @@ -195,9 +212,18 @@ public static List> setUpCombinedEvidence() { return evidence; } - private static void analyzeEvidence(Map evidence, - Pipeline pipeline, - OutputStream out) throws Exception { + /** + * Processes the evidence and writes both engines' results, showing each + * property the data files in use do not have as needing the Enterprise + * data file. + * @return true if a property needed the Enterprise data file + */ + private static boolean analyzeEvidence(Map evidence, + Pipeline pipeline, + DeviceDetectionHashEngine ddEngine, + IPIntelligenceOnPremiseEngine ipiEngine, + OutputStream out) throws Exception { + boolean needsEnterprise = false; PrintWriter writer = new PrintWriter(out); try (FlowData data = pipeline.createFlowData()) { @@ -219,40 +245,99 @@ private static void analyzeEvidence(Map evidence, writer.println("\nDevice Detection Results:"); writer.println("-------------------------"); DeviceData device = data.get(DeviceData.class); - writer.println("\tMobile Device: " + asString(tryGet(device::getIsMobile))); - writer.println("\tPlatform Name: " + asString(tryGet(device::getPlatformName))); - writer.println("\tPlatform Version: " + asString(tryGet(device::getPlatformVersion))); - writer.println("\tBrowser Name: " + asString(tryGet(device::getBrowserName))); - writer.println("\tBrowser Version: " + asString(tryGet(device::getBrowserVersion))); - writer.println("\tHardware Vendor: " + asString(tryGet(device::getHardwareVendor))); - writer.println("\tHardware Name: " + asString(tryGet(device::getHardwareName))); - writer.println("\tDevice Type: " + asString(tryGet(device::getDeviceType))); - writer.println("\tScreen Width: " + asString(tryGet(device::getScreenPixelsWidth))); - writer.println("\tScreen Height: " + asString(tryGet(device::getScreenPixelsHeight))); + needsEnterprise |= writeValue(writer, "Mobile Device", + hasProperty(ddEngine, "IsMobile"), device::getIsMobile); + needsEnterprise |= writeValue(writer, "Platform Name", + hasProperty(ddEngine, "PlatformName"), device::getPlatformName); + needsEnterprise |= writeValue(writer, "Platform Version", + hasProperty(ddEngine, "PlatformVersion"), device::getPlatformVersion); + needsEnterprise |= writeValue(writer, "Browser Name", + hasProperty(ddEngine, "BrowserName"), device::getBrowserName); + needsEnterprise |= writeValue(writer, "Browser Version", + hasProperty(ddEngine, "BrowserVersion"), device::getBrowserVersion); + needsEnterprise |= writeValue(writer, "Hardware Vendor", + hasProperty(ddEngine, "HardwareVendor"), device::getHardwareVendor); + needsEnterprise |= writeValue(writer, "Hardware Name", + hasProperty(ddEngine, "HardwareName"), device::getHardwareName); + needsEnterprise |= writeValue(writer, "Device Type", + hasProperty(ddEngine, "DeviceType"), device::getDeviceType); + needsEnterprise |= writeValue(writer, "Screen Width", + hasProperty(ddEngine, "ScreenPixelsWidth"), device::getScreenPixelsWidth); + needsEnterprise |= writeValue(writer, "Screen Height", + hasProperty(ddEngine, "ScreenPixelsHeight"), device::getScreenPixelsHeight); // Get IP Intelligence results writer.println("\nIP Intelligence Results:"); writer.println("------------------------"); IPIntelligenceData ipData = data.get(IPIntelligenceData.class); - writer.println("\tRegisteredName: " + asStringProperty(tryGet(ipData::getRegisteredName))); - writer.println("\tRegisteredOwner: " + asStringProperty(tryGet(ipData::getRegisteredOwner))); - writer.println("\tRegisteredCountry: " + asStringProperty(tryGet(ipData::getRegisteredCountry))); - writer.println("\tIpRangeStart: " + asIPAddressProperty(tryGet(ipData::getIpRangeStart))); - writer.println("\tIpRangeEnd: " + asIPAddressProperty(tryGet(ipData::getIpRangeEnd))); - writer.println("\tCountry: " + asStringProperty(tryGet(ipData::getCountry))); - writer.println("\tCountryCode: " + asStringProperty(tryGet(ipData::getCountryCode))); - writer.println("\tCountryCode3: " + asStringProperty(tryGet(ipData::getCountryCode3))); - writer.println("\tRegion: " + asStringProperty(tryGet(ipData::getRegion))); - writer.println("\tState: " + asStringProperty(tryGet(ipData::getState))); - writer.println("\tTown: " + asStringProperty(tryGet(ipData::getTown))); - writer.println("\tLatitude: " + asFloatProperty(tryGet(ipData::getLatitude))); - writer.println("\tLongitude: " + asFloatProperty(tryGet(ipData::getLongitude))); - writer.println("\tAreas: " + asWktStringProperty(tryGet(ipData::getAreas))); - writer.println("\tAccuracyRadiusMin: " + asIntegerProperty(tryGet(ipData::getAccuracyRadiusMin))); - writer.println("\tTimeZoneOffset: " + asIntegerProperty(tryGet(ipData::getTimeZoneOffset))); + needsEnterprise |= writeValue(writer, "RegisteredName", + DataFileHelper.hasProperty(ipiEngine, "RegisteredName"), ipData::getRegisteredName); + needsEnterprise |= writeValue(writer, "RegisteredOwner", + DataFileHelper.hasProperty(ipiEngine, "RegisteredOwner"), ipData::getRegisteredOwner); + needsEnterprise |= writeValue(writer, "RegisteredCountry", + DataFileHelper.hasProperty(ipiEngine, "RegisteredCountry"), ipData::getRegisteredCountry); + needsEnterprise |= writeValue(writer, "IpRangeStart", + DataFileHelper.hasProperty(ipiEngine, "IpRangeStart"), ipData::getIpRangeStart); + needsEnterprise |= writeValue(writer, "IpRangeEnd", + DataFileHelper.hasProperty(ipiEngine, "IpRangeEnd"), ipData::getIpRangeEnd); + needsEnterprise |= writeValue(writer, "Country", + DataFileHelper.hasProperty(ipiEngine, "Country"), ipData::getCountry); + needsEnterprise |= writeValue(writer, "CountryCode", + DataFileHelper.hasProperty(ipiEngine, "CountryCode"), ipData::getCountryCode); + needsEnterprise |= writeValue(writer, "CountryCode3", + DataFileHelper.hasProperty(ipiEngine, "CountryCode3"), ipData::getCountryCode3); + needsEnterprise |= writeValue(writer, "Region", + DataFileHelper.hasProperty(ipiEngine, "Region"), ipData::getRegion); + needsEnterprise |= writeValue(writer, "State", + DataFileHelper.hasProperty(ipiEngine, "State"), ipData::getState); + needsEnterprise |= writeValue(writer, "Town", + DataFileHelper.hasProperty(ipiEngine, "Town"), ipData::getTown); + needsEnterprise |= writeValue(writer, "Latitude", + DataFileHelper.hasProperty(ipiEngine, "Latitude"), ipData::getLatitude); + needsEnterprise |= writeValue(writer, "Longitude", + DataFileHelper.hasProperty(ipiEngine, "Longitude"), ipData::getLongitude); + needsEnterprise |= writeValue(writer, "Areas", + DataFileHelper.hasProperty(ipiEngine, "Areas"), ipData::getAreas); + needsEnterprise |= writeValue(writer, "AccuracyRadiusMin", + DataFileHelper.hasProperty(ipiEngine, "AccuracyRadiusMin"), ipData::getAccuracyRadiusMin); + needsEnterprise |= writeValue(writer, "TimeZoneOffset", + DataFileHelper.hasProperty(ipiEngine, "TimeZoneOffset"), ipData::getTimeZoneOffset); } writer.println(); writer.flush(); + return needsEnterprise; + } + + /** + * Writes a property, or that it needs the Enterprise data file when the + * data file in use does not have it. The value is only read when the + * data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean writeValue(PrintWriter writer, + String name, + boolean hasProperty, + Supplier> value) { + if (hasProperty) { + writer.println("\t" + name + ": " + asString(value.get())); + return false; + } + writer.println("\t" + name + ": " + DataFileHelper.NEEDS_ENTERPRISE); + return true; + } + + /** + * True when the Device Detection engine's data file has the property. + * The Lite data file leaves out the properties only the Enterprise data + * file has, such as HardwareVendor, HardwareName and DeviceType. + */ + private static boolean hasProperty(DeviceDetectionHashEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; } /** diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 41a7712a5..268d00aa3 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -33,8 +33,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing. The Lite data file has the registration properties, and the example shows each + * property only the Enterprise data file has as needing it. To obtain an Enterprise data file, + * please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -62,16 +64,12 @@ import java.io.PrintWriter; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIntegerProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asFloatProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIPAddressProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWktStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.pipeline.util.FileFinder.getFilePath; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; /** * Provides an illustration of the fundamental elements of carrying out IP Intelligence using @@ -86,23 +84,23 @@ public class GettingStartedOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, the 51degrees IP Intelligence data file needs to be specified as a command line parameter. - - To test this example, you need to: - 1. Host an IP Intelligence data file (.ipi) at a custom URL accessible to this application - 2. Provide the path to that hosted data file as a command line parameter - 3. No license key is required when using a custom URL - - For production use, you will eventually need to use a Distributor service and license key - to keep your data file updated. - - To obtain access to enterprise data files for hosting, please contact us: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=main */ + /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from + the data repository in cxx/data, and is downloaded if it is missing, or you may specify another + file as a command line parameter. + + Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=main */ + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=enterprise-data-file"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); @@ -152,15 +150,20 @@ public static void run(String dataFile, .buildFromConfiguration(pipelineOptions)) { - // carry out some sample detections - for (Map evidence : evidenceList) { - analyzeEvidence(evidence, pipeline, outputStream); - } - /* Get the 'engine' element within the pipeline that performs IP Intelligence. We can use this to get details about the data file as well as meta-data describing things such as the available properties. */ IPIntelligenceOnPremiseEngine engine = pipeline.getElement(IPIntelligenceOnPremiseEngine.class); + + // carry out some sample detections + boolean needsEnterprise = false; + for (Map evidence : evidenceList) { + needsEnterprise |= analyzeEvidence(evidence, pipeline, engine, outputStream); + } + if (needsEnterprise) { + DataFileHelper.writeEnterpriseNote(new PrintWriter(outputStream), CONTACT_US); + } + DataFileHelper.logDataFileInfo(engine); logger.info("All done"); @@ -172,11 +175,15 @@ public static void run(String dataFile, * supplied and output the IP Intelligence results to the output stream. * @param evidence a map representing HTTP headers * @param pipeline a pipeline set up to process the evidence + * @param engine the engine, which says which properties its data file has * @param out somewhere to send the detection results + * @return true if a property needed the Enterprise data file */ - private static void analyzeEvidence(Map evidence, - Pipeline pipeline, - OutputStream out) throws Exception { + private static boolean analyzeEvidence(Map evidence, + Pipeline pipeline, + IPIntelligenceOnPremiseEngine engine, + OutputStream out) throws Exception { + boolean needsEnterprise = false; PrintWriter writer = new PrintWriter(out); /* FlowData is a data structure that is used to convey information required for a lookup and the results of the lookup through the pipeline. Information required for @@ -210,25 +217,45 @@ private static void analyzeEvidence(Map evidence, property dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=analyzeevidence for details of all available properties. */ - // Output all the properties using shared PropertyHelper methods - writer.println("\tRegisteredName: " + asStringProperty(tryGet(ipData::getRegisteredName))); - writer.println("\tRegisteredOwner: " + asStringProperty(tryGet(ipData::getRegisteredOwner))); - writer.println("\tRegisteredCountry: " + asStringProperty(tryGet(ipData::getRegisteredCountry))); - writer.println("\tIpRangeStart: " + asIPAddressProperty(tryGet(ipData::getIpRangeStart))); - writer.println("\tIpRangeEnd: " + asIPAddressProperty(tryGet(ipData::getIpRangeEnd))); - writer.println("\tCountry: " + asStringProperty(tryGet(ipData::getCountry))); - writer.println("\tCountryCode: " + asStringProperty(tryGet(ipData::getCountryCode))); - writer.println("\tCountryCode3: " + asStringProperty(tryGet(ipData::getCountryCode3))); - writer.println("\tRegion: " + asStringProperty(tryGet(ipData::getRegion))); - writer.println("\tState: " + asStringProperty(tryGet(ipData::getState))); - writer.println("\tTown: " + asStringProperty(tryGet(ipData::getTown))); - writer.println("\tLatitude: " + asFloatProperty(tryGet(ipData::getLatitude))); - writer.println("\tLongitude: " + asFloatProperty(tryGet(ipData::getLongitude))); - writer.println("\tAreas: " + asWktStringProperty(tryGet(ipData::getAreas))); - writer.println("\tAccuracyRadiusMin: " + asIntegerProperty(tryGet(ipData::getAccuracyRadiusMin))); - writer.println("\tTimeZoneOffset: " + asIntegerProperty(tryGet(ipData::getTimeZoneOffset))); + // The free Lite data file has the registration properties, and + // only the Enterprise data file has the others. + needsEnterprise |= writeValue(writer, engine, "RegisteredName", ipData::getRegisteredName); + needsEnterprise |= writeValue(writer, engine, "RegisteredOwner", ipData::getRegisteredOwner); + needsEnterprise |= writeValue(writer, engine, "RegisteredCountry", ipData::getRegisteredCountry); + needsEnterprise |= writeValue(writer, engine, "IpRangeStart", ipData::getIpRangeStart); + needsEnterprise |= writeValue(writer, engine, "IpRangeEnd", ipData::getIpRangeEnd); + needsEnterprise |= writeValue(writer, engine, "Country", ipData::getCountry); + needsEnterprise |= writeValue(writer, engine, "CountryCode", ipData::getCountryCode); + needsEnterprise |= writeValue(writer, engine, "CountryCode3", ipData::getCountryCode3); + needsEnterprise |= writeValue(writer, engine, "Region", ipData::getRegion); + needsEnterprise |= writeValue(writer, engine, "State", ipData::getState); + needsEnterprise |= writeValue(writer, engine, "Town", ipData::getTown); + needsEnterprise |= writeValue(writer, engine, "Latitude", ipData::getLatitude); + needsEnterprise |= writeValue(writer, engine, "Longitude", ipData::getLongitude); + needsEnterprise |= writeValue(writer, engine, "Areas", ipData::getAreas); + needsEnterprise |= writeValue(writer, engine, "AccuracyRadiusMin", ipData::getAccuracyRadiusMin); + needsEnterprise |= writeValue(writer, engine, "TimeZoneOffset", ipData::getTimeZoneOffset); } writer.println(); writer.flush(); + return needsEnterprise; + } + + /** + * Writes a property, or that it needs the Enterprise data file when the + * data file in use does not have it. The value is only read when the + * data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean writeValue(PrintWriter writer, + IPIntelligenceOnPremiseEngine engine, + String property, + Supplier> value) { + if (DataFileHelper.hasProperty(engine, property)) { + writer.println("\t" + property + ": " + asString(value.get())); + return false; + } + writer.println("\t" + property + ": " + DataFileHelper.NEEDS_ENTERPRISE); + return true; } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 6e127722d..b7d2d7206 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -42,7 +42,7 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.pipeline.util.FileFinder.getFilePath; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; /** * This example shows how to access metadata about the IP Intelligence properties that are available @@ -56,20 +56,28 @@ * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing. To obtain an Enterprise data file, which has many more properties, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=header). */ public class MetadataOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, by default, the 51degrees IP Intelligence data file needs to be somewhere in the project - space, or you may specify another file as a command line parameter. + /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from + the data repository in cxx/data, and is downloaded if it is missing, or you may specify another + file as a command line parameter. - For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=main */ + Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=main */ + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=enterprise-data-file"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; run(dataFile, System.out); } @@ -119,6 +127,9 @@ public static void run(String dataFile, OutputStream output) throws Exception { writer.flush(); DataFileHelper.logDataFileInfo(engine); + if (DataFileHelper.isFreeDataFile(engine)) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 347c35b60..3d3ecc248 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -40,8 +40,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=contact-us). + * This example needs the location properties only the Enterprise IP Intelligence data file has. + * With the free Lite data file, which it uses by default, it says which and stops. To obtain an + * Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=contact-us). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -72,6 +73,7 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,7 +89,7 @@ import java.util.function.Predicate; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.firstValue; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -115,6 +117,12 @@ public class MetricsOnPrem { */ public static final String DEFAULT_OUTPUT_FILE = "metrics-output.csv"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=enterprise-data-file"; + /** * Properties needed by the metrics. The first eight form the metric key. */ @@ -289,13 +297,23 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = DataFileHelper.getDataFileLocation( + args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH); // Get the location for the output file String outputFile = args.length > 1 ? args[1] : DEFAULT_OUTPUT_FILE; // Get the sample percentage or use the default double samplePercentage = args.length > 2 ? Double.parseDouble(args[2]) : DEFAULT_SAMPLE_PERCENTAGE; + // The free data files have none of the location properties. + if (DataFileHelper.lacksEnterpriseProperties(dataFile, + "Measuring the location areas", + Arrays.asList(PROPERTIES), + new PrintWriter(System.out), + CONTACT_US)) { + return; + } + try (PrintWriter writer = new PrintWriter(new FileWriter(outputFile))) { // Only include IP addresses with periods in them, i.e. IPv4. // There are too many IPv6 addresses for the metrics example to diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 608989b73..540c69a66 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -44,8 +44,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing, and records Country, which only the Enterprise data file has, as needing it. To + * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -74,8 +75,9 @@ import java.util.stream.Collectors; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -95,10 +97,10 @@ */ public class OfflineProcessing { static final Logger logger = LoggerFactory.getLogger(OfflineProcessing.class); - // This 51degrees IP Intelligence data file (distributed with the source) needs to - // be somewhere in the project space + // By default the free 51degrees Lite IP Intelligence data file is used from + // the data repository in cxx/data, and is downloaded if it is missing. // - // For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=datadir + // Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=datadir private static final String dataDir = "ip-intelligence"; // This 51degrees file of 20,000 examples (distributed with the source) // needs to be somewhere in the project space. Additional evidence files @@ -106,13 +108,19 @@ public class OfflineProcessing { public static final String HEADER_EVIDENCE_YML = dataDir + "/evidence.yml"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use provided data file argument, or fall back to default data file path - String dataFile = (args.length > 0) ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; - - File evidenceFile = getFilePath(HEADER_EVIDENCE_YML); + String dataFile = (args.length > 0) ? args[0] : LITE_DATA_FILE_REL_PATH; + + File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(dataFile, Files.newInputStream(evidenceFile.toPath()), System.out); } @@ -192,6 +200,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws try (Writer writer = new OutputStreamWriter(os)) { // read a batch of IP evidence from the stream int count = 0; + boolean needsEnterprise = false; while (evidenceIterator.hasNext() && count < 20) { // Flow data is the container for inputs and outputs that // flow through the pipeline a flowdata instance is @@ -229,9 +238,16 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws resultMap.put("ip.RegisteredName", asStringProperty(ipData.getRegisteredName())); resultMap.put("ip.RegisteredOwner", asStringProperty(ipData.getRegisteredOwner())); resultMap.put("ip.RegisteredCountry", asStringProperty(ipData.getRegisteredCountry())); + // Only the Enterprise data file has Country. + if (DataFileHelper.hasProperty(engine, "Country")) { + resultMap.put("ip.Country", asString(ipData.getCountry())); + } else { + resultMap.put("ip.Country", DataFileHelper.NEEDS_ENTERPRISE); + needsEnterprise = true; + } // To output all populated IP Intelligence properties instead of - // the fixed subset above, replace the three puts with the helper: + // the fixed subset above, replace the puts with the helper: // resultMap.putAll(getPopulatedProperties(ipData, "ip.")); // write document to output stream as a YAML document @@ -242,18 +258,16 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws } count++; } + if (needsEnterprise) { + // A comment keeps the output a valid YAML stream. + writer.write("# Some of these properties need the " + + "Enterprise data file. Contact us for a copy at " + + CONTACT_US + "\n"); + } // finish the last YAML document writer.write("...\n"); writer.flush(); logger.info("Finished processing {} records", count); - - if (engine.getDataSourceTier().equals("Lite")) { - logger.warn("You have used a Lite data file which has " + - "limited properties and is of limited accuracy"); - logger.info("The example requires an Enterprise data file " + - "to work fully. Find out about the Enterprise " + - "data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=enterprise-data-file"); - } } } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index 2be3b4599..e9b5f5688 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -34,8 +34,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). + * By default the example uses the free Asn IP Intelligence data file, which it downloads when it + * is missing, and is far smaller than the Lite file. It asks for RegisteredName and AsnName, so + * it runs with any of the data files. To obtain an Enterprise data file, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -54,6 +56,7 @@ import fiftyone.pipeline.core.data.IWeightedValue; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.util.FileFinder; import org.apache.commons.lang3.BooleanUtils; import org.slf4j.Logger; @@ -83,6 +86,18 @@ public class PerformanceBenchmark { public static final Logger logger = LoggerFactory.getLogger(PerformanceBenchmark.class); + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=enterprise-data-file"; + + // true when the benchmarks ran on a free data file + private boolean usedFreeDataFile; + // true when the data file has RegisteredName, which the free Asn data + // file does not, having AsnName in its place + private boolean readRegisteredName; + // where the results of the tests are gathered private List> resultList; private int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; @@ -100,7 +115,8 @@ public class PerformanceBenchmark { public static void main(String[] args) throws Exception { LogbackHelper.configureLogback(FileFinder.getFilePath("logback.xml")); - String dataFilename = args.length > 0 ? args[0] : null; + String dataFilename = args.length > 0 ? args[0] : + DataFileHelper.ASN_DATA_FILE_REL_PATH; String evidenceFilename = args.length > 1 ? args[1] : null; int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; if (args.length > 2) { @@ -147,6 +163,9 @@ protected void runBenchmarks(PerformanceConfiguration[] performanceConfiguration } executeBenchmark(config); } + if (usedFreeDataFile) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } logger.info("Finished Performance example"); } @@ -173,7 +192,11 @@ private void executeBenchmark(PerformanceConfiguration config) throws Exception setPipelinePerformanceProperties(builder, config); pipeline = builder.build(); - DataFileHelper.logDataFileInfo(pipeline.getElement(IPIntelligenceOnPremiseEngine.class)); + IPIntelligenceOnPremiseEngine engine = + pipeline.getElement(IPIntelligenceOnPremiseEngine.class); + DataFileHelper.logDataFileInfo(engine); + usedFreeDataFile = DataFileHelper.isFreeDataFile(engine); + readRegisteredName = DataFileHelper.hasProperty(engine, "RegisteredName"); // run the benchmarks twice, once to warm up the JVM logger.info("Warming up"); @@ -215,8 +238,12 @@ private void setPipelinePerformanceProperties( // reduces processing time compared with requesting properties from multiple components. // If you don't specify any properties, then all properties are returned, // here we choose "all properties" by specifying none, or just "RegisteredName" + // and "AsnName". The free Asn data file has AsnName, and the Lite and + // Enterprise data files have RegisteredName, so the benchmark runs + // with any of them. if (BooleanUtils.isFalse(config.allProperties)) { builder.setProperty("RegisteredName"); + builder.setProperty("AsnName"); } } @@ -260,7 +287,7 @@ private long runTests(Pipeline pipeline) throws Exception { // create a list of callables List> callables = new ArrayList<>(); for (int i = 0; i < numberOfThreads; i++) { - callables.add(new BenchmarkRunnable(pipeline, evidence)); + callables.add(new BenchmarkRunnable(pipeline, evidence, readRegisteredName)); } // start multiple threads in a fixed pool ExecutorService service = Executors.newFixedThreadPool(numberOfThreads); @@ -286,11 +313,15 @@ private static class BenchmarkRunnable implements Callable { private final BenchmarkResult result; private final List> testList; private final Pipeline pipeline; + private final boolean readRegisteredName; - BenchmarkRunnable(Pipeline pipeline, List> evidence) { + BenchmarkRunnable(Pipeline pipeline, + List> evidence, + boolean readRegisteredName) { this.testList = evidence; // initialise the benchmark variables this.pipeline = pipeline; + this.readRegisteredName = readRegisteredName; this.result = new BenchmarkResult(); result.elapsedMillis = 0; @@ -318,8 +349,10 @@ public BenchmarkResult call() { // the same data. IPIntelligenceData ipData = flowData.get(IPIntelligenceData.class); if (ipData != null) { - if (ipData.getRegisteredName().hasValue()) { - String value = ipData.getRegisteredName().getValue(); + AspectPropertyValue name = readRegisteredName ? + ipData.getRegisteredName() : ipData.getAsnName(); + if (name.hasValue()) { + String value = name.getValue(); if (value != null) { result.checkSum += value.hashCode(); // for (IWeightedValue weightedValue : value) { diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index 3df68b732..cfaa02ccd 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -35,8 +35,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us). + * This example needs properties only the Enterprise IP Intelligence data file has. With the free + * Lite data file, which it uses by default, it says which and stops. To obtain an Enterprise data + * file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -57,11 +58,12 @@ import java.io.OutputStream; import java.io.PrintWriter; +import java.util.Arrays; import java.util.List; import java.util.Map; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.firstValue; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -80,16 +82,37 @@ public class SuspiciousOnPrem { private static final Logger logger = LoggerFactory.getLogger(SuspiciousOnPrem.class); - /* In this example, by default, the 51degrees IP Intelligence data file needs to be somewhere - in the project space, or you may specify another file as a command line parameter. + /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from + the data repository in cxx/data, and is downloaded if it is missing, or you may specify another + file as a command line parameter. The Lite data file has none of the properties the example + needs, so it says so and stops. For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us */ + /** + * The properties the judgement uses, which only the Enterprise data file + * has. + */ + public static final List ENTERPRISE_PROPERTIES = Arrays.asList( + "IsCellular", + "HardwareDiversity", + "BrowserDiversity", + "LocationConfidence", + "IsHosted", + "CountryCode", + "HumanProbability"); + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); @@ -117,6 +140,15 @@ public static void run(String dataFile, throw e; } + // The judgement needs properties only the Enterprise data file has. + if (DataFileHelper.lacksEnterpriseProperties(dataFileLocation, + "Judging whether an IP address is suspicious", + ENTERPRISE_PROPERTIES, + new PrintWriter(outputStream), + CONTACT_US)) { + return; + } + /* In this example, we use the IPIntelligencePipelineBuilder and configure it in code. For more information about builders in general see the documentation at diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index 144931e2c..43f14a200 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.PrintWriter; import java.nio.file.*; import java.text.SimpleDateFormat; import java.util.Objects; @@ -65,12 +66,11 @@ *
  • Filesystem Watcher
    • *
  • Daily auto-update
    • * - * ## Distributor Service and License Key - * For production use, you will eventually need to use a Distributor service and license key - * to keep your data file updated. However, for now this example requires a custom URL where - * an updated data file will be hosted. - * - * To obtain access to enterprise data files, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header). + * ## License Key + * Updating the data file downloads the Enterprise data file from the Distributor service, which + * needs a 51Degrees licence key. Without one, the example shows the free Asn data file and says + * where to ask for a key. To obtain a key and an Enterprise data file, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header). * * ## Configuration * - the pipeline must be configured to use a temp file @@ -153,15 +153,8 @@ * # Location * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java). * - * To test this example, you need to: - * 1. Host an IP Intelligence data file (.ipi) at a custom URL accessible to this application - * 2. Provide that custom URL using the WithDataUpdateUrl parameter - * 3. No license key is required when using a custom URL - * - * For production use, you will eventually need to use a Distributor service and license key - * to keep your data file updated. - * - * To obtain access to enterprise data files for hosting, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header-2). + * To test this example you need a licence key for the Enterprise data file. To obtain one, + * please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header-2). * */ @@ -177,11 +170,8 @@ *
  • Daily auto-update
    • * *

    - * For production use, you will eventually need to use a Distributor service and license key - * to keep your data file updated. However, for now this example requires a custom URL where - * an updated data file will be hosted. - * - * To obtain access to enterprise data files, please contact us. + * Updating the data file needs a licence key for the Enterprise data file. To obtain one, please + * contact us. * This license key must be supplied as a command line argument or by setting * an environment variable or system property called {@link UpdateDataFile#UPDATE_EXAMPLE_LICENSE_KEY_NAME} */ @@ -193,6 +183,12 @@ public class UpdateDataFile { private static final Logger logger = LoggerFactory.getLogger(UpdateDataFile.class); + /** + * Where to ask for a licence key and a copy of the Enterprise data file, + * tagged with this example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=licence-key"; + static final UpdateCompletionListener completionListener = new UpdateCompletionListener(); static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss Z"); @@ -222,13 +218,8 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr licenseKey = KeyUtils.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); } if (Objects.isNull(licenseKey) || KeyUtils.isInvalidKey(licenseKey)) { - logger.error("For production use, you will eventually need to use a Distributor service and " + - "license key to keep your data file updated. However, for now this example requires " + - "a custom URL where an updated data file will be hosted. To obtain access to enterprise " + - "data files, please contact us at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=license-key-required. You must supply the " + - "license key as an argument to this program, or as an environment or system variable " + - "named '{}'", UPDATE_EXAMPLE_LICENSE_KEY_NAME); - throw new IllegalArgumentException("No license key available"); + explainLicenseKey(dataFilename, new PrintWriter(System.out, true)); + return; } // work out where the downloaded file will be put, directory must exist @@ -258,9 +249,9 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr DataFileHelper.DatafileInfo metadata = getDatafileMetaData(dataFilename); // and output the results logFileInfo(metadata); - if (metadata.getTier().equals("Lite")) { + if (metadata.isFree()) { logger.error("Will not download an 'Enterprise' data file over the top of " + - "a 'Lite' data file, please supply another location."); + "a free data file, please supply another location."); throw new IllegalArgumentException("File supplied has wrong data tier"); } logger.info("Existing data file will be replaced with downloaded data file"); @@ -305,10 +296,8 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // pass in the update listener which has been configured // to notify when update complete .setDataUpdateService(dataUpdateService) - // For production use, you will eventually need to use a Distributor service and - // license key to keep your data file updated. However, for now this example - // requires a custom URL where an updated data file will be hosted. - // To obtain access to enterprise data files, please contact us at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=run + // Updates from the Distributor service need a licence key for the + // Enterprise data file. To obtain one, please contact us at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=run .setDataUpdateLicenseKey(licenseKey) // Enable update on startup, the auto update system // will be used to check for an update before the @@ -363,6 +352,40 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr } } + /** + * Without a licence key the example cannot download a data file, so it + * shows the data file it would start from, the free Asn file unless + * another is given, and says what an update needs. + * @param dataFilename the data file given, or null + * @param output where to write + * @throws Exception if the data file cannot be read + */ + static void explainLicenseKey(String dataFilename, PrintWriter output) throws Exception { + File dataFile = null; + try { + dataFile = Objects.isNull(dataFilename) ? + DataFileHelper.findDataFile(DataFileHelper.ASN_DATA_FILE_REL_PATH) : + new File(dataFilename); + } catch (IllegalArgumentException e) { + logger.warn(e.getMessage()); + } + if (Objects.nonNull(dataFile) && dataFile.exists()) { + DataFileHelper.DatafileInfo metadata = + getDatafileMetaData(dataFile.getAbsolutePath()); + output.println("The data file in use is the '" + metadata.getTier() + + "' file published on " + simpleDateFormat.format( + metadata.getFileInfo().getDataPublishedDateTime()) + + ", at '" + metadata.getFileInfo().getDataFilePath() + "'."); + } + output.println(); + output.println("Updating the data file downloads the Enterprise data " + + "file, which needs a licence key. Give the key as the first " + + "command line argument, or in the environment variable or " + + "system property '" + UPDATE_EXAMPLE_LICENSE_KEY_NAME + "'. " + + "Contact us for a key and a copy at " + CONTACT_US); + output.flush(); + } + private static void logFileInfo(DataFileHelper.DatafileInfo metadata) { logger.info("Current data file is a {} tier data file, last updated at {}, at " + "location {}. Next update expected {}.", metadata.getTier(), diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java new file mode 100644 index 000000000..45a906637 --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java @@ -0,0 +1,164 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Runs each on-premise console example's main with no arguments, no data + * file setting and no licence key, as someone who has just cloned the + * repository runs it, and checks that it says what the Enterprise data file + * adds, with the link to the contact page. The examples default to the free + * Lite data file, or the free Asn file for performance and updates. CI gives + * the other tests the Enterprise data file, so nothing else tests what a + * fresh clone shows. + */ +public class ExamplesWithoutSettingsTest { + + private static final String CONTACT_US = "https://51degrees.com/contact-us?"; + + private static final String LICENSE_KEY = + UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME; + + /** + * An example's main method. + */ + private interface Main { + void main(String[] args) throws Exception; + } + + /** + * Runs the example's main with no arguments and the data file and + * licence key system properties removed, and returns what it wrote to + * standard output. + */ + private static String runWithoutSettings(Main main) throws Exception { + assumeTrue("Skipping, a data file or licence key is set in the environment", + System.getenv(DataFileHelper.IPI_PATH_ENV_VAR) == null && + System.getenv(LICENSE_KEY) == null && + System.getenv(LICENSE_KEY.toUpperCase()) == null); + String dataFile = System.clearProperty(DataFileHelper.IPI_PATH_ENV_VAR); + String licenseKey = System.clearProperty(LICENSE_KEY); + PrintStream out = System.out; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + System.setOut(new PrintStream(buffer, true, "UTF-8")); + try { + main.main(new String[0]); + } finally { + System.setOut(out); + if (dataFile != null) { + System.setProperty(DataFileHelper.IPI_PATH_ENV_VAR, dataFile); + } + if (licenseKey != null) { + System.setProperty(LICENSE_KEY, licenseKey); + } + } + String output = buffer.toString("UTF-8"); + out.println(output); + return output; + } + + private static void assertContains(String output, String expected) { + assertTrue("The output should contain '" + expected + "'", + output.contains(expected)); + } + + private static void assertStops(String output, String purpose) { + assertContains(output, purpose + " needs the "); + assertContains(output, "so the example stops here."); + assertContains(output, CONTACT_US); + } + + private static void assertOffer(String output) { + assertContains(output, "This example used a free data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void gettingStarted() throws Exception { + String output = runWithoutSettings(GettingStartedOnPrem::main); + assertContains(output, "RegisteredName: "); + assertContains(output, "Country: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "Some of these properties need the Enterprise data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void suspicious() throws Exception { + assertStops(runWithoutSettings(SuspiciousOnPrem::main), + "Judging whether an IP address is suspicious"); + } + + @Test + public void compare() throws Exception { + assertStops(runWithoutSettings(CompareOnPrem::main), + "Comparing known locations with IP Intelligence"); + } + + @Test + public void metrics() throws Exception { + assertStops(runWithoutSettings(MetricsOnPrem::main), + "Measuring the location areas"); + } + + @Test + public void offlineProcessing() throws Exception { + String output = runWithoutSettings(OfflineProcessing::main); + assertContains(output, "ip.Country: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "# Some of these properties need the Enterprise " + + "data file. Contact us for a copy at " + CONTACT_US); + } + + @Test + public void metadata() throws Exception { + assertOffer(runWithoutSettings(MetadataOnPrem::main)); + } + + @Test + public void performance() throws Exception { + assertOffer(runWithoutSettings(PerformanceBenchmark::main)); + } + + @Test + public void updateDataFile() throws Exception { + String output = runWithoutSettings(UpdateDataFile::main); + assertContains(output, "The data file in use is the "); + assertContains(output, "which needs a licence key"); + assertContains(output, CONTACT_US); + } + + @Test + public void mixed() throws Exception { + String output = runWithoutSettings(GettingStartedMixed::main); + assertContains(output, "Hardware Vendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "Country: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, CONTACT_US); + } +} diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index 4cfa22723..149d5dd85 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -25,7 +25,9 @@ import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; import fiftyone.ipintelligence.shared.testhelpers.FileUtils; +import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.util.FileFinder; import org.slf4j.Logger; @@ -33,6 +35,7 @@ import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,7 +43,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Locale; @@ -53,10 +55,24 @@ public class DataFileHelper { /** * Sent with each download, so the Distributor can count the downloads - * made by these examples. + * made by these examples. It is left out inside GitHub Actions, where + * the data repository's script names the build instead, so that test + * runs are not counted as people using the examples. */ private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + private static final boolean IN_GITHUB_ACTIONS = + "true".equals(System.getenv("GITHUB_ACTIONS")); + + private static final String WINDOWS_POWERSHELL = "powershell"; + + /** + * Shown in place of a value for a property that the data file in use + * does not have. Every property the examples ask for is in the + * Enterprise data file. + */ + public static final String NEEDS_ENTERPRISE = "needs the Enterprise data file"; + /** * Folder and name of the enterprise IP Intelligence data file, which * belongs in cxx/data/ip-intelligence, a folder of the data repository. @@ -122,10 +138,12 @@ public static String findAvailableDataFile(String... candidates) { public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; + boolean free; - public DatafileInfo(FiftyOneDataFile fileInfo, String dataSourceTier) { + public DatafileInfo(FiftyOneDataFile fileInfo, String dataSourceTier, boolean free) { this.fileInfo = fileInfo; this.tier = dataSourceTier; + this.free = free; } public AspectEngineDataFile getFileInfo(){ @@ -134,15 +152,30 @@ public AspectEngineDataFile getFileInfo(){ public String getTier() { return tier; } + + /** + * @return true for the free Lite and Asn data files + */ + public boolean isFree() { + return free; + } } public static DatafileInfo getDatafileMetaData(String dataFileLocation) throws Exception { try(IPIntelligenceOnPremiseEngine ddhe = new IPIntelligenceOnPremiseEngineBuilder() .setAutoUpdate(false) .build(dataFileLocation,false)) { - return new DatafileInfo((FiftyOneDataFile) ddhe.getDataFileMetaData(), ddhe.getDataSourceTier()); + return new DatafileInfo((FiftyOneDataFile) ddhe.getDataFileMetaData(), + ddhe.getDataSourceTier(), isFreeDataFile(ddhe)); } } + /** + * Logs the data file in use, and a warning if it is old. An example run + * on a free data file says what the Enterprise data file adds in its + * own output, with {@link #writeEnterpriseNote}, + * {@link #writeEnterpriseOffer} or {@link #lacksEnterpriseProperties}. + * @param engine the engine whose data file is logged + */ public static void logDataFileInfo(IPIntelligenceOnPremiseEngine engine) { // Lite or Enterprise String dataTier = engine.getDataSourceTier(); @@ -154,13 +187,6 @@ public static void logDataFileInfo(IPIntelligenceOnPremiseEngine engine) { String displayDate = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(fileDate); logger.info("Used a '{}' data file, created {}, {} days ago, from location '{}'", dataTier, displayDate, daysOld, dataFileLocation); - if (dataTier.equals("Lite")) { - logger.warn("This example is using the 'Lite ' data file. " + - "This is used for illustration, and has limited " + - "accuracy and capabilities. Find out about the " + - "Enterprise data file on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-datafilehelper.java&utm_term=lite-data-file"); - } if (daysOld > 28) { logger.warn("This example is using a data file that is more " + "than {} days old. A more recent data file " + @@ -168,6 +194,134 @@ public static void logDataFileInfo(IPIntelligenceOnPremiseEngine engine) { } } + /** + * True when the engine's data file has the property. The free Lite data + * file has only RegisteredCountry, RegisteredName and RegisteredOwner, + * and the free Asn data file the Asn properties, such as AsnName. + * @param engine the engine to check + * @param name the name of the property + * @return true if the data file has the property + */ + public static boolean hasProperty(IPIntelligenceOnPremiseEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + /** + * True when the engine uses one of the free data files, Lite or Asn. + * Only the Enterprise data file has the range properties, so a check + * for IpRangeStart tells them apart. + * @param engine the engine to check + * @return true for a free data file + */ + public static boolean isFreeDataFile(IPIntelligenceOnPremiseEngine engine) { + return hasProperty(engine, "IpRangeStart") == false; + } + + /** + * For an example that needs properties only the Enterprise data file + * has, reports when the data file has none of them, and then says so + * and where to ask for a copy of the Enterprise data file. The data + * file is opened with the low memory profile, which reads little of it. + * @param dataFile the data file the example would use + * @param purpose what the example does, as the start of a sentence, for + * example "Comparing known locations with IP Intelligence" + * @param properties the properties the example needs + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + * @return true if the data file has none of the properties + * @throws Exception if the data file cannot be opened + */ + public static boolean lacksEnterpriseProperties(String dataFile, + String purpose, + List properties, + PrintWriter output, + String contactUs) throws Exception { + try (IPIntelligenceOnPremiseEngine engine = new IPIntelligenceOnPremiseEngineBuilder() + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) + .setAutoUpdate(false) + .build(dataFile, false)) { + return lacksEnterpriseProperties( + engine, purpose, properties, output, contactUs); + } + } + + /** + * As {@link #lacksEnterpriseProperties(String, String, List, PrintWriter, String)} + * for an engine the example has already built. + * @param engine the engine the example uses + * @param purpose what the example does, as the start of a sentence + * @param properties the properties the example needs + * @param output where to write + * @param contactUs the link to the contact page, tagged with the example + * @return true if the data file has none of the properties + */ + public static boolean lacksEnterpriseProperties(IPIntelligenceOnPremiseEngine engine, + String purpose, + List properties, + PrintWriter output, + String contactUs) { + for (String property : properties) { + if (hasProperty(engine, property)) { + return false; + } + } + output.println(purpose + " needs the " + joinNames(properties) + + " properties, which only the Enterprise data file has. The " + + "data file in use has none of them, so the example stops " + + "here. Contact us for a copy of the Enterprise data file at " + + contactUs); + output.flush(); + return true; + } + + /** + * Writes, after an example has shown properties that the data file in + * use does not have, where to ask for a copy of the Enterprise data + * file. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseNote(PrintWriter output, String contactUs) { + output.println(); + output.println("Some of these properties need the Enterprise data " + + "file. Contact us for a copy at " + contactUs); + output.flush(); + } + + /** + * Writes, after an example that shows no location properties has run on + * a free data file, what the Enterprise data file adds and where to ask + * for a copy. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { + output.println(); + output.println("This example used a free data file. The Enterprise " + + "data file adds the location, range and connection " + + "properties. Contact us for a copy at " + contactUs); + output.flush(); + } + + /** + * Joins names as "A, B and C". + */ + private static String joinNames(List names) { + if (names.size() == 1) { + return names.get(0); + } + return String.join(", ", names.subList(0, names.size() - 1)) + + " and " + names.get(names.size() - 1); + } + public static void cantFindDataFile(String dataFile) { logger.error("Could not find the data file '{}' which must be " + "somewhere in the project space to be found. An explicit " + @@ -229,7 +383,7 @@ public static String getDataFileLocation(String dataFilename) throws Exception { } if (Objects.isNull(dataFilename)) { - dataFilename = ENTERPRISE_DATA_FILE_REL_PATH; + dataFilename = LITE_DATA_FILE_REL_PATH; } // Check if it's an absolute path @@ -272,28 +426,31 @@ public static String getDataFileLocation(String dataFilename) throws Exception { /** * Finds a file below the project root, as - * {@link FileFinder#getFilePath(String)} does. A free data file that is - * missing has a {@code .link} in the data repository, so the data - * repository's script is run to download it, and the file is looked for - * again. + * {@link FileFinder#getFilePath(String)} does, and then in the data + * repository at the root of this repository, which is above the folder + * of the examples that their READMEs run them from. A free data file + * that is missing has a {@code .link} in the data repository, so + * the data repository's script is run to download it, and the file is + * looked for again. * @param file the name of the file, or the end of its path * @return the file * @throws IllegalArgumentException if the file was not found, naming the * command that downloads it when there is one */ public static File findDataFile(String file) { + File repository = findDataRepository(); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException notFound) { File link; try { - link = getFilePath(file + ".link"); + link = find(file + ".link", repository); } catch (IllegalArgumentException noLink) { throw notFound; } String command = runDownload(link); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException stillNotFound) { String message = "Could not download '" + file + "'. Run this command to download it: " + command; @@ -303,6 +460,37 @@ public static File findDataFile(String file) { } } + /** + * Finds a file below the project root, then below the data repository. + */ + private static File find(String file, File repository) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + if (repository == null) { + throw notFound; + } + return getFilePath(file, repository.getPath()); + } + } + + /** + * The data repository, cxx/data, in the nearest folder at or above the + * working directory that has one. + * @return the folder, or null if there is none + */ + private static File findDataRepository() { + File folder = new File(System.getProperty("user.dir")).getAbsoluteFile(); + while (folder != null) { + File data = new File(new File(folder, "cxx"), "data"); + if (new File(data, "get-data-files.sh").isFile()) { + return data; + } + folder = folder.getParentFile(); + } + return null; + } + /** * Runs the data repository's script for the file that a link is for, * and waits for it to finish. The data repository is the folder above @@ -314,17 +502,30 @@ private static String runDownload(File link) { String name = link.getName().substring( 0, link.getName().length() - ".link".length()); File data = link.getAbsoluteFile().getParentFile().getParentFile(); - List command; + List command = new ArrayList<>(); + String powerShell = null; if (System.getProperty("os.name", "") .toLowerCase(Locale.ROOT).startsWith("windows")) { - command = Arrays.asList("powershell", "-NoProfile", - "-ExecutionPolicy", "Bypass", "-File", - new File(data, "get-data-files.ps1").getPath(), name, - "-Source", DOWNLOAD_SOURCE); + powerShell = powerShell(); + command.add(powerShell); + command.add("-NoProfile"); + command.add("-ExecutionPolicy"); + command.add("Bypass"); + command.add("-File"); + command.add(new File(data, "get-data-files.ps1").getPath()); + command.add(name); + if (IN_GITHUB_ACTIONS == false) { + command.add("-Source"); + command.add(DOWNLOAD_SOURCE); + } } else { - command = Arrays.asList("sh", - new File(data, "get-data-files.sh").getPath(), - "-s", DOWNLOAD_SOURCE, name); + command.add("sh"); + command.add(new File(data, "get-data-files.sh").getPath()); + if (IN_GITHUB_ACTIONS == false) { + command.add("-s"); + command.add(DOWNLOAD_SOURCE); + } + command.add(name); } List typed = new ArrayList<>(); for (String argument : command) { @@ -335,7 +536,13 @@ private static String runDownload(File link) { logger.info("'{}' is missing, so it is being downloaded by running {}", name, line); try { - new ProcessBuilder(command).inheritIO().start().waitFor(); + ProcessBuilder builder = new ProcessBuilder(command).inheritIO(); + if (WINDOWS_POWERSHELL.equals(powerShell)) { + // Started from PowerShell 7, Windows PowerShell inherits its + // module path and then cannot load its own commands. + builder.environment().remove("PSModulePath"); + } + builder.start().waitFor(); } catch (IOException e) { logger.error("Could not run {}. {}", line, e.getMessage()); } catch (InterruptedException e) { @@ -343,4 +550,19 @@ private static String runDownload(File link) { } return line; } + + /** + * PowerShell 7 when it is on the path, otherwise Windows PowerShell. + */ + private static String powerShell() { + String path = System.getenv("PATH"); + if (path != null) { + for (String folder : path.split(File.pathSeparator)) { + if (new File(folder.replace("\"", ""), "pwsh.exe").isFile()) { + return "pwsh"; + } + } + } + return WINDOWS_POWERSHELL; + } } diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 9887afc57..34b00b036 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -85,7 +85,7 @@ import java.util.zip.GZIPOutputStream; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -110,7 +110,7 @@ public class GettingStartedApi extends HttpServlet { public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; String dataFileLocation = DataFileHelper.getDataFileLocation(dataFile); logger.info("Running GettingStartedApi server using data file '{}'", dataFileLocation); diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index b6addc8d7..1accfb336 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -34,9 +34,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java). * - * This example requires: - * - A Device Detection data file (.hash format) - * - An IP Intelligence data file (.ipi format) + * By default the example uses the free Lite Device Detection and IP Intelligence data files, + * which it downloads when they are missing, and shows each property only the Enterprise data + * files have as needing them. To obtain the Enterprise data files, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.mixed-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebmixed.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -48,9 +49,13 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.web.services.FlowDataProviderCore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,9 +70,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_ENV_VAR; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.*; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -79,9 +85,7 @@ *

    * The configuration file for the pipeline is at src/main/webapp/WEB-INF/51Degrees-Mixed.xml *

    - * This example requires: - * - A Device Detection data file (.hash format) - * - An IP Intelligence data file (.ipi format) + * By default the free Lite data files are used. */ public class GettingStartedWebMixed extends HttpServlet { private static final long serialVersionUID = 1734154705981153541L; @@ -95,7 +99,7 @@ public static void main(String[] args) throws Exception { // Accept two command line arguments: device detection data file and IP intelligence data file String ddDataFile = args.length > 0 ? args[0] : DD_DATA_FILE_DEFAULT; - String ipiDataFile = args.length > 1 ? args[1] : ENTERPRISE_DATA_FILE_REL_PATH; + String ipiDataFile = args.length > 1 ? args[1] : LITE_DATA_FILE_REL_PATH; // Set Device Detection data file location try { @@ -110,7 +114,7 @@ public static void main(String[] args) throws Exception { // Set IP Intelligence data file location try { - String ipiPath = fiftyone.ipintelligence.examples.shared.DataFileHelper.getDataFileLocation(ipiDataFile); + String ipiPath = DataFileHelper.getDataFileLocation(ipiDataFile); System.setProperty("TestDataFile", ipiPath); logger.info("Using IP Intelligence data file: {}", ipiPath); } catch (Exception e) { @@ -184,61 +188,99 @@ private String loadTemplate(String templatePath) throws IOException { private String substituteTemplateValues(String template, DeviceData deviceData, IPIntelligenceData ipiData, String inputIp, FlowData flowData) { + // The engines say which properties their data files have. + DeviceDetectionHashEngine ddEngine = + flowData.getPipeline().getElement(DeviceDetectionHashEngine.class); + IPIntelligenceOnPremiseEngine ipiEngine = + flowData.getPipeline().getElement(IPIntelligenceOnPremiseEngine.class); return template .replace("${DATA_FILE_WARNING}", HtmlContentHelper.getDataFileAgeWarning(flowData)) .replace("${INPUT_IP_ADDRESS}", inputIp != null ? inputIp : "") // Device Detection properties - .replace("${HARDWARE_VENDOR}", asString(tryGet(deviceData::getHardwareVendor))) - .replace("${HARDWARE_NAME}", asString(tryGet(deviceData::getHardwareName))) - .replace("${DEVICE_TYPE}", asString(tryGet(deviceData::getDeviceType))) - .replace("${PLATFORM_NAME}", asString(tryGet(deviceData::getPlatformName))) - .replace("${PLATFORM_VERSION}", asString(tryGet(deviceData::getPlatformVersion))) - .replace("${BROWSER_NAME}", asString(tryGet(deviceData::getBrowserName))) - .replace("${BROWSER_VERSION}", asString(tryGet(deviceData::getBrowserVersion))) - .replace("${SCREEN_WIDTH}", asString(tryGet(deviceData::getScreenPixelsWidth))) - .replace("${SCREEN_HEIGHT}", asString(tryGet(deviceData::getScreenPixelsHeight))) + .replace("${HARDWARE_VENDOR}", value(ddEngine, "HardwareVendor", deviceData::getHardwareVendor)) + .replace("${HARDWARE_NAME}", value(ddEngine, "HardwareName", deviceData::getHardwareName)) + .replace("${DEVICE_TYPE}", value(ddEngine, "DeviceType", deviceData::getDeviceType)) + .replace("${PLATFORM_NAME}", value(ddEngine, "PlatformName", deviceData::getPlatformName)) + .replace("${PLATFORM_VERSION}", value(ddEngine, "PlatformVersion", deviceData::getPlatformVersion)) + .replace("${BROWSER_NAME}", value(ddEngine, "BrowserName", deviceData::getBrowserName)) + .replace("${BROWSER_VERSION}", value(ddEngine, "BrowserVersion", deviceData::getBrowserVersion)) + .replace("${SCREEN_WIDTH}", value(ddEngine, "ScreenPixelsWidth", deviceData::getScreenPixelsWidth)) + .replace("${SCREEN_HEIGHT}", value(ddEngine, "ScreenPixelsHeight", deviceData::getScreenPixelsHeight)) + // DeviceId is a match metric, which every data file gives. .replace("${DEVICE_ID}", asString(tryGet(deviceData::getDeviceId))) // IP Intelligence properties - .replace("${REGISTERED_NAME}", asStringProperty(tryGet(ipiData::getRegisteredName))) - .replace("${REGISTERED_OWNER}", asStringProperty(tryGet(ipiData::getRegisteredOwner))) - .replace("${REGISTERED_COUNTRY}", asStringProperty(tryGet(ipiData::getRegisteredCountry))) - .replace("${IP_RANGE_START}", asIPAddressProperty(tryGet(ipiData::getIpRangeStart))) - .replace("${IP_RANGE_END}", asIPAddressProperty(tryGet(ipiData::getIpRangeEnd))) - .replace("${COUNTRY}", asStringProperty(tryGet(ipiData::getCountry))) - .replace("${COUNTRY_CODE}", asStringProperty(tryGet(ipiData::getCountryCode))) - .replace("${COUNTRY_CODE3}", asStringProperty(tryGet(ipiData::getCountryCode3))) - .replace("${REGION}", asStringProperty(tryGet(ipiData::getRegion))) - .replace("${STATE}", asStringProperty(tryGet(ipiData::getState))) - .replace("${TOWN}", asStringProperty(tryGet(ipiData::getTown))) - .replace("${LATITUDE}", asFloatProperty(tryGet(ipiData::getLatitude))) - .replace("${LONGITUDE}", asFloatProperty(tryGet(ipiData::getLongitude))) - .replace("${AREAS}", asWktStringProperty(tryGet(ipiData::getAreas))) - .replace("${AREAS_JS}", escapeForJs(asWktStringProperty(tryGet(ipiData::getAreas)))) - .replace("${ACCURACY_RADIUS}", asIntegerProperty(tryGet(ipiData::getAccuracyRadiusMin))) - .replace("${TIME_ZONE_OFFSET}", asIntegerProperty(tryGet(ipiData::getTimeZoneOffset))) + .replace("${REGISTERED_NAME}", value(ipiEngine, "RegisteredName", ipiData::getRegisteredName)) + .replace("${REGISTERED_OWNER}", value(ipiEngine, "RegisteredOwner", ipiData::getRegisteredOwner)) + .replace("${REGISTERED_COUNTRY}", value(ipiEngine, "RegisteredCountry", ipiData::getRegisteredCountry)) + .replace("${IP_RANGE_START}", value(ipiEngine, "IpRangeStart", ipiData::getIpRangeStart)) + .replace("${IP_RANGE_END}", value(ipiEngine, "IpRangeEnd", ipiData::getIpRangeEnd)) + .replace("${COUNTRY}", value(ipiEngine, "Country", ipiData::getCountry)) + .replace("${COUNTRY_CODE}", value(ipiEngine, "CountryCode", ipiData::getCountryCode)) + .replace("${COUNTRY_CODE3}", value(ipiEngine, "CountryCode3", ipiData::getCountryCode3)) + .replace("${REGION}", value(ipiEngine, "Region", ipiData::getRegion)) + .replace("${STATE}", value(ipiEngine, "State", ipiData::getState)) + .replace("${TOWN}", value(ipiEngine, "Town", ipiData::getTown)) + .replace("${LATITUDE}", value(ipiEngine, "Latitude", ipiData::getLatitude)) + .replace("${LONGITUDE}", value(ipiEngine, "Longitude", ipiData::getLongitude)) + .replace("${AREAS}", value(ipiEngine, "Areas", ipiData::getAreas)) + // The map shows nothing when the data file has no areas. + .replace("${AREAS_JS}", DataFileHelper.hasProperty(ipiEngine, "Areas") ? + escapeForJs(asString(ipiData.getAreas())) : "") + .replace("${ACCURACY_RADIUS}", value(ipiEngine, "AccuracyRadiusMin", ipiData::getAccuracyRadiusMin)) + .replace("${TIME_ZONE_OFFSET}", value(ipiEngine, "TimeZoneOffset", ipiData::getTimeZoneOffset)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) - //.replace("${HumanProbability}",asStringProperty(tryGet(ipiData::getHumanProbability))) - .replace("${LocationConfidence}",asStringProperty(tryGet(ipiData::getLocationConfidence))) - .replace("${ConnectionType}",asStringProperty(tryGet(ipiData::getConnectionType))) - .replace("${HumanProbability}",asIntegerProperty(tryGet(ipiData::getHumanProbability))) - .replace("${HardwareDiversity}",asIntegerProperty(tryGet(ipiData::getHardwareDiversity))) - .replace("${BrowserDiversity}",asIntegerProperty(tryGet(ipiData::getBrowserDiversity))) - .replace("${PlatformDiversity}",asIntegerProperty(tryGet(ipiData::getPlatformDiversity))) - // On-premise engines: only invite the user to contact us about more - // properties and features when running against the free Lite tier. + .replace("${LocationConfidence}", value(ipiEngine, "LocationConfidence", ipiData::getLocationConfidence)) + .replace("${ConnectionType}", value(ipiEngine, "ConnectionType", ipiData::getConnectionType)) + .replace("${HumanProbability}", value(ipiEngine, "HumanProbability", ipiData::getHumanProbability)) + .replace("${HardwareDiversity}", value(ipiEngine, "HardwareDiversity", ipiData::getHardwareDiversity)) + .replace("${BrowserDiversity}", value(ipiEngine, "BrowserDiversity", ipiData::getBrowserDiversity)) + .replace("${PlatformDiversity}", value(ipiEngine, "PlatformDiversity", ipiData::getPlatformDiversity)) + // On-premise engines: only invite the user to contact us about the + // Enterprise data files when running on a free data file. .replace("${CONTACT_MESSAGE}", HtmlContentHelper.getContactMessage( - HtmlContentHelper.ContactMessageVariant.ON_PREMISE, isLiteDataFile(flowData))); + HtmlContentHelper.ContactMessageVariant.ON_PREMISE, + "Lite".equals(ddEngine.getDataSourceTier()) || + DataFileHelper.isFreeDataFile(ipiEngine))); } /** - * Determine whether the on-premise IP Intelligence engine is using the free Lite tier data file. + * The value of a property for the page, or that it needs the Enterprise + * data file when the data file in use does not have it. The value is + * only read when the data file has the property. */ - private boolean isLiteDataFile(FlowData flowData) { - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine engine = - flowData.getPipeline().getElement( - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine.class); - return engine != null && engine.getDataSourceTier().equals("Lite"); + private static String value(IPIntelligenceOnPremiseEngine engine, + String property, + Supplier> value) { + return DataFileHelper.hasProperty(engine, property) ? + asString(value.get()) : + DataFileHelper.NEEDS_ENTERPRISE; + } + + /** + * As {@link #value(IPIntelligenceOnPremiseEngine, String, Supplier)} for + * a Device Detection property. + */ + private static String value(DeviceDetectionHashEngine engine, + String property, + Supplier> value) { + return hasProperty(engine, property) ? + asString(value.get()) : + DataFileHelper.NEEDS_ENTERPRISE; + } + + /** + * True when the Device Detection engine's data file has the property. + * The Lite data file leaves out the properties only the Enterprise data + * file has, such as HardwareVendor, HardwareName and DeviceType. + */ + private static boolean hasProperty(DeviceDetectionHashEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; } private String buildEvidenceRows(FlowData flowData) { diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index ec48e2df9..0520012b8 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -33,8 +33,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing, and shows each property only the Enterprise data file has as needing it. To + * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -60,9 +61,12 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.web.services.FlowDataProviderCore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,16 +80,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_ENV_VAR; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIntegerProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asFloatProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIPAddressProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWktStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -102,8 +102,9 @@ * * The configuration file for the pipeline is at src/main/webapp/WEB-INF/51Degrees-OnPrem.xml * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=gettingstartedwebonprem). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing, and shows each property only the Enterprise data file has as needing it. To + * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=gettingstartedwebonprem). */ public class GettingStartedWebOnPrem extends HttpServlet { private static final long serialVersionUID = 1734154705981153540L; @@ -114,11 +115,11 @@ public static void main(String[] args) throws Exception { logger.info("Running Example {}", GettingStartedWebOnPrem.class); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; // Set data file location using DataFileHelper try { - String dataFilePath = fiftyone.ipintelligence.examples.shared.DataFileHelper.getDataFileLocation(dataFile); + String dataFilePath = DataFileHelper.getDataFileLocation(dataFile); System.setProperty("TestDataFile", dataFilePath); logger.info("Using data file: {}", dataFilePath); } catch (Exception e) { @@ -201,41 +202,50 @@ private String loadTemplate(String templatePath) throws IOException { * Replace template variables with IP Intelligence data */ private String substituteTemplateValues(String template, IPIntelligenceData ipiData, String inputIp, FlowData flowData) { + // The engine says which properties its data file has. + IPIntelligenceOnPremiseEngine ipiEngine = + flowData.getPipeline().getElement(IPIntelligenceOnPremiseEngine.class); return template .replace("${DATA_FILE_WARNING}", HtmlContentHelper.getDataFileAgeWarning(flowData)) .replace("${INPUT_IP_ADDRESS}", inputIp != null ? inputIp : "") - .replace("${REGISTERED_NAME}", asStringProperty(tryGet(ipiData::getRegisteredName))) - .replace("${REGISTERED_OWNER}", asStringProperty(tryGet(ipiData::getRegisteredOwner))) - .replace("${REGISTERED_COUNTRY}", asStringProperty(tryGet(ipiData::getRegisteredCountry))) - .replace("${IP_RANGE_START}", asIPAddressProperty(tryGet(ipiData::getIpRangeStart))) - .replace("${IP_RANGE_END}", asIPAddressProperty(tryGet(ipiData::getIpRangeEnd))) - .replace("${COUNTRY}", asStringProperty(tryGet(ipiData::getCountry))) - .replace("${COUNTRY_CODE}", asStringProperty(tryGet(ipiData::getCountryCode))) - .replace("${COUNTRY_CODE3}", asStringProperty(tryGet(ipiData::getCountryCode3))) - .replace("${REGION}", asStringProperty(tryGet(ipiData::getRegion))) - .replace("${STATE}", asStringProperty(tryGet(ipiData::getState))) - .replace("${TOWN}", asStringProperty(tryGet(ipiData::getTown))) - .replace("${LATITUDE}", asFloatProperty(tryGet(ipiData::getLatitude))) - .replace("${LONGITUDE}", asFloatProperty(tryGet(ipiData::getLongitude))) - .replace("${AREAS}", asWktStringProperty(tryGet(ipiData::getAreas))) - .replace("${AREAS_JS}", escapeForJs(asWktStringProperty(tryGet(ipiData::getAreas)))) - .replace("${ACCURACY_RADIUS}", asIntegerProperty(tryGet(ipiData::getAccuracyRadiusMin))) - .replace("${TIME_ZONE_OFFSET}", asIntegerProperty(tryGet(ipiData::getTimeZoneOffset))) + .replace("${REGISTERED_NAME}", value(ipiEngine, "RegisteredName", ipiData::getRegisteredName)) + .replace("${REGISTERED_OWNER}", value(ipiEngine, "RegisteredOwner", ipiData::getRegisteredOwner)) + .replace("${REGISTERED_COUNTRY}", value(ipiEngine, "RegisteredCountry", ipiData::getRegisteredCountry)) + .replace("${IP_RANGE_START}", value(ipiEngine, "IpRangeStart", ipiData::getIpRangeStart)) + .replace("${IP_RANGE_END}", value(ipiEngine, "IpRangeEnd", ipiData::getIpRangeEnd)) + .replace("${COUNTRY}", value(ipiEngine, "Country", ipiData::getCountry)) + .replace("${COUNTRY_CODE}", value(ipiEngine, "CountryCode", ipiData::getCountryCode)) + .replace("${COUNTRY_CODE3}", value(ipiEngine, "CountryCode3", ipiData::getCountryCode3)) + .replace("${REGION}", value(ipiEngine, "Region", ipiData::getRegion)) + .replace("${STATE}", value(ipiEngine, "State", ipiData::getState)) + .replace("${TOWN}", value(ipiEngine, "Town", ipiData::getTown)) + .replace("${LATITUDE}", value(ipiEngine, "Latitude", ipiData::getLatitude)) + .replace("${LONGITUDE}", value(ipiEngine, "Longitude", ipiData::getLongitude)) + .replace("${AREAS}", value(ipiEngine, "Areas", ipiData::getAreas)) + // The map shows nothing when the data file has no areas. + .replace("${AREAS_JS}", DataFileHelper.hasProperty(ipiEngine, "Areas") ? + escapeForJs(asString(ipiData.getAreas())) : "") + .replace("${ACCURACY_RADIUS}", value(ipiEngine, "AccuracyRadiusMin", ipiData::getAccuracyRadiusMin)) + .replace("${TIME_ZONE_OFFSET}", value(ipiEngine, "TimeZoneOffset", ipiData::getTimeZoneOffset)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) - // On-premise engine: only invite the user to contact us about more - // properties and features when running against the free Lite tier. + // On-premise engine: only invite the user to contact us about the + // Enterprise data file when running on a free data file. .replace("${CONTACT_MESSAGE}", HtmlContentHelper.getContactMessage( - HtmlContentHelper.ContactMessageVariant.ON_PREMISE, isLiteDataFile(flowData))); + HtmlContentHelper.ContactMessageVariant.ON_PREMISE, + DataFileHelper.isFreeDataFile(ipiEngine))); } /** - * Determine whether the on-premise engine is using the free Lite tier data file. + * The value of a property for the page, or that it needs the Enterprise + * data file when the data file in use does not have it. The value is + * only read when the data file has the property. */ - private boolean isLiteDataFile(FlowData flowData) { - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine engine = - flowData.getPipeline().getElement( - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine.class); - return engine != null && engine.getDataSourceTier().equals("Lite"); + private static String value(IPIntelligenceOnPremiseEngine engine, + String property, + Supplier> value) { + return DataFileHelper.hasProperty(engine, property) ? + asString(value.get()) : + DataFileHelper.NEEDS_ENTERPRISE; } /** diff --git a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java index 0d70be0f1..2c4266b3f 100644 --- a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java +++ b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java @@ -80,9 +80,8 @@ public static String getDataFileAgeWarning(FlowData flowData) { "A more recent data file may be needed for the most accurate IP " + "intelligence results. The latest Lite data file is downloaded by the " + "scripts in the data " + - "repository on GitHub. Find out about the full Enterprise data file, " + - "which includes automatic daily updates, on our " + - "pricing page.\n" + + "repository on GitHub. Contact us " + + "about the Enterprise data file, which includes automatic daily updates.\n" + ""; } @@ -90,8 +89,8 @@ public static String getDataFileAgeWarning(FlowData flowData) { * Build the contact-us message banner shown at the bottom of the web example * pages. Cloud examples are free by design, so the cloud variant is shown * unconditionally. On-premise examples only show the banner when running - * against the free Lite tier data file, so the {@code show} flag should be - * wired to {@code engine.getDataSourceTier().equals("Lite")}. + * against a free data file, so the {@code show} flag should be wired to + * {@code DataFileHelper.isFreeDataFile(engine)}. * * @param variant which message wording to use * @param show whether the banner should be rendered at all From 80233d4927d8c6679e8b3b6984bc3a462e040125 Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Sat, 7 Nov 2026 16:24:11 +0000 Subject: [PATCH 20/95] Run the copyright update once a year (#52) --- .github/workflows/monthly-copyright-update.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/monthly-copyright-update.yml b/.github/workflows/monthly-copyright-update.yml index b9b4f9d4b..fdc6d3383 100644 --- a/.github/workflows/monthly-copyright-update.yml +++ b/.github/workflows/monthly-copyright-update.yml @@ -7,8 +7,8 @@ on: type: boolean default: false schedule: - # * is a special character in YAML so quote this string - - cron: '0 0 1 * *' + # The copyright year only changes once a year, so this runs on 1 January. + - cron: '0 0 1 1 *' jobs: Monthly_Copyright_Update: From d4abce9ef71023b3bbe83e68bb9be5d774750aa0 Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Sat, 7 Nov 2026 16:25:32 +0000 Subject: [PATCH 21/95] Run the Selenium contract tests against the on-premise web example too (#54) --- ci/run-integration-tests.ps1 | 100 ++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ae705f41a..99621cbd9 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -73,67 +73,107 @@ if ($Version) { Write-Host "No version given, so the examples were tested by the unit tests against the reactor." } -# The shared Selenium contract tests, run against the Device Detection cloud -# web example. +# The shared Selenium contract tests, run against one web example. The example +# is built, started in the background on its own port and left to serve until +# the tests finish. Every web example, cloud and on-premise, is expected to give +# the same client-side results, so each is tested the same way. function Invoke-ContractTests { - $port = 8099 + param( + [Parameter(Mandatory)][string]$Label, + [Parameter(Mandatory)][string]$Module, + [Parameter(Mandatory)][int]$Port, + # Environment variables the example needs, removed again afterwards. + [hashtable]$ExampleEnv = @{} + ) + Write-Host "Running the Selenium contract tests against the $Label example on port $Port" $example = $null Push-Location "$RepoName/examples/device-detection" try { - mvn @MavenOptions -pl web/getting-started.cloud -am package -DskipTests @versionArgs - if ($LASTEXITCODE -ne 0) { throw "Building the cloud web example failed." } - $jar = (Get-ChildItem web/getting-started.cloud/target/*-jar-with-dependencies.jar | Select-Object -First 1).FullName - $env:PORT = $port - $env:TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" - $env:TestResourceKey = $TestResourceKey + mvn @MavenOptions -pl $Module -am package -DskipTests @versionArgs + if ($LASTEXITCODE -ne 0) { throw "Building the $Label web example failed." } + $jar = (Get-ChildItem "$Module/target/*-jar-with-dependencies.jar" | Select-Object -First 1).FullName + $env:PORT = $Port + foreach ($name in $ExampleEnv.Keys) { + Set-Item "env:$name" $ExampleEnv[$name] + } $example = java -jar $jar 2>&1 & } finally { Pop-Location } try { - # SELENIUM_TESTS_REF picks a branch of the suite, so a change there - # can be tried here before it is merged. - $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } - if (-not (Test-Path selenium-api-tests)) { - Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" - git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git - } - # Wait for the example to come up. - curl -sS -o $(if ($IsWindows) { 'NUL' } else { '/dev/null' }) --retry 20 --retry-delay 3 --retry-max-time 120 --retry-connrefused "http://localhost:$port" - if ($LASTEXITCODE -ne 0) { throw "The cloud web example did not start on port $port." } + curl -sS -o $(if ($IsWindows) { 'NUL' } else { '/dev/null' }) --retry 20 --retry-delay 3 --retry-max-time 120 --retry-connrefused "http://localhost:$Port" + if ($LASTEXITCODE -ne 0) { throw "The $Label web example did not start on port $Port." } - $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" - $env:PAID_RESOURCE_KEY = $TestResourceKey - $env:EXAMPLE_URL = "http://localhost:$port" + $env:EXAMPLE_URL = "http://localhost:$Port" $env:EXAMPLE_LANG = 'java' dotnet test selenium-api-tests -c Release --filter TestCategory=Contract # PowerShell does not stop for a native command that returns non-zero, # so without this the browser tests could fail every case and the job # would still be reported as a success. if ($LASTEXITCODE -ne 0) { - throw "The Selenium contract tests failed, returning $LASTEXITCODE. Their output is above." + throw "The Selenium contract tests failed against the $Label example, returning $LASTEXITCODE. Their output is above." } } catch { - if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } + if ($example) { Write-Host ">>> $Label example output >>>"; Receive-Job $example | Out-Host; Write-Host "<<< $Label example output <<<" } throw } finally { if ($example) { Remove-Job -Force $example } + Remove-Item env:PORT, env:EXAMPLE_URL -ErrorAction SilentlyContinue + foreach ($name in $ExampleEnv.Keys) { + Remove-Item "env:$name" -ErrorAction SilentlyContinue + } } } Write-Host "::group::Selenium contract tests" if (-not $TestResourceKey) { - Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the cloud example." + Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the web examples." } elseif ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Write-Host "::warning title=Selenium skipped::Selenium contract skipped on linux-arm64, which has no selenium-manager build. It runs on x64 and macOS arm64." } else { - try { - Invoke-ContractTests - } catch { - Write-Host "::error title=Selenium contract failed::$_" - $failed += 'Selenium contract tests' + # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of the + # suite, so a change there can be tried here before it is merged. + $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } + if (-not (Test-Path selenium-api-tests)) { + Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" + git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git + } + $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" + $env:PAID_RESOURCE_KEY = $TestResourceKey + + $contracts = @( + @{ + Label = 'cloud' + Module = 'web/getting-started.cloud' + Port = 8099 + ExampleEnv = @{ + TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" + TestResourceKey = $TestResourceKey + } + } + # The on-premise example, against the TAC data file that + # fetch-assets.ps1 links into the data repository. The Lite data file + # has neither DeviceType nor the JavaScript properties the contract + # tests need. + @{ + Label = 'on-premise' + Module = 'web/getting-started.onprem' + Port = 8098 + ExampleEnv = @{ + '51DEGREES_DD_PATH' = (Resolve-Path "$RepoName/cxx/data/device-detection/TAC-HashV41.hash").Path + } + } + ) + # Each example is tested whether or not the one before it failed. + foreach ($contract in $contracts) { + try { + Invoke-ContractTests @contract + } catch { + Write-Host "::error title=Selenium contract failed::$_" + $failed += "Selenium contract tests, $($contract.Label) example" + } } } Write-Host "::endgroup::" From 22995070ccd8992ec089d6b00ff57b375a2361de Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Sat, 7 Nov 2026 16:58:12 +0000 Subject: [PATCH 22/95] Run the IP Intelligence cloud tests again, and let a property have no value (#57) --- .../ipintelligence/cloud/ValueTests.java | 17 +++++++++++++++-- .../cloud/data/ValueCloudTests.java | 2 -- .../MissingPropertyHandlingTests.java | 4 +--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java index 3b7ee8bb1..80be99739 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java @@ -70,12 +70,25 @@ public static void valueTypes(WrapperCloud wrapper) throws Exception { assertNotNull("Value of " + property.getName() + " is null. ", value); assertTrue(AspectPropertyValue.class.isAssignableFrom(value.getClass())); + AspectPropertyValue propertyValue = + (AspectPropertyValue) value; + // A property the resource key is not entitled to, or one + // the client script populates, is listed but carries no + // value and says why instead. Asking such a value for its + // type throws, so what is checked here is that the reason + // was given, which is what a caller reads. + if (propertyValue.hasValue() == false) { + assertNotNull("Property '" + property.getName() + + "' has no value and no reason was given.", + propertyValue.getNoValueMessage()); + continue; + } assertTrue("Value of '" + property.getName() + - "' was of type " + ((AspectPropertyValue) value).getValue().getClass().getSimpleName() + + "' was of type " + propertyValue.getValue().getClass().getSimpleName() + " but should have been " + expectedType.getSimpleName() + ".", expectedType.isAssignableFrom( - ((AspectPropertyValue) value).getValue().getClass())); + propertyValue.getValue().getClass())); } } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java index 83cb2b82d..fc703b82a 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java @@ -27,10 +27,8 @@ import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -@Ignore public class ValueCloudTests extends TestsBase { @Before diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java index 2e860d039..e212f527f 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java @@ -33,7 +33,6 @@ import fiftyone.pipeline.engines.services.MissingPropertyReason; import fiftyone.pipeline.engines.services.MissingPropertyResult; import fiftyone.pipeline.engines.services.MissingPropertyService; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; @@ -46,7 +45,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@Ignore public class MissingPropertyHandlingTests { protected static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); @@ -136,7 +134,7 @@ public void PropertyNotInResource() throws Exception { private static String nullValueJson = "{\n" + - " 'ip-intelligence': {\n" + + " 'ip': {\n" + " 'registeredname': 'Example Network',\n" + " 'registeredcountry': 'US',\n" + " 'latitude': null,\n" + From 6353cb45a21777831e1532638c78383c1f375666 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 8 Nov 2026 23:44:49 +0000 Subject: [PATCH 23/95] Store a data file's future before its check can read it (#71) * Test a check that starts before registration stores its future registerDataFile schedules the first check and only then stores the future it gets back, so a check with a short delay can start in between. If the check reads the future in that gap it finds none, clears nothing, and its own re-registration then finds the stored future and schedules no next check. DataUpdateService_UpdateFromFile_FileUpdated failed this way once in a full build on macOS x64 with Java 11. The two new tests force that order on every run, one for a check that applies an update and one for a check that applies none, which is the path the retry after a 429 response takes. Registration is held after scheduling until the check has read the future or is blocked on a lock that registration holds, and the check's first read waits until registration has stored the future. Every wait is bounded, so a design that deadlocks fails rather than hangs. Each failure message lists the order in which the two threads read and wrote the future. For #70 * Store a data file's future before its check can read it Every read and write of a data file's future now happens while holding one lock in the data update service. registerDataFile holds it from scheduling the check to storing the future, and the check takes it before it cancels, clears or replaces the future, so a check that starts at once waits until registration has stored its future. Unregistering takes it for the same reason. The lock is the service's own rather than the data file's update lock, because the file system watcher holds the update lock while it refreshes the engine and while it waits for the new file to open. Taking that lock for the future would make unregistering wait for a refresh. Only scheduling, cancelling and storing happen while holding the new lock, so it is never held while waiting for the configuration lock or an update lock. For #70 * Word the new comments to match what the lock and the tests cover The lock's comment now says it covers the service's own reads and writes of the future, the test data file's comment says it records the check's reads, and the second test's comment says which checks share its path. No code changes. For #70 --- .../services/DataUpdateServiceDefault.java | 116 +++++---- .../services/DataUpdateServiceTests.java | 220 ++++++++++++++++++ 2 files changed, 287 insertions(+), 49 deletions(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 7430ea3cc..dc2aabe8b 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -69,6 +69,14 @@ public class DataUpdateServiceDefault implements DataUpdateService { private final boolean closeFutureFactory; private final List onUpdateCompleteList = new ArrayList<>(); private final Object configLock = new Object(); + /** + * Held for every read and write this service makes of a data file's + * future, so a check cannot read or clear its future before + * {@link #registerDataFile} has stored it. Only the future is scheduled, + * cancelled and stored while holding it, so it is never held while + * waiting for the configuration lock or a data file's update lock. + */ + private final Object futureLock = new Object(); /** * HTTP client used to download new data files. */ @@ -226,16 +234,18 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { // This will repeat until the update is acquired. if (manualUpdate == false) { logger.debug("Cancelling and resetting update timer"); - if (dataFile.getFuture() != null) { - dataFile.getFuture().cancel(true); - dataFile.setFuture(futureFactory.schedule( - new Runnable() { - @Override - public void run() { - checkForUpdate(dataFile, false); - } - }, - getInterval(dataFile.getConfiguration()))); + synchronized (futureLock) { + if (dataFile.getFuture() != null) { + dataFile.getFuture().cancel(true); + dataFile.setFuture(futureFactory.schedule( + new Runnable() { + @Override + public void run() { + checkForUpdate(dataFile, false); + } + }, + getInterval(dataFile.getConfiguration()))); + } } } } @@ -268,10 +278,12 @@ public void run() { private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { AspectEngineDataFileDefault aspectDataFile = (AspectEngineDataFileDefault) dataFile; - if (aspectDataFile.getFuture() != null) { - // Dispose of the old timer object - aspectDataFile.getFuture().cancel(true); - aspectDataFile.setFuture(null); + synchronized (futureLock) { + if (aspectDataFile.getFuture() != null) { + // Dispose of the old timer object + aspectDataFile.getFuture().cancel(true); + aspectDataFile.setFuture(null); + } } if (dataFile.getEngine() == null) { @@ -417,39 +429,43 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { // Only create an automatic update timer if auto updates are // enabled for this engine and there is not already an associated - // timer. - if (aspectDataFile.getAutomaticUpdatesEnabled() && - aspectDataFile.getFuture() == null) { - long delay = getInterval(aspectDataFile.getConfiguration()); - if (aspectDataFile.getUpdateAvailableTime() != null) { - if (aspectDataFile.getUpdateAvailableTime().getTime() - > System.currentTimeMillis()) { - delay = applyIntervalRandomisation( - aspectDataFile.getUpdateAvailableTime().getTime() - - System.currentTimeMillis(), - aspectDataFile.getConfiguration()); + // timer. The lock is held from scheduling to storing, because + // the check can start as soon as it is scheduled and must find + // its own future. + synchronized (futureLock) { + if (aspectDataFile.getAutomaticUpdatesEnabled() && + aspectDataFile.getFuture() == null) { + long delay = getInterval(aspectDataFile.getConfiguration()); + if (aspectDataFile.getUpdateAvailableTime() != null) { + if (aspectDataFile.getUpdateAvailableTime().getTime() + > System.currentTimeMillis()) { + delay = applyIntervalRandomisation( + aspectDataFile.getUpdateAvailableTime().getTime() + - System.currentTimeMillis(), + aspectDataFile.getConfiguration()); + } + } + // Create a timer that will go off when the engine expects + // updated data to be available. + logger.debug("Scheduling update check with delay {}", delay); + try { + ScheduledFuture future = futureFactory.schedule( + new Runnable() { + @Override + public void run() { + logger.debug("running checkForUpdate {}", + aspectDataFile.getDataFilePath()); + checkForUpdate(aspectDataFile, false); + logger.debug("Returning from checkForUpdate"); + } + }, + delay); + logger.debug("Update check scheduled for {} millis", + future.getDelay(TimeUnit.MILLISECONDS)); + aspectDataFile.setFuture(future); + } catch (Throwable t) { + logger.error("Exception when scheduling update check", t); } - } - // Create a timer that will go off when the engine expects - // updated data to be available. - logger.debug("Scheduling update check with delay {}", delay); - try { - ScheduledFuture future = futureFactory.schedule( - new Runnable() { - @Override - public void run() { - logger.debug("running checkForUpdate {}", - aspectDataFile.getDataFilePath()); - checkForUpdate(aspectDataFile, false); - logger.debug("Returning from checkForUpdate"); - } - }, - delay); - logger.debug("Update check scheduled for {} millis", - future.getDelay(TimeUnit.MILLISECONDS)); - aspectDataFile.setFuture(future); - } catch (Throwable t) { - logger.error("Exception when scheduling update check", t); } } @@ -539,9 +555,11 @@ public void unregisterDataFile(AspectEngineDataFile dataFile) { aspectDataFile.getConfiguration().getWatchKey().cancel(); aspectDataFile.getConfiguration().setWatchKey(null); } - if (aspectDataFile.getFuture() != null) { - aspectDataFile.getFuture().cancel(true); - aspectDataFile.setFuture(null); + synchronized (futureLock) { + if (aspectDataFile.getFuture() != null) { + aspectDataFile.getFuture().cancel(true); + aspectDataFile.setFuture(null); + } } } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 1e24bb485..d8768cf6c 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -51,6 +51,9 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; @@ -61,8 +64,10 @@ import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; @@ -492,6 +497,73 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { anyLong()); } + /** + * A check that finds an update and starts before registration has + * stored its future must still leave the next check scheduled and + * stored. + */ + @Test + public void DataUpdateService_UpdateFromFile_FileUpdated_CheckBeforeFutureStored() + throws InterruptedException { + // Arrange + RaceDataFile file = new RaceDataFile(); + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(false); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + configureScheduleRace(file); + Semaphore completeFlag = onUpdateCompleteFlag(); + + // Act + dataUpdate.registerDataFile(file); + boolean completed = completeFlag.tryAcquire( + RACE_WAIT_SECONDS * 3, TimeUnit.SECONDS); + + // Assert + assertTrue("The OnUpdateComplete event was never fired", completed); + verify(engine, times(1)).refreshData((String) any()); + assertNextCheckScheduled(file); + } + + /** + * A check that finds no update and starts before registration has + * stored its future must still leave the next check scheduled and + * stored. A check whose download fails takes the same path, the retry + * after a 429 response among them. + */ + @Test + public void DataUpdateService_UpdateFromFile_FileNotUpdated_CheckBeforeFutureStored() + throws InterruptedException { + // Arrange + RaceDataFile file = new RaceDataFile(); + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(false); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileNoUpdate(engine); + configureScheduleRace(file); + Semaphore completeFlag = onUpdateCompleteFlag(); + + // Act + dataUpdate.registerDataFile(file); + boolean completed = completeFlag.tryAcquire( + RACE_WAIT_SECONDS * 3, TimeUnit.SECONDS); + + // Assert + assertTrue("The OnUpdateComplete event was never fired", completed); + verify(engine, never()).refreshData(anyString()); + assertNextCheckScheduled(file); + } + @Test public void DataUpdateService_UpdateFromUrl_NoUpdate() throws InterruptedException, IOException { // Arrange @@ -1409,6 +1481,154 @@ private void configureTimerImmediateCallbackOnce() { configureTimerImmediateCallback(1, null); } + private static final long RACE_WAIT_SECONDS = 10; + + /** + * Data file that records every write of its future and every read the + * check makes. The check's first read waits until registration has + * stored the future, so a check that reads it too early goes on with a + * stale value. + */ + private static class RaceDataFile extends AspectEngineDataFileDefault { + private final Thread registering = Thread.currentThread(); + private final List events = new CopyOnWriteArrayList<>(); + private final AtomicInteger schedules = new AtomicInteger(); + private final AtomicReference> first = + new AtomicReference<>(); + private final AtomicReference> next = + new AtomicReference<>(); + private final AtomicReference check = new AtomicReference<>(); + private final CountDownLatch checkRead = new CountDownLatch(1); + private final CountDownLatch stored = new CountDownLatch(1); + private volatile boolean registrationReleased; + private volatile boolean storedInTime = true; + + private String who() { + return Thread.currentThread() == registering ? + "registration" : "check"; + } + + private String name(ScheduledFuture future) { + if (future == null) { + return "no future"; + } + if (future == first.get()) { + return "the first future"; + } + return future == next.get() ? "the next future" : "another future"; + } + + @Override + public ScheduledFuture getFuture() { + ScheduledFuture future = super.getFuture(); + if (Thread.currentThread() != registering) { + events.add("check read " + name(future)); + if (checkRead.getCount() > 0) { + checkRead.countDown(); + try { + storedInTime = stored.await( + RACE_WAIT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + storedInTime = false; + Thread.currentThread().interrupt(); + } + } + } + return future; + } + + @Override + public void setFuture(ScheduledFuture future) { + super.setFuture(future); + events.add(future == null ? + who() + " cleared the future" : + who() + " stored " + name(future)); + if (Thread.currentThread() == registering) { + stored.countDown(); + } + } + } + + /** + * Run the first check at once, and hold registration after scheduling + * it until the check has read the future or waits for a lock that + * registration holds. Later checks are never run. + */ + private void configureScheduleRace(final RaceDataFile file) { + final ThreadMXBean threads = ManagementFactory.getThreadMXBean(); + doAnswer(invocation -> { + final Runnable check = invocation.getArgument(0); + if (file.schedules.incrementAndGet() > 1) { + ScheduledFuture next = mock(ScheduledFuture.class); + file.next.set(next); + file.events.add(file.who() + " scheduled the next check"); + return next; + } + file.events.add(file.who() + " scheduled the first check"); + ScheduledFuture first = executorService.schedule(() -> { + file.check.set(Thread.currentThread()); + check.run(); + }, 0, TimeUnit.MILLISECONDS); + file.first.set(first); + file.registrationReleased = waitForCheck(file, threads); + return first; + }).when(futureFactory).schedule(any(Runnable.class), anyLong()); + } + + /** + * Wait until the check has read the future, or is blocked on a lock + * that the registering thread holds and so cannot read it until + * registration has finished. Thread.getId() is deprecated from Java 19 + * but is the id that ThreadMXBean reports on every version. + */ + @SuppressWarnings("deprecation") + private static boolean waitForCheck( + RaceDataFile file, + ThreadMXBean threads) throws InterruptedException { + long deadline = System.nanoTime() + + TimeUnit.SECONDS.toNanos(RACE_WAIT_SECONDS); + while (System.nanoTime() < deadline) { + if (file.checkRead.await(10, TimeUnit.MILLISECONDS)) { + return true; + } + Thread check = file.check.get(); + if (check != null) { + ThreadInfo info = threads.getThreadInfo(check.getId()); + if (info != null && + info.getLockOwnerId() == file.registering.getId()) { + return true; + } + } + } + return false; + } + + /** + * Assert that one check was scheduled at registration and one after it, + * and that the future stored is the next check's. Each message carries + * the order in which the two threads read and wrote the future. + */ + private static void assertNextCheckScheduled(RaceDataFile file) { + String events = " Events: " + file.events; + assertTrue("The check neither read the future nor waited for " + + "registration." + events, + file.registrationReleased); + assertTrue("Registration did not store the future while the " + + "check waited." + events, + file.storedInTime); + assertEquals("Schedules." + events, 2, file.schedules.get()); + assertSame("The future stored is not the next check's." + events, + file.next.get(), + file.getFuture()); + } + + private Semaphore onUpdateCompleteFlag() throws InterruptedException { + final Semaphore completeFlag = new Semaphore(1); + completeFlag.acquire(); + dataUpdate.onUpdateComplete((sender, args) -> completeFlag.release()); + return completeFlag; + } + private void configureFileNoUpdate(OnPremiseAspectEngine engine) { String dataFile = "C:/test/tempFile.dat"; String tempFile = "C:/test/dataFile.dat"; From c781d83ce82082829445846dd9842c771a0d7ca5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 8 Nov 2026 23:44:56 +0000 Subject: [PATCH 24/95] Fail the example cloud tests when there is no resource key (#66) The cloud tests under examples/ skipped when no resource key was set, so a run with no key reported success having called nothing. Each now asks its KeyHelper for the key, which reads _51DEGREES_RESOURCE_KEY, then _51DEGREES_RESOURCE_KEY_BESPOKE, then the names the example reads today, each as an environment variable and then as a system property, and fails the test naming every name tried, the product the key has to carry, how to see what a key carries and where to create one. KeyHelper.getOrSetResourceKey passed a null key to System.setProperty, which throws a NullPointerException, whenever no key was found and it was asked not to throw. It now keeps only a valid key. The Device Detection web cloud test sends a known User-Agent and checks that the page shows the browser and platform the cloud engine detected, rather than accepting any 200 response. --- .../console/GettingStartedCloudTest.java | 10 +- .../examples/console/MetadataCloudTest.java | 8 +- .../examples/console/MinimalExampleTest.java | 8 +- .../console/NativeModelCloudTest.java | 11 +- .../examples/console/TacCloudTest.java | 9 +- .../examples/shared/KeyHelper.java | 124 +++++++++++++++- .../examples/shared/KeyHelperTest.java | 136 ++++++++++++++++++ .../web/GettingStartedWebCloudTest.java | 83 +++++++++-- .../console/GetAllPropertiesCloudTest.java | 7 +- .../console/GettingStartedCloudMixedTest.java | 9 +- .../console/GettingStartedCloudTest.java | 7 +- .../examples/console/MetadataCloudTest.java | 7 +- .../examples/shared/KeyHelper.java | 92 +++++++++++- .../examples/shared/KeyHelperTest.java | 129 +++++++++++++++++ .../web/GettingStartedWebCloudMixedTest.java | 9 +- .../web/GettingStartedWebCloudTest.java | 8 +- 16 files changed, 579 insertions(+), 78 deletions(-) create mode 100644 examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java create mode 100644 examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java index ee38b77f4..c8a85dd39 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java @@ -24,18 +24,14 @@ import fiftyone.devicedetection.examples.shared.EvidenceHelper; import fiftyone.devicedetection.examples.shared.KeyHelper; - -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GettingStartedCloudTest { @Test public void gettingStartedCloudTest() throws Exception { - String resourceKey = KeyHelper.getOrSetTestResourceKey(false); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "whether a device is mobile, and its platform and browser", + "device"); GettingStartedCloud.run(resourceKey, EvidenceHelper.setUpEvidence(), System.out); diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java index cebb3ffed..c818aae69 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java @@ -23,17 +23,13 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class MetadataCloudTest { @Test public void gettingMetaDataCloudTest() throws Exception { - String resourceKey = KeyHelper.getOrSetTestResourceKey(false); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the device detection properties it offers", "device"); MetadataCloud.run(resourceKey, System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java index f1622ec37..12bd91179 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java @@ -23,17 +23,13 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class MinimalExampleTest { @Test public void minimalExampleTest() throws Exception { - String resourceKey = KeyHelper.getOrSetTestResourceKey(false); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "whether a device is mobile", "device"); MinimalExample.run(resourceKey); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java index 85facd407..9b59297f3 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java @@ -23,18 +23,17 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; +import java.util.Collections; public class NativeModelCloudTest { @Test public void run() throws Exception { - String resourceKey = - KeyUtils.getNamedKey(NativeModelCloud.NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireResourceKey( + Collections.singletonList( + NativeModelCloud.NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME), + "the devices that match a native model name", "hardware"); NativeModelCloud.run(resourceKey, System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java index a07d2ea14..7d3a356ff 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java @@ -23,18 +23,17 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; +import java.util.Collections; public class TacCloudTest { @Test public void run() throws Exception { - String resourceKey = KeyUtils.getNamedKey(TacCloud.TAC_EXAMPLE_RESOURCE_KEY_NAME); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireResourceKey( + Collections.singletonList(TacCloud.TAC_EXAMPLE_RESOURCE_KEY_NAME), + "the devices that match a TAC", "hardware"); TacCloud.run(resourceKey, System.out); } } \ No newline at end of file diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java index b6fc1a25f..467a443be 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java @@ -27,12 +27,127 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.function.Consumer; +import java.util.function.Function; public class KeyHelper { public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); + /** + * The names every cloud test reads a resource key from first, before + * the names its own example reads. + */ + public static final List SHARED_RESOURCE_KEY_NAMES = + Collections.unmodifiableList(Arrays.asList( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE")); + + static final String CREATE_KEY_URL = + "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=no-resource-key"; + + /** + * The resource key for a test that calls the live cloud service, from + * {@link #SHARED_RESOURCE_KEY_NAMES} and then {@link #TEST_RESOURCE_KEY}. + * Fails the test when there is none, because a cloud test that passes + * without a key has called nothing. + * @param asksFor what the test asks the service for + * @param products the products the key has to carry + * @return the resource key + */ + public static String requireTestResourceKey(String asksFor, + String... products) { + return requireResourceKey(Collections.singletonList(TEST_RESOURCE_KEY), + asksFor, products); + } + + /** + * The resource key for a test that calls the live cloud service, from + * {@link #SHARED_RESOURCE_KEY_NAMES} and then the given names. + * @param exampleNames the names the example reads, in the order it + * reads them + * @param asksFor what the test asks the service for + * @param products the products the key has to carry + * @return the resource key + */ + public static String requireResourceKey(List exampleNames, + String asksFor, + String... products) { + return requireResourceKey(KeyHelper::readKey, exampleNames, asksFor, + products); + } + + static String requireResourceKey(Function read, + List exampleNames, + String asksFor, + String... products) { + List names = new ArrayList<>(SHARED_RESOURCE_KEY_NAMES); + names.addAll(exampleNames); + for (String name : names) { + String value = read.apply(name); + if (KeyUtils.isInvalidKey(value) == false) { + return value.trim(); + } + } + throw new AssertionError(noResourceKey(names, asksFor, products)); + } + + static String noResourceKey(List names, String asksFor, + String... products) { + return "This test calls the live 51Degrees cloud service, so it " + + "needs a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", names) + " is set, as an " + + "environment variable or as a system property, tried in that " + + "order. The key has to carry the " + productList(products) + + ", because the test asks the service for " + asksFor + ". To " + + "see the products a key carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + CREATE_KEY_URL + "."; + } + + /** + * The message for a value the cloud answered without. + * @param thing the property or element that has no value + * @param reason the reason the cloud gave, or null + * @param products the products a key needs for the value + * @return the message + */ + public static String answeredWithout(String thing, String reason, + String... products) { + return "The cloud answered without " + thing + " (" + + (reason == null || reason.trim().isEmpty() ? + "no reason given" : reason.trim()) + + "), which is what a key that does not carry the " + + productList(products) + " gets."; + } + + private static String productList(String... products) { + StringBuilder list = new StringBuilder(); + for (int i = 0; i < products.length; i++) { + if (i > 0) { + list.append(i == products.length - 1 ? " and " : ", "); + } + list.append(products[i]); + } + return list + (products.length == 1 ? " product" : " products"); + } + + /** + * A key read as an environment variable, as its upper case name, and + * then as a system property, skipping any that is not a key, such as + * the empty string the example poms pass for a key that is not set. + */ + private static String readKey(String name) { + for (String value : new String[]{ + System.getenv(name), + System.getenv(name.toUpperCase()), + System.getProperty(name)}) { + if (KeyUtils.isInvalidKey(value) == false) { + return value; + } + } + return null; + } + /** * Obtain a resource key either from environment variable or from a property. */ @@ -79,8 +194,11 @@ public static String getOrSetResourceKey(String value, String variableName, } } - // capture the passed parameter for next time called - System.setProperty(variableName, value); + // Keep a valid key for next time. System.setProperty throws on null, + // and an invalid key is not worth keeping. + if (KeyUtils.isInvalidKey(value) == false) { + System.setProperty(variableName, value); + } return value; } public static String getOrSetSuperResourceKey(String value, String variablename) { diff --git a/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java new file mode 100644 index 000000000..1a75b45a0 --- /dev/null +++ b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java @@ -0,0 +1,136 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.shared; + +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +public class KeyHelperTest { + /** + * Values of the form KeyUtils accepts as a key. Neither is a real key. + */ + private static final String KEY_A = "AQAAAAAAAAAAAAAAAAAAAAAA"; + private static final String KEY_B = "AgAAAAAAAAAAAAAAAAAAAAAA"; + + /** + * With no key anywhere, the lookup logs that there is none and stores + * nothing, rather than passing null to System.setProperty. + */ + @Test + public void getOrSetTestResourceKey_NoKey_StoresNothing() { + String name = KeyHelper.TEST_RESOURCE_KEY; + // A test can clear a system property but not an environment + // variable. + assumeTrue(name + " is set in the environment", + System.getenv(name) == null && + System.getenv(name.toUpperCase()) == null); + String saved = System.clearProperty(name); + try { + assertNull(KeyHelper.getOrSetTestResourceKey(false)); + assertNull(System.getProperty(name)); + } finally { + if (saved != null) { + System.setProperty(name, saved); + } + } + } + + @Test + public void requireResourceKey_NoKey_FailsNamingEveryName() { + AssertionError error = assertThrows(AssertionError.class, () -> + KeyHelper.requireResourceKey(name -> null, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); + assertEquals("This test calls the live 51Degrees cloud service, so " + + "it needs a resource key and there is nothing to fall back " + + "on. None of _51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, TestResourceKey is set, as " + + "an environment variable or as a system property, tried in " + + "that order. The key has to carry the device product, " + + "because the test asks the service for whether a device is " + + "mobile. To see the products a key carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + KeyHelper.CREATE_KEY_URL + ".", + error.getMessage()); + } + + @Test + public void requireResourceKey_TwoProducts_NamesBoth() { + AssertionError error = assertThrows(AssertionError.class, () -> + KeyHelper.requireResourceKey(name -> null, + Collections.singletonList("SuperResourceKey"), + "the devices that match a TAC", "hardware", "device")); + assertTrue(error.getMessage(), error.getMessage().contains( + "None of _51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, SuperResourceKey is set")); + assertTrue(error.getMessage(), error.getMessage().contains( + "carry the hardware and device products, because")); + } + + /** + * A name holding something that is not a key, such as the empty + * string the example poms pass for a key that is not set, is passed + * over. + */ + @Test + public void requireResourceKey_FirstValidNameWins() { + Map values = new HashMap<>(); + values.put("_51DEGREES_RESOURCE_KEY", ""); + values.put("_51DEGREES_RESOURCE_KEY_BESPOKE", KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); + } + + @Test + public void requireResourceKey_ExampleNameAfterSharedNames() { + Map values = new HashMap<>(); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); + } + + @Test + public void answeredWithout_NoReason_SaysSo() { + assertEquals("The cloud answered without BrowserName (no reason " + + "given), which is what a key that does not carry the device " + + "product gets.", + KeyHelper.answeredWithout("BrowserName", null, "device")); + assertEquals("The cloud answered without BrowserName (BrowserName " + + "is a paid feature), which is what a key that does not carry " + + "the device product gets.", + KeyHelper.answeredWithout("BrowserName", + "BrowserName is a paid feature", "device")); + } +} diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java index e2890ac00..11fa1f57a 100644 --- a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java @@ -24,7 +24,6 @@ import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.examples.web.EmbedJetty; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -33,42 +32,100 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static fiftyone.devicedetection.examples.web.GettingStartedWebCloud.resourceBase; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; -import static org.junit.Assume.assumeFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +/** + * Requests the example's page with a known User-Agent and checks that the + * page shows what the cloud detected for it. + */ public class GettingStartedWebCloudTest { + /** + * Chrome 98 on Windows 10. + */ + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + /** + * What the page shows in place of a value the cloud did not return, + * followed by the reason. + */ + private static final String UNKNOWN = "Unknown."; + private static Server SERVER = null; @BeforeClass public static void startJetty() throws Exception { + String resourceKey = KeyHelper.requireTestResourceKey( + "the browser and platform of a known User-Agent", "device"); + // The example's configuration file reads the key from here. + System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); } @Test public void testWebCloud() throws Exception { - String resourceKey = KeyUtils.getNamedKey("TestResourceKey"); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); - HttpURLConnection connection = (HttpURLConnection) new URL("http://localhost:8081/").openConnection(); + connection.setRequestProperty("User-Agent", USER_AGENT); - InputStream response = connection.getInputStream(); - try (Scanner scanner = new Scanner(response)) { - String responseBody = scanner.useDelimiter("\\A").next(); - System.out.println(responseBody); - } int code = connection.getResponseCode(); + String body = read(code < 400 ? + connection.getInputStream() : connection.getErrorStream()); connection.disconnect(); - assertEquals(200, code); + + assertEquals("The page answered " + code + " with " + body, + 200, code); + assertTrue("The page does not say the cloud engine detected the " + + "values it shows. " + body, + body.contains("cloud device detection engine")); + assertDetected(body, "Browser Name", "BrowserName", "Chrome"); + assertDetected(body, "Platform Name", "PlatformName", "Windows"); + } + + /** + * Checks the value in the page's row for a property, which is the + * reason when the cloud returned no value. + */ + private static void assertDetected(String body, String label, + String property, String expected) { + Matcher row = Pattern.compile(Pattern.quote(label + + "") + "(.*?)") + .matcher(body); + assertTrue("The page has no " + label + " row. " + body, row.find()); + String value = row.group(1).trim(); + if (value.startsWith(UNKNOWN)) { + String reason = value.substring(UNKNOWN.length()).trim(); + fail(KeyHelper.answeredWithout(property, + "null".equals(reason) ? null : reason, "device")); + } + assertEquals(label + " on the page", expected, value); + } + + private static String read(InputStream stream) { + if (stream == null) { + return ""; + } + try (Scanner scanner = new Scanner(stream, + StandardCharsets.UTF_8.name())) { + scanner.useDelimiter("\\A"); + return scanner.hasNext() ? scanner.next() : ""; + } } @AfterClass public static void stopJetty() throws Exception { - SERVER.stop(); + if (SERVER != null) { + SERVER.stop(); + } } } \ No newline at end of file diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java index be4be97b1..e024e7506 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java @@ -26,15 +26,12 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GetAllPropertiesCloudTest { @Test public void getAllPropertiesCloudTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "every IP Intelligence property the key carries", "ip"); GetAllPropertiesCloud.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java index fc0aab3ac..9e43a0078 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java @@ -26,15 +26,14 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GettingStartedCloudMixedTest { @Test public void gettingStartedCloudMixedTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network and location of an IP address, and whether a " + + "device is mobile", + "ip", "device"); GettingStartedCloudMixed.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), GettingStartedMixed.setUpCombinedEvidence(), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java index 37dddd104..5030e2f2f 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java @@ -27,15 +27,12 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GettingStartedCloudTest { @Test public void gettingStartedCloudTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network, location and time zone of an IP address", "ip"); GettingStartedCloud.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), EvidenceHelper.setUpEvidence(), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java index c1ac46032..a531deb55 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java @@ -26,15 +26,12 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class MetadataCloudTest { @Test public void metadataCloudTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the IP Intelligence properties it offers", "ip"); MetadataCloud.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), System.out); diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java index 653ecc7b9..846c714f6 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.function.Function; public class KeyHelper { /** @@ -38,6 +39,90 @@ public class KeyHelper { public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); + /** + * The names every cloud test reads a resource key from first, before + * the names its own example reads. + */ + public static final List SHARED_RESOURCE_KEY_NAMES = + Collections.unmodifiableList(Arrays.asList( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE")); + + static final String CREATE_KEY_URL = + "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=no-resource-key"; + + /** + * The resource key for a test that calls the live cloud service, from + * {@link #SHARED_RESOURCE_KEY_NAMES} and then the names + * {@link #getOrSetTestResourceKey()} reads. Fails the test when there + * is none, because a cloud test that passes without a key has called + * nothing. + * @param asksFor what the test asks the service for + * @param products the products the key has to carry + * @return the resource key + */ + public static String requireTestResourceKey(String asksFor, + String... products) { + return requireResourceKey(KeyHelper::readKey, Arrays.asList( + RESOURCE_KEY_ENV_VAR, TEST_RESOURCE_KEY), asksFor, products); + } + + static String requireResourceKey(Function read, + List exampleNames, + String asksFor, + String... products) { + List names = new ArrayList<>(SHARED_RESOURCE_KEY_NAMES); + names.addAll(exampleNames); + for (String name : names) { + String value = read.apply(name); + if (KeyUtils.isInvalidKey(value) == false) { + return value.trim(); + } + } + throw new AssertionError(noResourceKey(names, asksFor, products)); + } + + static String noResourceKey(List names, String asksFor, + String... products) { + return "This test calls the live 51Degrees cloud service, so it " + + "needs a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", names) + " is set, as an " + + "environment variable or as a system property, tried in that " + + "order. The key has to carry the " + productList(products) + + ", because the test asks the service for " + asksFor + ". To " + + "see the products a key carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + CREATE_KEY_URL + "."; + } + + private static String productList(String... products) { + StringBuilder list = new StringBuilder(); + for (int i = 0; i < products.length; i++) { + if (i > 0) { + list.append(i == products.length - 1 ? " and " : ", "); + } + list.append(products[i]); + } + return list + (products.length == 1 ? " product" : " products"); + } + + /** + * A key read as an environment variable, as its upper case name, and + * then as a system property, skipping any that is not a key, such as + * the empty string the example poms pass for a key that is not set. + */ + private static String readKey(String name) { + for (String value : new String[]{ + System.getenv(name), + System.getenv(name.toUpperCase()), + System.getProperty(name)}) { + if (KeyUtils.isInvalidKey(value) == false) { + return value; + } + } + return null; + } + /** * Obtain a resource key either from environment variable or from a property. */ @@ -92,8 +177,11 @@ public static String getOrSetResourceKey(String value, String variableName, } } - // capture the passed parameter for next time called - System.setProperty(variableName, value); + // Keep a valid key for next time. System.setProperty throws on null, + // and an invalid key is not worth keeping. + if (KeyUtils.isInvalidKey(value) == false) { + System.setProperty(variableName, value); + } return value; } public static String getOrSetSuperResourceKey(String value, String variablename) { diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java new file mode 100644 index 000000000..67b2660cb --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java @@ -0,0 +1,129 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assume.assumeTrue; + +public class KeyHelperTest { + /** + * Values of the form KeyUtils accepts as a key. Neither is a real key. + */ + private static final String KEY_A = "AQAAAAAAAAAAAAAAAAAAAAAA"; + private static final String KEY_B = "AgAAAAAAAAAAAAAAAAAAAAAA"; + + /** + * The names getOrSetTestResourceKey reads, in its order. + */ + private static final List EXAMPLE_NAMES = Arrays.asList( + KeyHelper.RESOURCE_KEY_ENV_VAR, KeyHelper.TEST_RESOURCE_KEY); + + /** + * With no key anywhere, the lookup logs that there is none and stores + * nothing, rather than passing null to System.setProperty. + */ + @Test + public void getOrSetTestResourceKey_NoKey_StoresNothing() { + // A test can clear a system property but not an environment + // variable. + for (String name : EXAMPLE_NAMES) { + assumeTrue(name + " is set in the environment", + System.getenv(name) == null && + System.getenv(name.toUpperCase()) == null); + } + Map saved = new HashMap<>(); + for (String name : EXAMPLE_NAMES) { + saved.put(name, System.clearProperty(name)); + } + try { + assertNull(KeyHelper.getOrSetTestResourceKey(false)); + for (String name : EXAMPLE_NAMES) { + assertNull(name, System.getProperty(name)); + } + } finally { + for (Map.Entry entry : saved.entrySet()) { + if (entry.getValue() != null) { + System.setProperty(entry.getKey(), entry.getValue()); + } + } + } + } + + @Test + public void requireResourceKey_NoKey_FailsNamingEveryName() { + AssertionError error = assertThrows(AssertionError.class, () -> + KeyHelper.requireResourceKey(name -> null, EXAMPLE_NAMES, + "the network and location of an IP address, and whether a " + + "device is mobile", + "ip", "device")); + assertEquals("This test calls the live 51Degrees cloud service, so " + + "it needs a resource key and there is nothing to fall back " + + "on. None of _51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, 51DEGREES_RESOURCE_KEY, " + + "TestResourceKey is set, as an environment variable or as a " + + "system property, tried in that order. The key has to carry " + + "the ip and device products, because the test asks the " + + "service for the network and location of an IP address, and " + + "whether a device is mobile. To see the products a key " + + "carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + KeyHelper.CREATE_KEY_URL + ".", + error.getMessage()); + } + + /** + * A name holding something that is not a key, such as the empty + * string the example poms pass for a key that is not set, is passed + * over. + */ + @Test + public void requireResourceKey_FirstValidNameWins() { + Map values = new HashMap<>(); + values.put("_51DEGREES_RESOURCE_KEY", ""); + values.put("_51DEGREES_RESOURCE_KEY_BESPOKE", KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + EXAMPLE_NAMES, "every IP Intelligence property", "ip")); + } + + @Test + public void requireResourceKey_ExampleNamesInTheirOrder() { + Map values = new HashMap<>(); + values.put(KeyHelper.RESOURCE_KEY_ENV_VAR, KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + EXAMPLE_NAMES, "every IP Intelligence property", "ip")); + values.remove(KeyHelper.RESOURCE_KEY_ENV_VAR); + assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, + EXAMPLE_NAMES, "every IP Intelligence property", "ip")); + } +} diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java index ca1b1ecfc..ff49ca9e7 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.examples.web; import fiftyone.ipintelligence.examples.shared.KeyHelper; -import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,7 +35,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; public class GettingStartedWebCloudMixedTest { private static Server SERVER; @@ -44,9 +42,10 @@ public class GettingStartedWebCloudMixedTest { @BeforeClass public static void startJetty() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network and location of the address a request comes from, " + + "and the device that sent it", + "ip", "device"); // Make the resource key available to the pipeline configuration file. // The resource key must include both device detection and IP // Intelligence properties as this example uses both engines. diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java index ea87c40a7..06a7b8b48 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.examples.web; import fiftyone.ipintelligence.examples.shared.KeyHelper; -import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -38,7 +37,6 @@ import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; public class GettingStartedWebCloudTest { private static Server SERVER; @@ -46,9 +44,9 @@ public class GettingStartedWebCloudTest { @BeforeClass public static void startJetty() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network and location of the address a request comes from", + "ip"); // Make the resource key available to the pipeline configuration file System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); From 3bedce52f90f2336f54373b6ffdb2a72d2268a40 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 06:15:59 +0000 Subject: [PATCH 25/95] Default IP Intelligence to the Balanced performance profile (#69) * Default IP Intelligence to the Balanced performance profile The IP Intelligence on-premise specification gives Balanced as the default performance profile. The on-premise pipeline builder defaulted to MaxPerformance, and the engine builder applied no profile unless the caller set one, so it ran with whatever configuration the native ConfigIpi constructor starts from, which is the in-memory one in the C library this repository pins. Its setPerformanceProfile documentation and @DefaultValue already said Balanced. The engine builder now applies Balanced to its native configuration in a field initializer, before the constructor sets the concurrency, so a builder whose caller sets no profile is Balanced whatever the native library starts from. The pipeline builder holds no profile until the caller sets one and passes it on only then, because applying Balanced again would replace the concurrency the engine builder set. BuilderTests and the new OnPremisePerformanceProfileTests compare the collection settings each builder gives the engine with the Balanced and in-memory settings the native library defines. * Say which profile the metadata example uses, and the Java way to get the engine The comment above the engine in MetadataOnPrem said the max performance profile while the code sets LowMemory, and gave the C# call for getting the engine from a pipeline. It now names the low memory profile and gives pipeline.getElement(IPIntelligenceOnPremiseEngine.class). * Give up ownership of the collection settings the profile tests read The generated ConfigIpiSwig wrappers mark getComponents, getProperties, getValues and getProfiles as owning the CollectionConfigSwig they return, although each getter returns a reference to a member of the native configuration. When the garbage collector finalized one of those proxies, the native library freed memory inside the configuration, which aborted the test JVM. The tests now give up ownership of every collection they read, so finalizing those proxies frees nothing. The tests that name MaxPerformance and LowMemory are renamed to say that they prove the named profile replaced Balanced, because the two profiles give every collection the same settings. --- .../examples/console/MetadataOnPrem.java | 9 +- .../IPIntelligenceOnPremiseEngineBuilder.java | 22 +- .../onpremise/flowelements/BuilderTests.java | 121 ++++++++++ ...PIntelligenceOnPremisePipelineBuilder.java | 19 +- .../OnPremisePerformanceProfileTests.java | 207 ++++++++++++++++++ 5 files changed, 366 insertions(+), 12 deletions(-) create mode 100644 ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index b7d2d7206..be168e638 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -94,11 +94,10 @@ public static void run(String dataFile, OutputStream output) throws Exception { dataFile, DataFileHelper.IPI_PATH_ENV_VAR); throw e; } - // Build a new on-premise IP Intelligence engine with the max performance profile. - // Note that there is no need to construct a complete pipeline in order to access - // the meta-data. - // If you already have a pipeline and just want to get a reference to the engine - // then you can use `var engine = pipeline.GetElement();` + // Build an on-premise IP Intelligence engine with the low memory + // profile. There is no need to build a complete pipeline to read the + // metadata. Code that already has a pipeline can get its engine with + // pipeline.getElement(IPIntelligenceOnPremiseEngine.class). try (IPIntelligenceOnPremiseEngine engine = new IPIntelligenceOnPremiseEngineBuilder(LoggerFactory.getILoggerFactory()) // We use the low memory profile to minimize memory usage for this diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java index c48e88184..c403aee93 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java @@ -62,9 +62,27 @@ public class IPIntelligenceOnPremiseEngineBuilder private final String dataDownloadType = "IPIV41"; /** - * Native configuration instance for this engine. + * Native configuration instance for this engine, which starts from the + * Balanced performance profile. */ - private final ConfigIpiSwig config = new ConfigIpiSwig(); + private final ConfigIpiSwig config = balancedConfig(); + + /** + * A native configuration with the Balanced profile applied. The profile + * is applied here rather than left to the native library, so that a + * builder whose caller sets no profile is Balanced whatever + * configuration the native library starts from. A field initializer + * runs before the constructor sets the concurrency, which matters + * because applying a profile replaces the concurrency of every + * collection. + * + * @return a new Balanced configuration + */ + private static ConfigIpiSwig balancedConfig() { + ConfigIpiSwig config = new ConfigIpiSwig(); + config.setBalanced(); + return config; + } /** * Default constructor which uses the {@link ILoggerFactory} implementation diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java index 4d825688c..30b9e0025 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java @@ -22,12 +22,21 @@ package fiftyone.ipintelligence.engine.onpremise.flowelements; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.CollectionConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ConfigIpiSwig; +import fiftyone.pipeline.engines.Constants.PerformanceProfiles; import fiftyone.pipeline.engines.configuration.CacheConfiguration; import org.junit.Before; import org.junit.Test; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; +import java.lang.reflect.Field; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; + public class BuilderTests { protected static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); @@ -46,4 +55,116 @@ public void init() { public void Builder_CacheDisabled() { builder.setCache(new CacheConfiguration(1000)); } + + /** + * Check that a builder whose caller sets no performance profile gives + * the engine the collection settings of the Balanced profile, and not + * the in-memory settings of MaxPerformance. + */ + @Test + public void Builder_NoProfile_CollectionsAreBalanced() throws Exception { + long[] given = collections(configOf(builder)); + + assertArrayEquals( + collections(PerformanceProfiles.Balanced), given); + assertFalse(Arrays.equals( + collections(PerformanceProfiles.MaxPerformance), given)); + } + + /** + * Check that a profile the caller names, in code or by name as a + * configuration file gives it, replaces the Balanced default. + * MaxPerformance and LowMemory give every collection the same settings, + * so this cannot tell those two apart. + */ + @Test + public void Builder_MaxPerformance_ReplacesBalanced() throws Exception { + long[] inMemory = collections(PerformanceProfiles.MaxPerformance); + + builder.setPerformanceProfile(PerformanceProfiles.MaxPerformance); + assertArrayEquals(inMemory, collections(configOf(builder))); + + IPIntelligenceOnPremiseEngineBuilder named = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory) + .setPerformanceProfile("MaxPerformance"); + assertArrayEquals(inMemory, collections(configOf(named))); + assertFalse(Arrays.equals( + collections(PerformanceProfiles.Balanced), inMemory)); + } + + /** + * The native configuration the builder hands to the engine it builds. + */ + private static ConfigIpiSwig configOf( + IPIntelligenceOnPremiseEngineBuilder builder) throws Exception { + Field field = IPIntelligenceOnPremiseEngineBuilder.class + .getDeclaredField("config"); + field.setAccessible(true); + return (ConfigIpiSwig) field.get(builder); + } + + /** + * The collection settings the native library gives the profile. + */ + private static long[] collections(PerformanceProfiles profile) { + ConfigIpiSwig config = new ConfigIpiSwig(); + try { + switch (profile) { + case Balanced: + config.setBalanced(); + break; + case MaxPerformance: + config.setMaxPerformance(); + break; + default: + throw new IllegalArgumentException(profile.name()); + } + return collections(config); + } finally { + config.delete(); + } + } + + /** + * The number of entries loaded at the start and the cache capacity of + * every collection in the configuration, in a fixed order. Concurrency + * is left out, because the builder sets its own. + */ + private static long[] collections(ConfigIpiSwig config) { + CollectionConfigSwig[] all = { + borrowed(config.getStrings()), + borrowed(config.getComponents()), + borrowed(config.getMaps()), + borrowed(config.getProperties()), + borrowed(config.getValues()), + borrowed(config.getProfiles()), + borrowed(config.getGraphs()), + borrowed(config.getProfileGroups()), + borrowed(config.getProfileOffsets()), + borrowed(config.getPropertyTypes()), + borrowed(config.getGraph()) + }; + long[] settings = new long[all.length * 2]; + for (int i = 0; i < all.length; i++) { + settings[i * 2] = all[i].getLoaded(); + settings[i * 2 + 1] = all[i].getCapacity(); + } + return settings; + } + + /** + * The generated wrappers mark four ConfigIpiSwig getters as owned until + * they are regenerated with the C library's corrected MetaData.i. + */ + private static CollectionConfigSwig borrowed(CollectionConfigSwig proxy) { + try { + Field own = CollectionConfigSwig.class + .getDeclaredField("swigCMemOwn"); + own.setAccessible(true); + own.setBoolean(proxy, false); + return proxy; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } } diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java index 0ebff7a84..210fa80dd 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java @@ -61,8 +61,11 @@ public class IPIntelligenceOnPremisePipelineBuilder private String dataUpdateUrl = null; private Boolean dataUpdateVerifyMd5 = null; private DataUpdateUrlFormatter dataUpdateUrlFormatter = null; - private Constants.PerformanceProfiles performanceProfile = - Constants.PerformanceProfiles.MaxPerformance; + /** + * The profile the caller chose, or null for the engine builder's + * default, which is Balanced. + */ + private Constants.PerformanceProfiles performanceProfile = null; private DataUpdateService dataUpdateService; @@ -240,7 +243,8 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateLicenseKey( /** * Set the performance profile for the IP Intelligence engine. - * Defaults to MaxPerformance. + * Defaults to Balanced, which reads the data file as lookups need it, + * through caches. MaxPerformance loads the whole data file into memory. * @param profile The performance profile to use. * @return This builder instance. */ @@ -386,8 +390,13 @@ private IPIntelligenceOnPremiseEngine configureAndBuild( builder.setProperty(property); } } - // Configure performance profile - builder.setPerformanceProfile(performanceProfile); + // Configure the performance profile only when the caller chose one. + // The engine builder already starts from Balanced, and applying + // Balanced again here would also replace the concurrency the engine + // builder set. + if (performanceProfile != null) { + builder.setPerformanceProfile(performanceProfile); + } // Configure the concurrency if (concurrency > 1) { diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java new file mode 100644 index 000000000..3af3ff389 --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java @@ -0,0 +1,207 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.CollectionConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ConfigIpiSwig; +import fiftyone.ipintelligence.shared.testhelpers.FileUtils; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.Constants.PerformanceProfiles; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * Tests that the on-premise pipeline gives its engine the Balanced + * performance profile unless the caller names another, by reading the + * native configuration the engine was built with. + */ +public class OnPremisePerformanceProfileTests { + + private static File dataFile; + + @BeforeClass + public static void findDataFile() { + dataFile = FileUtils.getHashFile(); + } + + @Before + public void requireDataFile() { + Assume.assumeTrue( + "No IP Intelligence data file was found, so this test is " + + "skipped. Set " + FileUtils.IPI_PATH_ENV_VAR + " to the path " + + "of an .ipi file, or download the Lite file with " + + "cxx/data/get-data-files.", + dataFile != null); + } + + @Test + public void build_NoProfile_EngineIsGivenBalanced() throws Exception { + try (Pipeline pipeline = builder().build()) { + ConfigIpiSwig given = configOf(pipeline); + + assertArrayEquals( + collections(PerformanceProfiles.Balanced), + collections(given)); + assertFalse(Arrays.equals( + collections(PerformanceProfiles.MaxPerformance), + collections(given))); + // Applying a profile replaces the concurrency, so this checks + // the pipeline did not apply one over the engine builder's. + assertEquals( + configOf(new IPIntelligenceOnPremiseEngineBuilder()) + .getConcurrency(), + given.getConcurrency()); + } + } + + /** + * MaxPerformance and LowMemory give every collection the same settings, + * so this checks only that the profile named replaced Balanced. + */ + @Test + public void build_LowMemory_ReplacesBalanced() throws Exception { + try (Pipeline pipeline = builder() + .setPerformanceProfile(PerformanceProfiles.LowMemory) + .build()) { + assertArrayEquals( + collections(PerformanceProfiles.LowMemory), + collections(configOf(pipeline))); + } + } + + /** + * A pipeline builder with the profile and the concurrency left at their + * defaults. Data updates and usage sharing are switched off, so that + * the test needs no licence key and no network, and neither touches + * the profile. + */ + private static IPIntelligenceOnPremisePipelineBuilder builder() + throws Exception { + return new IPIntelligencePipelineBuilder( + LoggerFactory.getILoggerFactory()) + .useOnPremise(dataFile.getAbsolutePath(), false) + .setAutoUpdate(false) + .setDataUpdateOnStartup(false) + .setShareUsage(false); + } + + /** The native configuration the pipeline's engine was built with. */ + private static ConfigIpiSwig configOf(Pipeline pipeline) + throws Exception { + return (ConfigIpiSwig) field( + IPIntelligenceOnPremiseEngine.class, "config") + .get(pipeline.getElement(IPIntelligenceOnPremiseEngine.class)); + } + + /** The native configuration an engine builder would hand on. */ + private static ConfigIpiSwig configOf( + IPIntelligenceOnPremiseEngineBuilder builder) throws Exception { + return (ConfigIpiSwig) field( + IPIntelligenceOnPremiseEngineBuilder.class, "config") + .get(builder); + } + + private static Field field(Class type, String name) + throws NoSuchFieldException { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field; + } + + /** The collection settings the native library gives the profile. */ + private static long[] collections(PerformanceProfiles profile) { + ConfigIpiSwig config = new ConfigIpiSwig(); + try { + switch (profile) { + case Balanced: + config.setBalanced(); + break; + case MaxPerformance: + config.setMaxPerformance(); + break; + case LowMemory: + config.setLowMemory(); + break; + default: + throw new IllegalArgumentException(profile.name()); + } + return collections(config); + } finally { + config.delete(); + } + } + + /** + * The number of entries loaded at the start and the cache capacity of + * every collection in the configuration, in a fixed order. Concurrency + * is left out, because the engine builder sets its own. + */ + private static long[] collections(ConfigIpiSwig config) { + CollectionConfigSwig[] all = { + borrowed(config.getStrings()), + borrowed(config.getComponents()), + borrowed(config.getMaps()), + borrowed(config.getProperties()), + borrowed(config.getValues()), + borrowed(config.getProfiles()), + borrowed(config.getGraphs()), + borrowed(config.getProfileGroups()), + borrowed(config.getProfileOffsets()), + borrowed(config.getPropertyTypes()), + borrowed(config.getGraph()) + }; + long[] settings = new long[all.length * 2]; + for (int i = 0; i < all.length; i++) { + settings[i * 2] = all[i].getLoaded(); + settings[i * 2 + 1] = all[i].getCapacity(); + } + return settings; + } + + /** + * The generated wrappers mark four ConfigIpiSwig getters as owned until + * they are regenerated with the C library's corrected MetaData.i. + */ + private static CollectionConfigSwig borrowed(CollectionConfigSwig proxy) { + try { + field(CollectionConfigSwig.class, "swigCMemOwn") + .setBoolean(proxy, false); + return proxy; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } +} From 62f7ed88264da678d8d9a0c91eab3bef2a826287 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 09:01:37 +0000 Subject: [PATCH 26/95] Add the random delay before an available update as seconds (#74) * Test a check that starts before registration stores its future registerDataFile schedules the first check and only then stores the future it gets back, so a check with a short delay can start in between. If the check reads the future in that gap it finds none, clears nothing, and its own re-registration then finds the stored future and schedules no next check. DataUpdateService_UpdateFromFile_FileUpdated failed this way once in a full build on macOS x64 with Java 11. The two new tests force that order on every run, one for a check that applies an update and one for a check that applies none, which is the path the retry after a 429 response takes. Registration is held after scheduling until the check has read the future or is blocked on a lock that registration holds, and the check's first read waits until registration has stored the future. Every wait is bounded, so a design that deadlocks fails rather than hangs. Each failure message lists the order in which the two threads read and wrote the future. For #70 * Store a data file's future before its check can read it Every read and write of a data file's future now happens while holding one lock in the data update service. registerDataFile holds it from scheduling the check to storing the future, and the check takes it before it cancels, clears or replaces the future, so a check that starts at once waits until registration has stored its future. Unregistering takes it for the same reason. The lock is the service's own rather than the data file's update lock, because the file system watcher holds the update lock while it refreshes the engine and while it waits for the new file to open. Taking that lock for the future would make unregistering wait for a refresh. Only scheduling, cancelling and storing happen while holding the new lock, so it is never held while waiting for the configuration lock or an update lock. For #70 * Word the new comments to match what the lock and the tests cover The lock's comment now says it covers the service's own reads and writes of the future, the test data file's comment says it records the check's reads, and the second test's comment says which checks share its path. No code changes. For #70 * Test the random delay before an available update is in seconds * Add the random delay before an available update as seconds --- .../services/DataUpdateServiceDefault.java | 50 ++++++++++--------- .../services/DataUpdateServiceTests.java | 41 +++++++++++++++ 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index dc2aabe8b..007b21211 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -244,7 +244,7 @@ public void run() { checkForUpdate(dataFile, false); } }, - getInterval(dataFile.getConfiguration()))); + getIntervalMillis(dataFile.getConfiguration()))); } } } @@ -435,11 +435,12 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { synchronized (futureLock) { if (aspectDataFile.getAutomaticUpdatesEnabled() && aspectDataFile.getFuture() == null) { - long delay = getInterval(aspectDataFile.getConfiguration()); + long delayMillis = + getIntervalMillis(aspectDataFile.getConfiguration()); if (aspectDataFile.getUpdateAvailableTime() != null) { if (aspectDataFile.getUpdateAvailableTime().getTime() > System.currentTimeMillis()) { - delay = applyIntervalRandomisation( + delayMillis = applyIntervalRandomisationMillis( aspectDataFile.getUpdateAvailableTime().getTime() - System.currentTimeMillis(), aspectDataFile.getConfiguration()); @@ -447,7 +448,8 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { } // Create a timer that will go off when the engine expects // updated data to be available. - logger.debug("Scheduling update check with delay {}", delay); + logger.debug("Scheduling update check with delay {} millis", + delayMillis); try { ScheduledFuture future = futureFactory.schedule( new Runnable() { @@ -459,7 +461,7 @@ public void run() { logger.debug("Returning from checkForUpdate"); } }, - delay); + delayMillis); logger.debug("Update check scheduled for {} millis", future.getDelay(TimeUnit.MILLISECONDS)); aspectDataFile.setFuture(future); @@ -566,35 +568,37 @@ public void unregisterDataFile(AspectEngineDataFile dataFile) { } /** - * Get the random delay in milliseconds to add on to the scheduled time - * for a specific Engine, no greater than the Engine's max randomisation. + * Get the polling interval for a data file plus a random number of + * seconds, no greater than the configured maximum randomisation. * - * @param config to get the delay for - * @return a random delay in milliseconds. + * @param config to get the interval for + * @return the interval in milliseconds. */ - private long getInterval(DataFileConfiguration config) { - int seconds = 0; + private long getIntervalMillis(DataFileConfiguration config) { + long intervalMillis = 0; if (config.getPollingIntervalSeconds() > 0) { - seconds = config.getPollingIntervalSeconds(); + intervalMillis = config.getPollingIntervalSeconds() * 1000L; } - return applyIntervalRandomisation(seconds, config) * 1000; - + return applyIntervalRandomisationMillis(intervalMillis, config); } /** - * Add a random amount of time to the specified interval + * Add a random number of whole seconds, up to the maximum in the + * configuration, to the specified interval. * - * @param interval the interval to add a random amount of time to - * @param config the {@link DataFileConfiguration} object that specifies the - * maximum number of seconds to add - * @return the new interval + * @param intervalMillis the interval in milliseconds to add to + * @param config the {@link DataFileConfiguration} object that + * specifies the maximum number of seconds to add + * @return the new interval in milliseconds */ - private long applyIntervalRandomisation(long interval, DataFileConfiguration config) { - long seconds = 0; + private long applyIntervalRandomisationMillis( + long intervalMillis, + DataFileConfiguration config) { + long randomSeconds = 0; if (config.getMaxRandomisationSeconds() > 0) { - seconds = random.nextInt(config.getMaxRandomisationSeconds()); + randomSeconds = random.nextInt(config.getMaxRandomisationSeconds()); } - return interval + seconds; + return intervalMillis + randomSeconds * 1000; } /** diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index d8768cf6c..52176409e 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -42,6 +42,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; import org.mockito.internal.matchers.LessOrEqual; import org.mockito.invocation.InvocationOnMock; @@ -233,6 +234,46 @@ public boolean matches(Long aLong) { ); } + @Test + public void DataUpdateService_Register_UpdateAvailableRandomDelayInSeconds() { + // Arrange + final int registrations = 50; + final long availableInMillis = 3600000; + final long maxDelayMillis = 600 * 1000; + final Date availableAt = + new Date(System.currentTimeMillis() + availableInMillis); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setMaxRandomisationSeconds(600); + + // Act + for (int i = 0; i < registrations; i++) { + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setUpdateAvailableTime(availableAt); + file.setEngine(mock(OnPremiseAspectEngine.class)); + file.setConfiguration(config); + dataUpdate.registerDataFile(file); + } + + // Assert + // The delay is the time until the update is available plus a random + // number of seconds, so across many registrations at least one adds + // more than a second, and none adds more than the maximum. + ArgumentCaptor delays = ArgumentCaptor.forClass(Long.class); + verify(futureFactory, times(registrations)).schedule( + any(Runnable.class), + delays.capture()); + long largestExtra = Long.MIN_VALUE; + for (long delay : delays.getAllValues()) { + long extra = delay - availableInMillis; + largestExtra = Math.max(largestExtra, extra); + assertTrue("Delay " + delay + " adds more than the maximum", + extra <= maxDelayMillis); + } + assertTrue("Largest random delay was " + largestExtra + " millis", + largestExtra > 1000); + } + @Test public void DataUpdateService_Register_AutoUpdateConfiguredInterval() { // Arrange From 142615df086cf5a2378ffa47cea55bcb87a34488 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:15:47 +0100 Subject: [PATCH 27/95] Merged Pull Request '#76 update-example-assets/rc->rc: Update example assets' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- .../main/resources/fodid/creator-context/examples-main.min.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css index 8007642a3..42a9f40d7 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css @@ -1 +1 @@ -/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}.a-size--10{width:.1615055829rem;height:.1615055829rem}.a-size--9{width:.1938066995rem;height:.1938066995rem}.a-size--8{width:.2325680394rem;height:.2325680394rem}.a-size--7{width:.2790816472rem;height:.2790816472rem}.a-size--6{width:.3348979767rem;height:.3348979767rem}.a-size--5{width:.401877572rem;height:.401877572rem}.a-size--4{width:.4822530864rem;height:.4822530864rem}.a-size--3{width:.5787037037rem;height:.5787037037rem}.a-size--2{width:.6944444444rem;height:.6944444444rem}.a-size--1{width:.8333333333rem;height:.8333333333rem}.a-size-0{width:1rem;height:1rem}.a-size-1{width:1.2rem;height:1.2rem}.a-size-2{width:1.44rem;height:1.44rem}.a-size-3{width:1.728rem;height:1.728rem}.a-size-4{width:2.0736rem;height:2.0736rem}.a-size-5{width:2.48832rem;height:2.48832rem}.a-size-6{width:2.985984rem;height:2.985984rem}.a-size-7{width:3.5831808rem;height:3.5831808rem}.a-size-8{width:4.29981696rem;height:4.29981696rem}.a-size-9{width:5.159780352rem;height:5.159780352rem}.a-size-10{width:6.1917364224rem;height:6.1917364224rem}.a-size-11{width:7.4300837069rem;height:7.4300837069rem}.a-size-12{width:8.9161004483rem;height:8.9161004483rem}.a-size-13{width:10.6993205379rem;height:10.6993205379rem}.a-size-14{width:12.8391846455rem;height:12.8391846455rem}.a-size-15{width:15.4070215746rem;height:15.4070215746rem}.a-size-16{width:18.4884258895rem;height:18.4884258895rem}.a-size-17{width:22.1861110674rem;height:22.1861110674rem}.a-size-18{width:26.6233332809rem;height:26.6233332809rem}.a-size-19{width:31.9479999371rem;height:31.9479999371rem}.a-size-20{width:38.3375999245rem;height:38.3375999245rem}.a-font-size--10{font-size:.1615055829rem}.a-font-size--9{font-size:.1938066995rem}.a-font-size--8{font-size:.2325680394rem}.a-font-size--7{font-size:.2790816472rem}.a-font-size--6{font-size:.3348979767rem}.a-font-size--5{font-size:.401877572rem}.a-font-size--4{font-size:.4822530864rem}.a-font-size--3{font-size:.5787037037rem}.a-font-size--2{font-size:.6944444444rem}.a-font-size--1{font-size:.8333333333rem}.a-font-size-0{font-size:1rem}.a-font-size-1{font-size:1.2rem}.a-font-size-2{font-size:1.44rem}.a-font-size-3{font-size:1.728rem}.a-font-size-4{font-size:2.0736rem}.a-font-size-5{font-size:2.48832rem}.a-font-size-6{font-size:2.985984rem}.a-font-size-7{font-size:3.5831808rem}.a-font-size-8{font-size:4.29981696rem}.a-font-size-9{font-size:5.159780352rem}.a-font-size-10{font-size:6.1917364224rem}.a-font-size-11{font-size:7.4300837069rem}.a-font-size-12{font-size:8.9161004483rem}.a-font-size-13{font-size:10.6993205379rem}.a-font-size-14{font-size:12.8391846455rem}.a-font-size-15{font-size:15.4070215746rem}.a-font-size-16{font-size:18.4884258895rem}.a-font-size-17{font-size:22.1861110674rem}.a-font-size-18{font-size:26.6233332809rem}.a-font-size-19{font-size:31.9479999371rem}.a-font-size-20{font-size:38.3375999245rem}.a-font--default{font-family:Arial,sans-serif}.a-font--code{font-family:Consolas,monospace}.b-text--hidden{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.b-text--default{font-family:Arial,sans-serif;font-size:1rem;color:#252525}.b-text--count{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b}.b-text--count-pill{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b;border:1px solid #BAC82A;border-radius:28px;padding:.093463879rem .4822530864rem;color:#252525;background-color:#fff;display:inline}.b-text--property-pill{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060}.b-text--heading-1{font-family:Arial,sans-serif;font-weight:700;font-size:1.44rem;color:#252525;margin:0 0 1.728rem}.b-text--heading-2{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.b-text--heading-3{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.b-text--code{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060}.b-text--code-snip{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;background-color:#fff;border-radius:5px;border:1px solid #EAEAEA;padding:0 .2325680394rem}.b-text--heading-caps{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525;font-weight:400;text-transform:uppercase;letter-spacing:.5px}.b-text--copyright{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525}.b-text--progress{font-family:Arial,sans-serif;font-size:.8333333333rem}@media(min-width:769px){.b-text--progress{font-size:1rem}}.b-text--progress{color:#252525;letter-spacing:.7px;text-transform:uppercase;font-weight:700}.b-text--red{color:#cc2b27}body{font-family:Arial,sans-serif;font-size:1rem;color:#252525;margin:0;line-height:1.5}*{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;box-sizing:border-box}.b-link{transition:color .3s ease;color:#252525;text-decoration:underline}.b-link:active,.b-link:hover,.b-link:focus{color:#f5841f;text-decoration:underline}.b-link{display:inline-flex;align-items:center}.b-link img{margin:0 .5787037037rem}.b-link--property{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060;color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--property:active,.b-link--property:hover,.b-link--property:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--dotted{color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--dotted:active,.b-link--dotted:hover,.b-link--dotted:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--unstyled{color:inherit;text-decoration:none}.b-link--unstyled:active,.b-link--unstyled:hover,.b-link--unstyled:focus{color:inherit;text-decoration:none}.b-link--docs{transition:color .3s ease;color:#00aeef;text-decoration:none;font-weight:700}.b-link--docs:active,.b-link--docs:hover,.b-link--docs:focus{text-decoration:underline}a.anchor{scroll-margin-top:128px}.b-btn{font-family:Arial,sans-serif;font-size:1rem;color:#252525;color:inherit;text-decoration:none}.b-btn:active,.b-btn:hover,.b-btn:focus{color:inherit;text-decoration:none}.b-btn{display:inline-flex;justify-content:center;align-items:center;padding:.8333333333rem 1.728rem;border:1px solid #BAC82A;background-color:#cbdb2a;cursor:pointer}.b-btn:active,.b-btn:hover,.b-btn:focus{background-color:#daed1a;border:1px solid #CBDB2A;text-decoration:none}.b-btn.b-btn--disabled,.b-btn:disabled{background-color:#f8f8f8;border:1px solid #EAEAEA;color:#646060;cursor:not-allowed;pointer-events:all!important}@media(min-width:769px){.b-btn--large{padding:1.2rem 2.48832rem}}.b-btn--secondary{background-color:#fff}.b-btn--secondary:active,.b-btn--secondary:hover,.b-btn--secondary:focus{background-color:#fff}.b-btn--block{width:100%}.b-btn__icon{margin-right:1rem}.b-btn__icon--after{margin-right:0;margin-left:1rem}input[type=text]{-webkit-appearance:none;-moz-appearance:none;appearance:none}.b-label{color:#646060;font-size:.8333333333rem;margin-bottom:.4822530864rem;width:100%}.b-input-group{display:flex;flex-direction:column}@media(min-width:769px){.b-input-group{flex-direction:row}}.b-input{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:solid 1px #646060;padding:.8333333333rem;line-height:1.2}.b-form-group--with-icon .b-input{padding-left:46px}.b-input::placeholder{color:#bfbfbf;opacity:1}.b-input,.b-input--block{width:100%}.b-form-group{display:flex;flex-direction:column}.b-form-group .b-btn{margin-top:1rem}@media(min-width:769px){.b-form-group .b-btn{margin-top:0;margin-left:.8333333333rem}}.b-form-group--with-icon{position:relative}.b-form-group--with-icon .b-icon{position:absolute;width:20px;height:20px;top:14px;left:10px}.c-eg-page{max-width:800px;margin:0 auto;padding:1.44rem 1.2rem}.c-eg-page__title{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.c-eg-page__heading{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:1.728rem 0 1.2rem}.c-eg-page__lead{margin-bottom:1.44rem}.c-eg-section{margin-bottom:1.728rem}.c-eg-alert{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:1px solid #CC2B27;border-radius:5px;background-color:#fef1f9;padding:.8333333333rem 1rem;margin:1.2rem 0}.c-eg-alert a{color:#cc2b27;font-weight:700}.c-eg-table{font-family:Arial,sans-serif;font-size:1rem;color:#252525;width:100%;max-width:800px;font-size:.8333333333rem;background-color:#fff;border-collapse:collapse;margin-bottom:1.2rem}.c-eg-table__head .c-eg-table__cell{font-weight:700;border-bottom:3px solid #EAEAEA}.c-eg-table__cell{padding:.5787037037rem .6944444444rem;text-align:left;vertical-align:top;border-bottom:1px solid #EAEAEA}.c-eg-table__cell--key{font-weight:700;white-space:nowrap}.c-eg-table__cell ul{margin:0;padding-left:1rem}.c-eg-table__row--used{background-color:#f9fbe9}.c-eg-table__row--present,.c-eg-table__row--alt{background-color:#f8f8f8}.c-eg-table__action{padding:.4822530864rem .6944444444rem;font-size:.6944444444rem;white-space:nowrap;width:auto}.c-eg-legend{font-size:.6944444444rem;color:#646060;margin-bottom:.6944444444rem}.c-eg-legend__swatch{padding:0 .4822530864rem;border-radius:5px}.c-eg-legend__swatch--used{background-color:#f9fbe9}.c-eg-legend__swatch--present{background-color:#f8f8f8}.c-eg-form{margin-bottom:1.44rem}.c-eg-form__row{display:flex;flex-direction:column}@media(min-width:769px){.c-eg-form__row{flex-direction:row;align-items:flex-end}}.c-eg-form .b-btn{margin-top:.8333333333rem}@media(min-width:769px){.c-eg-form .b-btn{margin-top:0;margin-left:.8333333333rem}}.c-eg-map{margin:1.44rem 0}.c-eg-map__title{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.c-eg-map__canvas{height:400px;width:100%;border:1px solid #EAEAEA;border-radius:5px}.c-eg-columns{display:grid;grid-template-columns:1fr;gap:1.44rem}@media(min-width:993px){.c-eg-columns{grid-template-columns:1fr 1fr}}.c-eg-details{margin:1.2rem 0 1.44rem;border:1px solid #EAEAEA;border-radius:5px;padding:0 1rem}.c-eg-details>summary{cursor:pointer;padding:.6944444444rem 0;font-weight:700}.c-eg-button-row{margin:1.2rem 0 1.44rem}.c-eg-message{display:flex;flex-direction:column;align-items:flex-start;gap:.8333333333rem;margin-top:2.0736rem;padding:1rem 1.2rem;background:#f8f8f8;border:1px solid #EAEAEA;border-radius:5px}@media(min-width:769px){.c-eg-message{flex-direction:row;align-items:center;justify-content:space-between;gap:1.2rem}}.c-eg-message__text{margin:0}.c-eg-message__cta{flex-shrink:0}.c-eg-status{font-family:Arial,sans-serif;font-size:1rem;color:#252525;font-weight:700}.c-eg-status--pass{color:#9ba90d}.c-eg-status--fail{color:#cc2b27}.c-eg-status--part{color:#f5841f}.c-eg-status--none{color:#646060;font-weight:400}.c-eg-value{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;word-break:break-all} +/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}.a-size--10{width:.1615055829rem;height:.1615055829rem}.a-size--9{width:.1938066995rem;height:.1938066995rem}.a-size--8{width:.2325680394rem;height:.2325680394rem}.a-size--7{width:.2790816472rem;height:.2790816472rem}.a-size--6{width:.3348979767rem;height:.3348979767rem}.a-size--5{width:.401877572rem;height:.401877572rem}.a-size--4{width:.4822530864rem;height:.4822530864rem}.a-size--3{width:.5787037037rem;height:.5787037037rem}.a-size--2{width:.6944444444rem;height:.6944444444rem}.a-size--1{width:.8333333333rem;height:.8333333333rem}.a-size-0{width:1rem;height:1rem}.a-size-1{width:1.2rem;height:1.2rem}.a-size-2{width:1.44rem;height:1.44rem}.a-size-3{width:1.728rem;height:1.728rem}.a-size-4{width:2.0736rem;height:2.0736rem}.a-size-5{width:2.48832rem;height:2.48832rem}.a-size-6{width:2.985984rem;height:2.985984rem}.a-size-7{width:3.5831808rem;height:3.5831808rem}.a-size-8{width:4.29981696rem;height:4.29981696rem}.a-size-9{width:5.159780352rem;height:5.159780352rem}.a-size-10{width:6.1917364224rem;height:6.1917364224rem}.a-size-11{width:7.4300837069rem;height:7.4300837069rem}.a-size-12{width:8.9161004483rem;height:8.9161004483rem}.a-size-13{width:10.6993205379rem;height:10.6993205379rem}.a-size-14{width:12.8391846455rem;height:12.8391846455rem}.a-size-15{width:15.4070215746rem;height:15.4070215746rem}.a-size-16{width:18.4884258895rem;height:18.4884258895rem}.a-size-17{width:22.1861110674rem;height:22.1861110674rem}.a-size-18{width:26.6233332809rem;height:26.6233332809rem}.a-size-19{width:31.9479999371rem;height:31.9479999371rem}.a-size-20{width:38.3375999245rem;height:38.3375999245rem}.a-font-size--10{font-size:.1615055829rem}.a-font-size--9{font-size:.1938066995rem}.a-font-size--8{font-size:.2325680394rem}.a-font-size--7{font-size:.2790816472rem}.a-font-size--6{font-size:.3348979767rem}.a-font-size--5{font-size:.401877572rem}.a-font-size--4{font-size:.4822530864rem}.a-font-size--3{font-size:.5787037037rem}.a-font-size--2{font-size:.6944444444rem}.a-font-size--1{font-size:.8333333333rem}.a-font-size-0{font-size:1rem}.a-font-size-1{font-size:1.2rem}.a-font-size-2{font-size:1.44rem}.a-font-size-3{font-size:1.728rem}.a-font-size-4{font-size:2.0736rem}.a-font-size-5{font-size:2.48832rem}.a-font-size-6{font-size:2.985984rem}.a-font-size-7{font-size:3.5831808rem}.a-font-size-8{font-size:4.29981696rem}.a-font-size-9{font-size:5.159780352rem}.a-font-size-10{font-size:6.1917364224rem}.a-font-size-11{font-size:7.4300837069rem}.a-font-size-12{font-size:8.9161004483rem}.a-font-size-13{font-size:10.6993205379rem}.a-font-size-14{font-size:12.8391846455rem}.a-font-size-15{font-size:15.4070215746rem}.a-font-size-16{font-size:18.4884258895rem}.a-font-size-17{font-size:22.1861110674rem}.a-font-size-18{font-size:26.6233332809rem}.a-font-size-19{font-size:31.9479999371rem}.a-font-size-20{font-size:38.3375999245rem}.a-font--default{font-family:Arial,sans-serif}.a-font--code{font-family:Consolas,monospace}.b-text--hidden{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.b-text--default{font-family:Arial,sans-serif;font-size:1rem;color:#252525}.b-text--count{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b}.b-text--count-pill{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b;border:1px solid #BAC82A;border-radius:28px;padding:.093463879rem .4822530864rem;color:#252525;background-color:#fff;display:inline}.b-text--property-pill{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060}.b-text--heading-1{font-family:Arial,sans-serif;font-weight:700;font-size:1.44rem;color:#252525;margin:0 0 1.728rem}.b-text--heading-2{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.b-text--heading-3{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.b-text--code{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060}.b-text--code-snip{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;background-color:#fff;border-radius:5px;border:1px solid #EAEAEA;padding:0 .2325680394rem}.b-text--heading-caps{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525;font-weight:400;text-transform:uppercase;letter-spacing:.5px}.b-text--copyright{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525}.b-text--progress{font-family:Arial,sans-serif;font-size:.8333333333rem}@media(min-width:769px){.b-text--progress{font-size:1rem}}.b-text--progress{color:#252525;letter-spacing:.7px;text-transform:uppercase;font-weight:700}.b-text--red{color:#cc2b27}body{font-family:Arial,sans-serif;font-size:1rem;color:#252525;margin:0;line-height:1.5}*{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;box-sizing:border-box}.b-link{transition:color .3s ease;color:#252525;text-decoration:underline}.b-link:active,.b-link:hover,.b-link:focus{color:#f5841f;text-decoration:underline}.b-link{display:inline-flex;align-items:center}.b-link img{margin:0 .5787037037rem}.b-link--property{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060;color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--property:active,.b-link--property:hover,.b-link--property:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--dotted{color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--dotted:active,.b-link--dotted:hover,.b-link--dotted:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--unstyled{color:inherit;text-decoration:none}.b-link--unstyled:active,.b-link--unstyled:hover,.b-link--unstyled:focus{color:inherit;text-decoration:none}.b-link--docs{transition:color .3s ease;color:#00aeef;text-decoration:none;font-weight:700}.b-link--docs:active,.b-link--docs:hover,.b-link--docs:focus{text-decoration:underline}a.anchor{scroll-margin-top:128px}.b-btn{font-family:Arial,sans-serif;font-size:1rem;color:#252525;color:inherit;text-decoration:none}.b-btn:active,.b-btn:hover,.b-btn:focus{color:inherit;text-decoration:none}.b-btn{display:inline-flex;justify-content:center;align-items:center;padding:.8333333333rem 1.728rem;border:1px solid #BAC82A;background-color:#cbdb2a;cursor:pointer}.b-btn:active,.b-btn:hover,.b-btn:focus{background-color:#daed1a;border:1px solid #CBDB2A;text-decoration:none}.b-btn.b-btn--disabled,.b-btn:disabled{background-color:#f8f8f8;border:1px solid #EAEAEA;color:#646060;cursor:not-allowed;pointer-events:all!important}@media(min-width:769px){.b-btn--large{padding:1.2rem 2.48832rem}}.b-btn--secondary{background-color:#fff}.b-btn--secondary:active,.b-btn--secondary:hover,.b-btn--secondary:focus{background-color:#fff}.b-btn--block{width:100%}.b-btn__icon{margin-right:1rem}.b-btn__icon--after{margin-right:0;margin-left:1rem}input[type=text]{-webkit-appearance:none;-moz-appearance:none;appearance:none}.b-label{color:#646060;font-size:.8333333333rem;margin-bottom:.4822530864rem;width:100%}.b-input-group{display:flex;flex-direction:column}@media(min-width:769px){.b-input-group{flex-direction:row}}.b-input{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:solid 1px #646060;padding:.8333333333rem;line-height:1.2}.b-form-group--with-icon .b-input{padding-left:46px}.b-input::placeholder{color:#bfbfbf;opacity:1}.b-input,.b-input--block{width:100%}.b-form-group{display:flex;flex-direction:column}.b-form-group .b-btn{margin-top:1rem}@media(min-width:769px){.b-form-group .b-btn{margin-top:0;margin-left:.8333333333rem}}.b-form-group--with-icon{position:relative}.b-form-group--with-icon .b-icon{position:absolute;width:20px;height:20px;top:14px;left:10px}.c-eg-page{max-width:800px;margin:0 auto;padding:1.44rem 1.2rem}.c-eg-page__title{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.c-eg-page__heading{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:1.728rem 0 1.2rem}.c-eg-page__lead{margin-bottom:1.44rem}.c-eg-section{margin-bottom:1.728rem}.c-eg-alert{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:1px solid #CC2B27;border-radius:5px;background-color:#fef1f9;padding:.8333333333rem 1rem;margin:1.2rem 0}.c-eg-alert a{color:#cc2b27;font-weight:700}.c-eg-table{font-family:Arial,sans-serif;font-size:1rem;color:#252525;width:100%;max-width:800px;font-size:.8333333333rem;background-color:#fff;border-collapse:collapse;margin-bottom:1.2rem}.c-eg-table__head .c-eg-table__cell{font-weight:700;border-bottom:3px solid #EAEAEA}.c-eg-table__cell{padding:.5787037037rem .6944444444rem;text-align:left;vertical-align:top;border-bottom:1px solid #EAEAEA}.c-eg-table__cell--key{font-weight:700;white-space:nowrap}.c-eg-table__cell ul{margin:0;padding-left:1rem}.c-eg-table__row--used{background-color:#f9fbe9}.c-eg-table__row--present,.c-eg-table__row--alt{background-color:#f8f8f8}.c-eg-legend{font-size:.6944444444rem;color:#646060;margin-bottom:.6944444444rem}.c-eg-legend__swatch{padding:0 .4822530864rem;border-radius:5px}.c-eg-legend__swatch--used{background-color:#f9fbe9}.c-eg-legend__swatch--present{background-color:#f8f8f8}.c-eg-form{margin-bottom:1.44rem}.c-eg-form__row{display:flex;flex-direction:column}@media(min-width:769px){.c-eg-form__row{flex-direction:row;align-items:flex-end}}.c-eg-form .b-btn{margin-top:.8333333333rem}@media(min-width:769px){.c-eg-form .b-btn{margin-top:0;margin-left:.8333333333rem}}.c-eg-map{margin:1.44rem 0}.c-eg-map__title{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.c-eg-map__canvas{height:400px;width:100%;border:1px solid #EAEAEA;border-radius:5px}.c-eg-columns{display:grid;grid-template-columns:1fr;gap:1.44rem}@media(min-width:993px){.c-eg-columns{grid-template-columns:1fr 1fr}}.c-eg-details{margin:1.2rem 0 1.44rem;border:1px solid #EAEAEA;border-radius:5px;padding:0 1rem}.c-eg-details>summary{cursor:pointer;padding:.6944444444rem 0;font-weight:700}.c-eg-button-row{margin:1.2rem 0 1.44rem}.c-eg-message{display:flex;flex-direction:column;align-items:flex-start;gap:.8333333333rem;margin-top:2.0736rem;padding:1rem 1.2rem;background:#f8f8f8;border:1px solid #EAEAEA;border-radius:5px}@media(min-width:769px){.c-eg-message{flex-direction:row;align-items:center;justify-content:space-between;gap:1.2rem}}.c-eg-message__text{margin:0}.c-eg-message__cta{flex-shrink:0} From 63d0bb8523a97eaf25ccfea8d5e6d503bd8ac787 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 14:12:55 +0000 Subject: [PATCH 28/95] Refuse a 51Did text with white space inside it, or both base64 alphabets (#65) * Refuse a 51Did text with white space inside it, or both base64 alphabets The 51Did specification (identifier-layout.md, specifications#88 and #90) says a reader ignores white space before and after the text, being any character with the Unicode White_Space property, refuses any other character outside the alphabet, white space inside the text included, and refuses a text holding a character from each alphabet, checking that before it maps - and _ onto + and /. FodId.toStandardBase64 trimmed with String.trim, which removes every character up to U+0020, control characters included, and nothing above it. It then mapped - and _ onto + and / without checking for a mix, and the OWID decoder skips spaces, tabs and line breaks inside the text. So a mixed text always read, and a text with spaces, tabs or line breaks inside it read whenever the padding worked out. toStandardBase64 now trims exactly the 25 White_Space characters, gives null for a text with white space left inside it or with a character from each alphabet, and only then maps one alphabet onto the other. tryFromBase64 reports INVALID_BASE64 for such a text, and fromBase64 throws the OwidException naming INVALID_BASE64. White space alone still reports MISSING_INPUT. This matches 51degrees-dotnet#85. * State why each character in the near-miss test is not white space * Say that the 51Did reader refuses white space inside and mixed alphabets itself The README's status table and the tryFromBase64 documentation said an envelope fault carries the OWID library's status as the envelope reader found it. White space inside the text and characters from both base64 alphabets are now refused by FodId before the OWID library sees the text, under the same INVALID_BASE64 status, so both now say so. The test helper that signs envelopes until one holds both + and / usually needs a few tries, not one or two as its comment said. --- pipeline/pipeline.did/README.md | 12 +- .../java/fiftyone/pipeline/did/FodId.java | 99 ++++++-- .../pipeline/did/FodIdParseStatus.java | 11 +- .../fiftyone/pipeline/did/FodIdTests.java | 212 +++++++++++++++++- 4 files changed, 311 insertions(+), 23 deletions(-) diff --git a/pipeline/pipeline.did/README.md b/pipeline/pipeline.did/README.md index f6bb2f1b5..8784643ad 100644 --- a/pipeline/pipeline.did/README.md +++ b/pipeline/pipeline.did/README.md @@ -154,9 +154,11 @@ status in the message, a payload shorter than its type's minimum throws Both string readers accept the standard alphabet the cloud issues (`+`, `/`, padded) and the URL-safe alphabet a page puts in a link (`-`, `_`, padding -optional), with or without whitespace around the value. `asBase64Url()` gives -the URL-safe form back, so an identifier can go into a URL without any -conversion by the caller. +optional), with or without white space around the value. A value with white +space inside it, or with characters from both alphabets, is refused as +`INVALID_BASE64`, because the issuer writes neither and either is a sign of a +damaged value. `asBase64Url()` gives the URL-safe form back, so an identifier +can go into a URL without any conversion by the caller. ### What the status means @@ -167,8 +169,8 @@ found it, and adds its own for the payload rules. | Status | Meaning | Layer | |---|---|---| | `PARSED` | A structurally valid 51Did. Says nothing about the signature. | | -| `MISSING_INPUT` | Null or empty input. | OWID | -| `INVALID_BASE64` | The string is not base64. | OWID | +| `MISSING_INPUT` | Null or empty input, or white space alone. | OWID | +| `INVALID_BASE64` | The string is not base64 in one alphabet. White space inside it, and characters from both alphabets, report this. | OWID, or 51Did for white space inside the text and mixed alphabets | | `UNSUPPORTED_VERSION` | The first byte names an envelope version the reader does not know. | OWID | | `UNEXPECTED_END` | The data stopped in the middle of an envelope field. | OWID | | `INVALID_DOMAIN_ENCODING` | The creator domain is unterminated or longer than a domain name can be. | OWID | diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java index 2ff930fac..e94ef24d2 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java @@ -87,7 +87,9 @@ *

    * The cloud issues a 51Did in standard base64 with padding, and a page that * puts one in a link converts it to the URL-safe alphabet without padding. - * Both readers of a string accept either form. + * Both readers of a string accept either form, and ignore white space + * around it. They refuse a text with white space inside it, or with + * characters from both alphabets, because the issuer writes neither. *

    * Java's {@link Owid} is {@code final}, so this type composes an OWID * rather than inheriting from it: it holds the envelope and delegates @@ -195,15 +197,21 @@ private FodId( * Reads a 51Did from its base64 form without throwing, in either the * standard alphabet ({@code +} and {@code /}, as the cloud issues it) or * the URL-safe alphabet ({@code -} and {@code _}, as a page puts it in a - * link), with or without padding, and with or without whitespace around - * it. + * link), with or without padding, and with or without white space around + * it. White space inside the text, or characters from both alphabets, + * give {@link FodIdParseStatus#INVALID_BASE64}, and a text of white space + * alone gives {@link FodIdParseStatus#MISSING_INPUT}. White space is + * every character with the Unicode White_Space property. *

    * The value may be anything at all, because it is external data and * failing to be a 51Did is an ordinary outcome rather than an error. The * result reports whether the read worked, the 51Did only when it did, * and a named reason either way. An envelope failure carries the OWID * library's own status unchanged, and a payload failure is one of the - * 51Did payload statuses. See {@link FodIdParseStatus}. + * 51Did payload statuses. White space inside the text and characters + * from both alphabets are refused by this reader itself, before the OWID + * library sees the text, under the same + * {@link FodIdParseStatus#INVALID_BASE64}. See {@link FodIdParseStatus}. *

    * Success means the input is structurally a 51Did. The signature has * not been checked. @@ -216,7 +224,11 @@ public static FodIdParseResult tryFromBase64(String base64) { if (base64 == null) { return FodIdParseResult.failed(FodIdParseStatus.MISSING_INPUT); } - return read(Owid.parse(toStandardBase64(base64))); + String standard = toStandardBase64(base64); + if (standard == null) { + return FodIdParseResult.failed(FodIdParseStatus.INVALID_BASE64); + } + return read(Owid.parse(standard)); } /** @@ -331,8 +343,9 @@ private static FodIdParseResult read(Owid owid) { * @param base64 base64 of the full OWID envelope * @return the parsed 51Did * @throws NullPointerException if {@code base64} is null - * @throws OwidException if the string is not valid base64 or not a - * valid OWID envelope, with the + * @throws OwidException if the string is not valid base64 in one + * alphabet, has white space inside it, or + * is not a valid OWID envelope, with the * {@link FodIdParseStatus} in the message * @throws IllegalArgumentException if the payload is shorter than the * minimum for its identifier type, names a @@ -347,22 +360,57 @@ public static FodId fromBase64(String base64) throws OwidException { /** * Restores a base64 string that may use the URL-safe alphabet, with or * without padding, to the standard alphabet with padding, which is the - * only alphabet the envelope library reads. Leading and trailing - * whitespace is removed first, because a value read from a header, a + * only alphabet the envelope library reads. White space before and + * after the text is removed first, being every character with the + * Unicode White_Space property, because a value read from a header, a * file or a form field often carries a newline or a space around it and * neither belongs to the identifier. Then {@code -} becomes {@code +}, * {@code _} becomes {@code /}, and {@code ==} or {@code =} is appended * when the length modulo 4 is 2 or 3. That padding is worked out from - * the trimmed length, so whitespace cannot push the value into the wrong - * case. A value already in the standard padded form with no whitespace - * around it is returned unchanged. Nothing here decides whether the - * result is base64 at all, which is the envelope library's answer. + * the trimmed length, so white space cannot push the value into the + * wrong case. A value already in the standard padded form with no white + * space around it is returned unchanged. + *

    + * A text with white space inside it, or with a character from each + * alphabet such as both {@code +} and {@code -}, is refused, as the + * 51Did specification requires. The issuer writes neither, so either is + * a sign of a damaged value, such as a {@code +} a form decoder turned + * into a space or a value converted only in part. Any other character + * outside the alphabet is left for the envelope library to refuse. * * @param value the base64 text in either alphabet - * @return the same value in the standard alphabet with padding + * @return the same value in the standard alphabet with padding, or null + * when the text is refused */ static String toStandardBase64(String value) { - String standard = value.trim().replace('-', '+').replace('_', '/'); + int start = 0; + int end = value.length(); + while (start < end && isWhiteSpace(value.charAt(start))) { + start++; + } + while (end > start && isWhiteSpace(value.charAt(end - 1))) { + end--; + } + String text = value.substring(start, end); + // Whether the text holds a character that only the standard + // alphabet uses, and one that only the URL-safe alphabet uses. + boolean hasStandard = false; + boolean hasUrlSafe = false; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + // The envelope library skips spaces and line breaks inside a + // text, so they are refused here, before it sees them. + if (isWhiteSpace(c)) { + return null; + } + hasStandard |= c == '+' || c == '/'; + hasUrlSafe |= c == '-' || c == '_'; + } + // Checked before the mapping below, which would hide a mix. + if (hasStandard && hasUrlSafe) { + return null; + } + String standard = text.replace('-', '+').replace('_', '/'); switch (standard.length() % 4) { case 2: return standard + "=="; @@ -373,6 +421,27 @@ static String toStandardBase64(String value) { } } + /** + * Whether the character has the Unicode White_Space property. The 25 + * code points are listed rather than taken from the runtime, because + * {@link String#trim()} and {@link Character#isWhitespace(char)} each + * differ from the property, and the Unicode version behind the regular + * expression classes changes with the Java release. + */ + private static boolean isWhiteSpace(char c) { + return (c >= '\t' && c <= '\r') + || c == ' ' + || c == 0x0085 + || c == 0x00A0 + || c == 0x1680 + || (c >= 0x2000 && c <= 0x200A) + || c == 0x2028 + || c == 0x2029 + || c == 0x202F + || c == 0x205F + || c == 0x3000; + } + /** * Parses a 51Did from the raw bytes of an OWID envelope, accepting the * same inputs as {@link #tryFromByteArray(byte[])}, and throws when the diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java index cbb98bdcd..2ce6d06fc 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java @@ -49,7 +49,10 @@ public enum FodIdParseStatus { */ PARSED, - /** Nothing was supplied to read, being a null or empty value. */ + /** + * Nothing was supplied to read, being a null or empty value, or a text + * of white space alone. + */ MISSING_INPUT, /** @@ -59,7 +62,11 @@ public enum FodIdParseStatus { */ INVALID_INPUT_TYPE, - /** The string is not valid base64, so there are no bytes to read. */ + /** + * The string is not valid base64 in one alphabet, so there are no bytes + * to read. White space inside the string, and characters from both + * alphabets, report this. + */ INVALID_BASE64, /** The first byte names an envelope version this reader does not know. */ diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java index adf06d49b..f30d70811 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java @@ -46,6 +46,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -456,7 +457,7 @@ public void roundtrip_ThroughBytesConstructor_PreservesAllFields() @Test public void fromBase64_AcceptsStandardUrlSafeAndUnpadded() throws Exception { - String standard = factory.signedOwidBase64(canonicalPayload()); + String standard = standardWithBothSpecials(); String urlSafe = standard.replace('+', '-').replace('/', '_'); String unpadded = urlSafe.replace("=", ""); // The envelope is 124 bytes, so the standard form always ends in @@ -511,6 +512,111 @@ public void toStandardBase64_RestoresAlphabetAndPadding() { assertEquals("+/8=", FodId.toStandardBase64(" -_8 ")); assertEquals("+/==", FodId.toStandardBase64("-_\r\n")); assertEquals("abcd", FodId.toStandardBase64(" abcd ")); + assertEquals("YQ==", FodId.toStandardBase64(" YQ\n")); + assertEquals("+/==", FodId.toStandardBase64("\u3000-_\u3000")); + } + + // ----- White space and the two alphabets ----- + + @Test + public void toStandardBase64_WhiteSpaceInsideOrBothAlphabets_GivesNull() { + String[] refused = { + "YW Jj", "YW\tJj", "YW\r\nJj", "YW\u00A0Jj", "YW\u3000Jj", + "+_", "-/", "+-", "/_" + }; + for (String text : refused) { + assertNull(show(text), FodId.toStandardBase64(text)); + } + } + + @Test + public void toStandardBase64_EachWhiteSpace_TrimmedAroundRefusedInside() { + assertEquals(25, WHITE_SPACE.length()); + for (char space : WHITE_SPACE.toCharArray()) { + String shown = show(String.valueOf(space)); + assertEquals(shown, + "YQ==", FodId.toStandardBase64(space + "YQ" + space)); + assertNull(shown, FodId.toStandardBase64("YW" + space + "Jj")); + } + } + + @Test + public void tryFromBase64_WhiteSpaceAroundTheText_Parses() throws Exception { + String standard = standardWithBothSpecials(); + String[][] around = { + { " ", "\n" }, + { "\t", "\r\n" }, + { "\u00A0", "\u00A0" }, + { "\u3000", "\u3000" }, + { WHITE_SPACE, WHITE_SPACE } + }; + for (String text : forms(standard)) { + for (String[] pair : around) { + assertParsesAs(standard, pair[0] + text + pair[1]); + } + for (char space : WHITE_SPACE.toCharArray()) { + assertParsesAs(standard, space + text + space); + } + } + } + + @Test + public void tryFromBase64_WhiteSpaceInsideTheText_InvalidBase64() + throws Exception { + String[] spaces = { " ", " ", " ", "\t", "\r\n", "\u00A0", "\u3000" }; + int[] counts = { 1, 2, 3, 4, 2, 4, 1 }; + for (String text : forms(standardWithBothSpecials())) { + for (int i = 0; i < spaces.length; i++) { + assertRefusedBeforeDecoding( + spaced(text, spaces[i], counts[i])); + } + for (char space : WHITE_SPACE.toCharArray()) { + assertRefusedBeforeDecoding( + spaced(text, String.valueOf(space), 1)); + } + } + } + + @Test + public void tryFromBase64_BothAlphabets_InvalidBase64() throws Exception { + String[] forms = forms(standardWithBothSpecials()); + // The standard text with one + or / changed to its URL-safe partner. + assertRefusedBeforeDecoding(replaceFirst(forms[0], '+', '-')); + assertRefusedBeforeDecoding(replaceFirst(forms[0], '/', '_')); + // Both URL-safe texts with one - or _ changed to its standard + // partner. + for (String urlSafe : new String[] { forms[1], forms[2] }) { + assertRefusedBeforeDecoding(replaceFirst(urlSafe, '-', '+')); + assertRefusedBeforeDecoding(replaceFirst(urlSafe, '_', '/')); + } + } + + @Test + public void tryFromBase64_OnlyWhiteSpace_MissingInput() { + String[] texts = { " ", "\r\n", "\u00A0", "\u3000", WHITE_SPACE }; + for (String text : texts) { + FodIdParseResult read = FodId.tryFromBase64(text); + assertFalse(show(text), read.isSuccess()); + assertEquals(show(text), + FodIdParseStatus.MISSING_INPUT, read.getStatus()); + } + } + + @Test + public void tryFromBase64_OtherCharacterAroundTheText_InvalidBase64() + throws Exception { + // Characters without the Unicode White_Space property that are + // easy to take for it, being controls String.trim removes, U+180E, + // which had the property before Unicode 6.3, and two zero width + // characters. + char[] others = { + '\u0000', '\u001C', '\u001F', '\u180E', '\u200B', '\uFEFF' + }; + for (String text : forms(standardWithBothSpecials())) { + for (char other : others) { + assertReadersRefuse(other + text + other); + } + } } @Test @@ -569,4 +675,108 @@ public void factory_SignedAtChosenDate_VerifiesWithItsKey() throws Exception { assertTrue(fodId.verify(factory.publicPem)); assertFalse(fodId.verify(Crypto.generate().publicKeyPem())); } + + // ----- Helpers ----- + + /** + * Every character with the Unicode White_Space property, as listed in + * the Unicode Character Database file PropList.txt. + */ + private static final String WHITE_SPACE = + "\t\n\u000B\f\r \u0085\u00A0\u1680" + + "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009" + + "\u200A\u2028\u2029\u202F\u205F\u3000"; + + /** + * A signed envelope whose standard base64 holds both characters that + * differ between the alphabets, so every conversion is exercised. The + * signature is random, so envelopes are signed until one qualifies, + * which usually takes a few tries. + */ + private String standardWithBothSpecials() throws OwidException { + for (int attempt = 0; attempt < 100; attempt++) { + String base64 = factory.signedOwidBase64(canonicalPayload()); + if (base64.indexOf('+') >= 0 && base64.indexOf('/') >= 0) { + return base64; + } + } + throw new IllegalStateException( + "No signed envelope held both + and / in 100 tries."); + } + + /** + * The standard text, and the URL-safe text with and without its + * padding. + */ + private static String[] forms(String standard) { + String urlSafe = standard.replace('+', '-').replace('/', '_'); + return new String[] { standard, urlSafe, urlSafe.replace("=", "") }; + } + + /** + * The text with the white space put in at a number of points spread + * evenly through it. + */ + private static String spaced(String text, String space, int count) { + StringBuilder spaced = new StringBuilder(text); + for (int point = count; point >= 1; point--) { + spaced.insert(text.length() * point / (count + 1), space); + } + return spaced.toString(); + } + + /** The text with its first {@code from} changed to {@code to}. */ + private static String replaceFirst(String text, char from, char to) { + int index = text.indexOf(from); + assertTrue(show(text) + " holds no " + from, index >= 0); + return text.substring(0, index) + to + text.substring(index + 1); + } + + /** + * The text with every character outside printable ASCII written as its + * code, so a failure message shows where it is. + */ + private static String show(String text) { + StringBuilder shown = new StringBuilder(); + for (char c : text.toCharArray()) { + if (c > ' ' && c < 0x7F) { + shown.append(c); + } else { + shown.append(String.format("\\u%04X", (int) c)); + } + } + return shown.toString(); + } + + private static void assertParsesAs(String standard, String text) + throws OwidException { + FodIdParseResult read = FodId.tryFromBase64(text); + assertTrue(read.getStatus() + " for " + show(text), read.isSuccess()); + assertEquals(show(text), standard, read.getValue().asBase64()); + assertEquals(show(text), standard, FodId.fromBase64(text).asBase64()); + } + + /** + * Checks that both readers refuse the text as invalid base64, the one + * that throws naming the status. + */ + private static void assertReadersRefuse(String text) { + FodIdParseResult read = FodId.tryFromBase64(text); + assertFalse("Parsed " + show(text), read.isSuccess()); + assertNull(show(text), read.getValue()); + assertEquals(show(text), + FodIdParseStatus.INVALID_BASE64, read.getStatus()); + OwidException thrown = assertThrows(show(text), + OwidException.class, () -> FodId.fromBase64(text)); + assertTrue(thrown.getMessage().contains("INVALID_BASE64")); + } + + /** + * Checks that both readers refuse the text, and that the step they + * share refuses it before the envelope library decodes anything. + */ + private static void assertRefusedBeforeDecoding(String text) { + assertReadersRefuse(text); + assertNull(show(text), FodId.toStandardBase64(text)); + } } From c8ffde8523c53df0bdeac1f2053cd2252ab92d06 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 17:50:17 +0000 Subject: [PATCH 29/95] Use LowMemory in the IP Intelligence examples built for the Enterprise file (#73) * Use LowMemory in the IP Intelligence examples built for the Enterprise file The Enterprise IP Intelligence data file is over 6 GB, and MaxPerformance loads all of it into memory, so the examples that need it, or that are built around it, now use the LowMemory profile, which reads the file from disk as it is needed. - SuspiciousOnPrem needs properties only the Enterprise file has, and moves from MaxPerformance to LowMemory. - UpdateDataFile downloads the Enterprise file and took the engine default, and now names LowMemory. - MetricsOnPrem and CompareOnPrem default to LowMemory and take a profile as an optional fourth argument, so MaxPerformance can be chosen for speed where there is memory for the whole file. Their existing run methods keep their signatures and use the default. - OfflineProcessing runs on the Lite file and keeps MaxPerformance. Its comment described the low memory profile directly above the MaxPerformance line, and now says what the code does and why. The examples README says which examples use LowMemory and why. * Read data file details and run OfflineProcessing with the low memory profile DataFileHelper.getDatafileMetaData built an engine with no profile only to read the tier and dates, so UpdateDataFile loaded an Enterprise file at the target path into memory before building its own LowMemory pipeline. The helper now opens the file with LowMemory, as lacksEnterpriseProperties already does. OfflineProcessing reads the data file from its first argument or from 51DEGREES_IPI_PATH before the Lite file, and the examples README tells people to set that variable to the Enterprise file, which is over 6 GB. It now uses LowMemory, and lists MaxPerformance among the commented alternatives. Metrics and Compare now accept the profile argument in any case, and when no profile has the name given they say which names are accepted rather than stopping with a stack trace. The Metrics comment no longer quotes the speeds measured for the .NET example, which nothing in Java measures. --- examples/ip-intelligence/README.md | 7 +++ .../examples/console/CompareOnPrem.java | 52 ++++++++++++++-- .../examples/console/MetricsOnPrem.java | 56 ++++++++++++++++-- .../examples/console/OfflineProcessing.java | 11 ++-- .../examples/console/SuspiciousOnPrem.java | 11 ++-- .../examples/console/UpdateDataFile.java | 5 ++ .../examples/shared/DataFileHelper.java | 35 +++++++++++ .../examples/shared/DataFileHelperTest.java | 59 +++++++++++++++++++ 8 files changed, 216 insertions(+), 20 deletions(-) create mode 100644 examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 2d989f323..1c39b2453 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -31,6 +31,13 @@ the Enterprise data file. To use it, place it in `cxx/data/ip-intelligence` and as the first argument, or set the `51DEGREES_IPI_PATH` environment variable or system property to its path. +The Enterprise data file is over 6 GB, and the MaxPerformance profile loads all of it into +memory, so the examples built for it, or that may be given it, use the LowMemory profile, +which reads the file from disk as it is needed. Those are the Compare, Metrics, +OfflineProcessing, Suspicious and UpdateDataFile examples. The Metrics and Compare examples +take a profile as their fourth argument, and `MaxPerformance` there makes them faster on a +machine with memory to hold the whole file. + The `ExamplesWithoutSettingsTest` tests run each console example as a fresh clone does. The other tests use the Enterprise data file, `51Degrees-EnterpriseIpiV41.ipi`, when it is in `cxx/data/ip-intelligence`. When it is absent, the tests for examples that work with any data diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index dfd8ccb11..0fc3c7bbf 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -54,6 +54,10 @@ * The output CSV file contains the input truth and the result fields for easy * evaluation. * + * The performance profile can be given as the fourth argument, for example + * MaxPerformance, which makes each lookup faster where there is memory to + * hold the whole data file. + * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). * * This example needs the location properties only the Enterprise IP Intelligence data file has. @@ -125,6 +129,18 @@ public class CompareOnPrem { */ public static final String DEFAULT_OUTPUT_FILE = "compare-output.csv"; + /** + * The performance profile used unless one is given as the fourth + * argument. This example needs the Enterprise data file, which is over + * 6 GB, and LowMemory reads it from disk as it is needed, so the example + * runs without memory to hold the whole file. MaxPerformance loads the + * whole file into memory, which makes each lookup faster where there is + * memory to hold it. + */ + public static final Constants.PerformanceProfiles + DEFAULT_PERFORMANCE_PROFILE = + Constants.PerformanceProfiles.LowMemory; + /** * The properties the comparison asks the engine for, which only the * Enterprise data file has. @@ -184,6 +200,15 @@ public static void main(String[] args) throws Exception { String csvTruthFile = args.length > 1 ? args[1] : GEOIP_COMPARISON_EVIDENCE_REL_PATH; // Get the location for the output file String outputFile = args.length > 2 ? args[2] : DEFAULT_OUTPUT_FILE; + // Get the performance profile named, ignoring case, such as + // MaxPerformance, or use the default + Constants.PerformanceProfiles profile = args.length > 3 ? + DataFileHelper.findPerformanceProfile( + args[3], new PrintWriter(System.out)) : + DEFAULT_PERFORMANCE_PROFILE; + if (profile == null) { + return; + } // The free data files have none of the location properties. if (DataFileHelper.lacksEnterpriseProperties(dataFile, @@ -195,7 +220,7 @@ public static void main(String[] args) throws Exception { } try (PrintWriter writer = new PrintWriter(new FileWriter(outputFile))) { - run(dataFile, csvTruthFile, writer); + run(dataFile, csvTruthFile, profile, writer); } logger.info("Output written to '{}'", outputFile); } @@ -209,6 +234,20 @@ public static void main(String[] args) throws Exception { public static void run(String dataFile, String csvTruthFile, PrintWriter output) throws Exception { + run(dataFile, csvTruthFile, DEFAULT_PERFORMANCE_PROFILE, output); + } + + /** + * Run the example + * @param dataFile an IP Intelligence data file + * @param csvTruthFile a CSV file of truth records + * @param profile the performance profile to build the engine with + * @param output somewhere for the result CSV + */ + public static void run(String dataFile, + String csvTruthFile, + Constants.PerformanceProfiles profile, + PrintWriter output) throws Exception { logger.info("Running CompareOnPrem example"); String dataFileLocation; @@ -225,14 +264,15 @@ public static void run(String dataFile, int processors = Runtime.getRuntime().availableProcessors(); // Build a new pipeline with an on-premise IP Intelligence engine using - // the max performance profile and restricted to the properties needed. + // the performance profile given and restricted to the properties + // needed. IPIntelligenceOnPremisePipelineBuilder builder = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) - // We use the max performance profile for optimal lookup - // speed in this example. See the documentation for more detail - // on this and other configuration options. + // The profile trades memory for speed, as + // DEFAULT_PERFORMANCE_PROFILE explains. See the documentation + // for more detail on this and other configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) + .setPerformanceProfile(profile) // inhibit sharing usage for this example .setShareUsage(false) // inhibit auto-update of the data file for this example diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 3d3ecc248..4730245ec 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -32,7 +32,9 @@ * to complete. * * The sample of IP addresses used in the metrics can be adjusted as a - * parameter. + * parameter. The performance profile can be given as the fourth argument, + * for example MaxPerformance, which is much faster where there is memory to + * hold the whole data file. * * This example is primarily designed for those who are interested in * verifying the published metrics associated with 51Degrees' @@ -117,6 +119,18 @@ public class MetricsOnPrem { */ public static final String DEFAULT_OUTPUT_FILE = "metrics-output.csv"; + /** + * The performance profile used unless one is given as the fourth + * argument. This example needs the Enterprise data file, which is over + * 6 GB, and LowMemory reads it from disk as it is needed, so the example + * runs without memory to hold the whole file. MaxPerformance loads the + * whole file into memory and is much faster here, because the example + * reads the values of every profile in the file. + */ + public static final Constants.PerformanceProfiles + DEFAULT_PERFORMANCE_PROFILE = + Constants.PerformanceProfiles.LowMemory; + /** * Where to ask for a copy of the Enterprise data file, tagged with this * example. @@ -304,6 +318,15 @@ public static void main(String[] args) throws Exception { // Get the sample percentage or use the default double samplePercentage = args.length > 2 ? Double.parseDouble(args[2]) : DEFAULT_SAMPLE_PERCENTAGE; + // Get the performance profile named, ignoring case, such as + // MaxPerformance, or use the default + Constants.PerformanceProfiles profile = args.length > 3 ? + DataFileHelper.findPerformanceProfile( + args[3], new PrintWriter(System.out)) : + DEFAULT_PERFORMANCE_PROFILE; + if (profile == null) { + return; + } // The free data files have none of the location properties. if (DataFileHelper.lacksEnterpriseProperties(dataFile, @@ -319,7 +342,7 @@ public static void main(String[] args) throws Exception { // There are too many IPv6 addresses for the metrics example to // complete in a short time frame. Set to null to include all. Predicate condition = range -> range[0].contains("."); - run(dataFile, samplePercentage, 0, condition, writer); + run(dataFile, samplePercentage, 0, condition, profile, writer); } logger.info("Output written to '{}'", outputFile); } @@ -339,6 +362,27 @@ public static void run(String dataFile, int maxRanges, Predicate condition, PrintWriter output) throws Exception { + run(dataFile, samplePercentage, maxRanges, condition, + DEFAULT_PERFORMANCE_PROFILE, output); + } + + /** + * Run the example + * @param dataFile an IP Intelligence data file + * @param samplePercentage percentage of possible IP addresses to include + * in the metrics where 1 is 100% + * @param maxRanges maximum number of IP ranges to process, or 0 for all + * @param condition optional function used to determine if an IP address + * range should be included, or null for all ranges + * @param profile the performance profile to build the engine with + * @param output somewhere for the metrics CSV + */ + public static void run(String dataFile, + double samplePercentage, + int maxRanges, + Predicate condition, + Constants.PerformanceProfiles profile, + PrintWriter output) throws Exception { logger.info("Running MetricsOnPrem example"); String dataFileLocation; @@ -353,16 +397,16 @@ public static void run(String dataFile, int processors = Runtime.getRuntime().availableProcessors(); // Build the pipeline with an on-premise IP Intelligence engine using - // the max performance profile. + // the performance profile given. IPIntelligencePipelineBuilder builder = new IPIntelligencePipelineBuilder(); fiftyone.ipintelligence.IPIntelligenceOnPremisePipelineBuilder onPremiseBuilder = builder.useOnPremise(dataFileLocation, false) - // We use the max performance profile for optimal - // detection speed in this example. See the + // The profile trades memory for speed, as + // DEFAULT_PERFORMANCE_PROFILE explains. See the // documentation for more detail on this and other // configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) + .setPerformanceProfile(profile) // inhibit sharing usage for this example .setShareUsage(false) // inhibit auto-update of the data file for this example diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 540c69a66..c800a3d37 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -180,11 +180,14 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws // Performance options for IP Intelligence //.setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) //.setPerformanceProfile(Constants.PerformanceProfiles.HighPerformance) - // Low memory profile has the data streamed from disk on - // demand and is conservative in its use of memory, but - // slower because of disk access - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) //.setPerformanceProfile(Constants.PerformanceProfiles.Balanced) + // We use the low memory profile, which reads the data file + // from disk as it is needed, because this example may be + // given the Enterprise data file, which is over 6 GB, through + // its first argument or 51DEGREES_IPI_PATH. MaxPerformance + // loads the whole data file into memory, which makes each + // lookup faster where there is memory to hold it. + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) .build()) { // get the details of the IP Intelligence engine from the pipeline, diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index cfaa02ccd..8f0e5e8f2 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -157,12 +157,15 @@ public static void run(String dataFile, Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ try (Pipeline pipeline = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) - // We use the max performance profile for optimal lookup speed in this - // example. See the documentation for more detail on this and other - // configuration options. + // We use the low memory profile, because this example needs + // properties only the Enterprise data file has, and that file + // is over 6 GB. MaxPerformance loads the whole data file into + // memory, which makes each lookup faster where there is memory + // to hold it. See the documentation for more detail on this and + // other configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates // https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) // inhibit sharing usage for this example, usually this should be set to "true" .setShareUsage(false) // inhibit auto-update of the data file for this example diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index 43f14a200..a57a9b3e0 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -26,6 +26,7 @@ import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.services.DataUpdateService; import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; import fiftyone.pipeline.engines.services.OnUpdateComplete; @@ -293,6 +294,10 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // overwritten the pipeline must be configured to copy the supplied // file to a temporary file (createTempDataCopy parameter == true. .useOnPremise(dataFilename, true) + // Use the low memory profile, because this example + // downloads the Enterprise data file, which is over 6 GB, + // and MaxPerformance would load all of it into memory. + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) // pass in the update listener which has been configured // to notify when update complete .setDataUpdateService(dataUpdateService) diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index 149d5dd85..19c44616e 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -160,8 +160,18 @@ public boolean isFree() { return free; } } + + /** + * Reads the details of a data file, such as its tier and publish date. + * The data file is opened with the low memory profile, which reads + * little of it. + * @param dataFileLocation the data file to read + * @return the data file's details + * @throws Exception if the data file cannot be opened + */ public static DatafileInfo getDatafileMetaData(String dataFileLocation) throws Exception { try(IPIntelligenceOnPremiseEngine ddhe = new IPIntelligenceOnPremiseEngineBuilder() + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) .setAutoUpdate(false) .build(dataFileLocation,false)) { return new DatafileInfo((FiftyOneDataFile) ddhe.getDataFileMetaData(), @@ -311,6 +321,31 @@ public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { output.flush(); } + /** + * Finds the performance profile an example is given by name, ignoring + * case. When no profile has the name, writes the names accepted. + * @param name the name given, such as MaxPerformance + * @param output where to write when no profile has the name + * @return the profile, or null when no profile has the name + */ + public static Constants.PerformanceProfiles findPerformanceProfile( + String name, + PrintWriter output) { + List names = new ArrayList<>(); + for (Constants.PerformanceProfiles profile : + Constants.PerformanceProfiles.values()) { + if (profile.name().equalsIgnoreCase(name)) { + return profile; + } + names.add(profile.name()); + } + output.println("'" + name + "' is not a performance profile. The " + + "names accepted, in upper or lower case, are " + + joinNames(names) + "."); + output.flush(); + return null; + } + /** * Joins names as "A, B and C". */ diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java new file mode 100644 index 000000000..6d578521f --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java @@ -0,0 +1,59 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import fiftyone.pipeline.engines.Constants.PerformanceProfiles; +import org.junit.Test; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class DataFileHelperTest { + + @Test + public void findPerformanceProfile_IgnoresCase() { + StringWriter written = new StringWriter(); + assertEquals(PerformanceProfiles.MaxPerformance, + DataFileHelper.findPerformanceProfile("maxperformance", + new PrintWriter(written))); + assertEquals(PerformanceProfiles.LowMemory, + DataFileHelper.findPerformanceProfile("LOWMEMORY", + new PrintWriter(written))); + assertEquals("", written.toString()); + } + + @Test + public void findPerformanceProfile_Unknown_WritesNamesAccepted() { + StringWriter written = new StringWriter(); + assertNull(DataFileHelper.findPerformanceProfile("Fastest", + new PrintWriter(written))); + for (PerformanceProfiles profile : PerformanceProfiles.values()) { + assertTrue(written.toString(), + written.toString().contains(profile.name())); + } + } +} From 68538b8283fe344179ebd8ef11df19397fa5353d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 19:11:03 +0000 Subject: [PATCH 30/95] Build the Translation Engine from a configuration file, as .NET does (#75) * Build the Translation Engine from a configuration file, as .NET does The specification's Translation Engine configuration has a configuration file set the options that take a single value, being SourceElementDataKey, FixedLanguage and MissingTranslationBehavior, with sources and translations added in code. .NET does this by taking the builder from its service collection, where code has prepared it. Java's PipelineBuilder always constructed a new builder, TranslationEngineBuilder was not annotated so no configuration file could name it, and the behavior had no parse method, so text from a file could not set it. - PipelineBuilder.addElementBuilder gives the configuration route a builder prepared in code, used in preference to constructing one. When more than one is of the type a configuration names, the first added is used, as .NET's FiftyOneServiceProvider and the services here do. - parseToType keeps the parse method's own message, and its exception as the cause, in the error it throws. - TranslationEngineBuilder is annotated with @ElementBuilder, and its build says where sources and translations come from when either is missing. - MissingTranslationBehavior.parse accepts Original, EmptyString and FlowError and the constant names, ignoring case, and refuses anything else naming the accepted values. * Give the build method's own reason when a configured element fails When a builder's build method threw, buildFromConfiguration reported only "Failed to build element 1 using '...'", with the reason two causes down behind the InvocationTargetException that reflection adds. So a file naming the Translation Engine with no prepared builder did not say at the top of the error that sources and translations are added in code. The error now appends the build method's own message and takes its exception as the cause, as parseToType does for a refused value. --- .../CountryBuildersConfigurationTest.java | 95 +++++++++ .../core/flowelements/PipelineBuilder.java | 69 ++++++- .../PipelineBuilderPreparedBuilderTest.java | 114 +++++++++++ .../data/MissingTranslationBehavior.java | 47 ++++- .../TranslationEngineBuilder.java | 23 +++ .../data/MissingTranslationBehaviorTests.java | 71 +++++++ .../flowelements/CountrySourceElement.java | 84 ++++++++ .../CountrySourceElementBuilder.java | 51 +++++ .../TranslationEngineConfigurationTests.java | 192 ++++++++++++++++++ 9 files changed, 736 insertions(+), 10 deletions(-) create mode 100644 ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java create mode 100644 pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java create mode 100644 pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java create mode 100644 pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java create mode 100644 pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java new file mode 100644 index 000000000..b959c506c --- /dev/null +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java @@ -0,0 +1,95 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.translation; + +import fiftyone.ipintelligence.translation.flowelements.CountriesTranslationEngine; +import fiftyone.ipintelligence.translation.flowelements.CountryCodeTranslationEngine; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.translation.flowelements.TranslationEngine; +import fiftyone.pipeline.translation.flowelements.TranslationEngineBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests that a configuration file still builds the country translation + * engines with their own builders when the general Translation Engine + * builder can be named too, and a prepared one is given. + */ +public class CountryBuildersConfigurationTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + @SuppressWarnings("rawtypes") + public void countryBuilders_AreUnaffectedByAPreparedGeneralBuilder() + throws Exception { + String xml = "" + + "" + + "CountryCodeTranslationEngine" + + "" + + "" + + "CountriesTranslationEngine" + + "" + + "" + + "TranslationEngine" + + "" + + "ip" + + "" + + "" + + ""; + File file = folder.newFile("pipeline.xml"); + Files.write(file.toPath(), xml.getBytes(StandardCharsets.UTF_8)); + TranslationEngineBuilder general = + new TranslationEngineBuilder(LoggerFactory.getILoggerFactory()) + .addSource("countries.fr_FR.yml", "England: Angleterre") + .addTranslation("Country", "CountryTranslated"); + + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(general) + .buildFromConfiguration( + PipelineOptionsFactory.getOptionsFromFile(file))) { + List elements = pipeline.getFlowElements(); + assertEquals(3, elements.size()); + assertTrue(elements.get(0).getClass().getName(), + elements.get(0) instanceof CountryCodeTranslationEngine); + assertTrue(elements.get(1).getClass().getName(), + elements.get(1) instanceof CountriesTranslationEngine); + assertTrue(elements.get(2).getClass().getName(), + elements.get(2) instanceof TranslationEngine); + } + } +} diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java index fbebda96e..f10d49d1c 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java @@ -62,6 +62,13 @@ public class PipelineBuilder private final Map, Class> primitiveTypes = getPrimitiveTypeMap(); private final Set> elementBuilders; + /** + * Element builders prepared in code, which + * {@link #buildFromConfiguration(PipelineOptions)} uses in preference to + * constructing new ones. + */ + private final List preparedBuilders = new ArrayList<>(); + /** * Construct a new builder. */ @@ -80,6 +87,27 @@ public PipelineBuilder(ILoggerFactory loggerFactory) { elementBuilders = getAvailableElementBuilders(); } + /** + * Add an element builder prepared in code, which + * {@link #buildFromConfiguration(PipelineOptions)} uses for any element + * whose builder name resolves to the builder's type, in place of + * constructing a new builder. Code can then set the options a + * configuration file cannot express, such as the sources and + * translations of a Translation Engine, and the configuration sets the + * remaining options and builds the element. + *

    + * The builder's type must still be one the configuration can name, being + * annotated with {@link ElementBuilder}. When more than one prepared + * builder is of that type, the first added is used, and every element + * of that type in the configuration uses the same instance. + * @param builder the prepared builder + * @return this builder + */ + public PipelineBuilder addElementBuilder(Object builder) { + preparedBuilders.add(getNotNull(builder, "builder cannot be null")); + return this; + } + @Override public Pipeline buildFromConfiguration(PipelineOptions options) throws Exception { @@ -244,10 +272,13 @@ private void addElementToList( } - // A service collection does not exist in the builder, so try - // to construct a builder instance from the assemblies - // currently loaded. - Object builderInstance = getBuilder(builderType); + // Use a builder prepared in code if there is one, because it can + // carry options a configuration file cannot express. Otherwise + // construct a new builder. + Object builderInstance = getPreparedBuilder(builderType); + if (builderInstance == null) { + builderInstance = getBuilder(builderType); + } if (builderInstance == null) { throw new PipelineConfigurationException( "Builder '" + builderType.getName() + "' for " + @@ -353,10 +384,14 @@ private void addElementToList( try { result = buildMethod.invoke(builderInstance, parameters.toArray()); } catch (Exception e) { + // Keep the build method's own message, which says why the + // element could not be built. + Throwable cause = e.getCause() != null ? e.getCause() : e; throw new PipelineConfigurationException( "Failed to build " + elementLocation + " using " + - "'" + builderType.getName() + "'.", - e); + "'" + builderType.getName() + "'." + + (cause.getMessage() == null ? "" : " " + cause.getMessage()), + cause); } if (result == null) { throw new PipelineConfigurationException( @@ -530,6 +565,21 @@ private Object callConstructorWithServices(Constructor constructor, Class } } + /** + * Get the first builder added with {@link #addElementBuilder(Object)} + * that is an instance of the builder type provided. + * @param builderType type of builder to find + * @return the prepared builder, or null if none is of the type + */ + private Object getPreparedBuilder(Class builderType) { + for (Object builder : preparedBuilders) { + if (builderType.isInstance(builder)) { + return builder; + } + } + return null; + } + /** * Get a new instance of the builder type provided. This tries to call the * constructor with the most arguments which can be fulfilled. Potential @@ -981,10 +1031,15 @@ private Object parseToType( try { parsedValue = parse.invoke(new Object(), value); } catch (Exception e) { + // Keep the parse method's own message, which says what is wrong + // with the value. + Throwable cause = e.getCause() != null ? e.getCause() : e; throw new PipelineConfigurationException( errorTextPrefix + ". Failed to parse value " + "'" + value + "' using the static " + - "'parse' method."); + "'parse' method." + + (cause.getMessage() == null ? "" : " " + cause.getMessage()), + cause); } return parsedValue; } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java new file mode 100644 index 000000000..93b18675e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java @@ -0,0 +1,114 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.configuration; + +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.core.testclasses.flowelements.ListSplitterElement; +import fiftyone.pipeline.core.testclasses.flowelements.ListSplitterElementBuilder; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByElementBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +/** + * Tests for element builders prepared in code and given to the pipeline + * builder, which building from configuration uses in place of new ones. + */ +public class PipelineBuilderPreparedBuilderTest { + + /** + * Options naming the list splitter and setting only its maximum length, + * so the delimiter is whatever the builder already holds. + */ + private static PipelineOptions listSplitterOptions(String maxLength) { + ElementOptions element = new ElementOptions(); + element.builderName = "ListSplitterElement"; + element.buildParameters.put("MaxLength", maxLength); + PipelineOptions options = new PipelineOptions(); + options.elements.add(element); + return options; + } + + private static ListSplitterElementBuilder splitter(String delimiter) { + ListSplitterElementBuilder builder = new ListSplitterElementBuilder(); + builder.setDelimiter(delimiter); + return builder; + } + + @Test + public void preparedBuilder_KeepsItsOptionsAndTakesTheConfiguredOnes() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(splitter("|")) + .buildFromConfiguration(listSplitterOptions("50"))) { + ListSplitterElement element = + pipeline.getElement(ListSplitterElement.class); + assertEquals("|", element.getDelimiter()); + assertEquals(50, element.getMaxLength()); + } + } + + @Test + public void twoPreparedBuildersOfTheSameType_TheFirstIsUsed() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(splitter("|")) + .addElementBuilder(splitter(";")) + .buildFromConfiguration(listSplitterOptions("50"))) { + assertEquals("|", pipeline.getElement(ListSplitterElement.class) + .getDelimiter()); + } + } + + @Test + public void preparedBuilderOfAnotherType_ANewBuilderIsConstructed() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(new MultiplyByElementBuilder()) + .buildFromConfiguration(listSplitterOptions("50"))) { + // A new list splitter builder starts with a comma. + assertEquals(",", pipeline.getElement(ListSplitterElement.class) + .getDelimiter()); + } + } + + @Test + public void nullBuilder_IsRefused() { + assertThrows(IllegalArgumentException.class, + () -> new PipelineBuilder().addElementBuilder(null)); + } + + @Test + public void valueTheParseMethodRefuses_ErrorKeepsTheParseMessage() { + PipelineConfigurationException thrown = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration( + listSplitterOptions("fifty"))); + assertTrue(thrown.getMessage(), + thrown.getMessage().contains("For input string")); + } +} diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java index 7704143bb..4adb068fc 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java @@ -32,16 +32,57 @@ public enum MissingTranslationBehavior { * Use the original value if there is no translation for it. This is the * default behaviour. */ - ORIGINAL, + ORIGINAL("Original"), /** * Use an empty string if there is no translation for the value. */ - EMPTY_STRING, + EMPTY_STRING("EmptyString"), /** * Add a flow error if there is no translation for the value. The error * contains the reason there was no translation. */ - FLOW_ERROR + FLOW_ERROR("FlowError"); + + /** + * The name the specification gives the behaviour, which a configuration + * file uses. + */ + private final String specificationName; + + MissingTranslationBehavior(String specificationName) { + this.specificationName = specificationName; + } + + /** + * Reads a behaviour from text, as a configuration file gives it. The + * names the specification uses, Original, EmptyString and FlowError, and + * the names of these constants are accepted, ignoring case and any white + * space around the text. + * @param value the text to read + * @return the behaviour the text names + * @throws IllegalArgumentException if the text names no behaviour + */ + public static MissingTranslationBehavior parse(String value) { + if (value != null) { + String text = value.trim(); + for (MissingTranslationBehavior behavior : values()) { + if (behavior.specificationName.equalsIgnoreCase(text) || + behavior.name().equalsIgnoreCase(text)) { + return behavior; + } + } + } + StringBuilder accepted = new StringBuilder(); + MissingTranslationBehavior[] behaviors = values(); + for (int i = 0; i < behaviors.length; i++) { + if (i > 0) { + accepted.append(i == behaviors.length - 1 ? " or " : ", "); + } + accepted.append(behaviors[i].specificationName); + } + throw new IllegalArgumentException("'" + value + "' is not a " + + "missing translation behavior. Use " + accepted + "."); + } } diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java index 2da02839a..3c460191d 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.translation.flowelements; +import fiftyone.pipeline.annotations.ElementBuilder; import fiftyone.pipeline.translation.data.MissingTranslationBehavior; import fiftyone.pipeline.translation.data.TranslationData; import fiftyone.pipeline.translation.data.TranslationProperty; @@ -41,7 +42,16 @@ /** * Fluent builder for {@link TranslationEngine}. + *

    + * A configuration file names this builder as {@code TranslationEngine} and + * can set {@code SourceElementDataKey}, {@code FixedLanguage} and + * {@code MissingTranslationBehavior}. Sources and translations are added in + * code, so to build the engine from a configuration file, add them to a + * builder and give it to the pipeline builder with + * {@code PipelineBuilder.addElementBuilder}, which then uses it for the + * element the file names. */ +@ElementBuilder public class TranslationEngineBuilder { private final ILoggerFactory loggerFactory; @@ -171,8 +181,21 @@ private static String readUtf8(Path path) throws IOException { /** * Build a translation engine. * @return a new {@link TranslationEngine} + * @throws IllegalArgumentException if no source or no translation has + * been added */ public TranslationEngine build() { + // Checked here as well as in the engine, so that a configuration + // file naming this builder is told where sources and translations + // come from. + if (sources.isEmpty() || translationProperties.isEmpty()) { + throw new IllegalArgumentException( + "A Translation Engine needs at least one source and one " + + "translation, which are added in code with addSource and " + + "addTranslation. To build one from a configuration file, " + + "add them to a TranslationEngineBuilder and pass it to " + + "PipelineBuilder.addElementBuilder."); + } return new TranslationEngine( sourceElementDataKey, translationProperties, diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java new file mode 100644 index 000000000..2124abef3 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.data; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class MissingTranslationBehaviorTests { + + @ParameterizedTest + @CsvSource({ + "Original, ORIGINAL", + "original, ORIGINAL", + "ORIGINAL, ORIGINAL", + "EmptyString, EMPTY_STRING", + "EMPTYSTRING, EMPTY_STRING", + "empty_string, EMPTY_STRING", + "FlowError, FLOW_ERROR", + "flowerror, FLOW_ERROR", + "FLOW_ERROR, FLOW_ERROR" + }) + public void parse_SpecificationAndConstantNames_IgnoringCase( + String text, MissingTranslationBehavior expected) { + assertEquals(expected, MissingTranslationBehavior.parse(text)); + } + + @Test + public void parse_WhiteSpaceAroundTheText_IsIgnored() { + assertEquals(MissingTranslationBehavior.FLOW_ERROR, + MissingTranslationBehavior.parse(" FlowError ")); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {"Sometimes", "Empty String", "Flow-Error"}) + public void parse_AnythingElse_IsRefusedNamingTheAcceptedValues( + String text) { + IllegalArgumentException thrown = assertThrows( + IllegalArgumentException.class, + () -> MissingTranslationBehavior.parse(text)); + assertEquals("'" + text + "' is not a missing translation " + + "behavior. Use Original, EmptyString or FlowError.", + thrown.getMessage()); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java new file mode 100644 index 000000000..1ca836f1c --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java @@ -0,0 +1,84 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; + +/** + * Stands in for the element a Translation Engine reads from, giving one + * country name under the element data key {@code source}. + */ +public class CountrySourceElement + extends FlowElementBase { + + private final String country; + + CountrySourceElement(String country) { + super(LoggerFactory.getLogger(CountrySourceElement.class), + (flowData, element) -> new ElementDataBase( + LoggerFactory.getLogger(CountrySourceElement.class), + flowData) { + }); + this.country = country; + } + + @Override + public String getElementDataKey() { + return "source"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void processInternal(FlowData data) { + ElementData elementData = + data.getOrAdd(getTypedDataKey(), getDataFactory()); + elementData.put("Country", country); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java new file mode 100644 index 000000000..a4e022d4c --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java @@ -0,0 +1,51 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; + +/** + * Builds a {@link CountrySourceElement}, so that a configuration file can + * put a source element in front of a Translation Engine. + */ +@ElementBuilder +public class CountrySourceElementBuilder { + + private String country = ""; + + /** + * @param country the country name the element gives + * @return this builder + */ + public CountrySourceElementBuilder setCountry(String country) { + this.country = country; + return this; + } + + /** + * @return a new element + */ + public CountrySourceElement build() { + return new CountrySourceElement(country); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java new file mode 100644 index 000000000..75f8fde48 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java @@ -0,0 +1,192 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.translation.data.MissingTranslationBehavior; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that a configuration file builds a Translation Engine from a builder + * that code gave its sources and translations, with the file setting the + * options that take a single value, under the names the specification + * gives them. + */ +public class TranslationEngineConfigurationTests { + + @TempDir + Path folder; + + /** + * A builder prepared in code with the one source and the one translation + * a configuration file cannot give it. + */ + private static TranslationEngineBuilder prepared() { + return new TranslationEngineBuilder(LoggerFactory.getILoggerFactory()) + .addSource("countries.fr_FR.yml", "England: Angleterre") + .addTranslation("Country", "CountryTranslated"); + } + + /** + * Options read from an XML configuration file that puts a country + * source in front of a Translation Engine and sets each of the + * engine's single-value options. + */ + private PipelineOptions options(String country, String behavior) + throws Exception { + String xml = "" + + "" + + "CountrySourceElement" + + "" + country + "" + + "" + + "" + + "" + + "TranslationEngine" + + "" + + "source" + + "fr_FR" + + "" + behavior + + "" + + "" + + "" + + ""; + Path file = folder.resolve("pipeline.xml"); + Files.write(file, xml.getBytes(StandardCharsets.UTF_8)); + return PipelineOptionsFactory.getOptionsFromFile(file.toFile()); + } + + /** Processes one request and gives the translated country. */ + private static Object translate(Pipeline pipeline) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + return data.get("translation").get("CountryTranslated"); + } + } + + @Test + public void preparedBuilder_ConfigurationSetsTheOptionsAndTranslates() + throws Exception { + // No language is in the evidence, so the French translation shows + // the fixed language from the file was used, and the translation + // shows the source element data key from the file was used. + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options("England", "Original"))) { + assertEquals("Angleterre", translate(pipeline)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"Original", "original", "ORIGINAL", " Original "}) + public void originalSpellings_KeepAValueWithNoTranslation(String spelling) + throws Exception { + // The prepared builder says EmptyString, so the original value + // shows the file's setting replaced it. + TranslationEngineBuilder builder = prepared() + .setMissingTranslationBehavior( + MissingTranslationBehavior.EMPTY_STRING); + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(builder) + .buildFromConfiguration(options("Wales", spelling))) { + assertEquals("Wales", translate(pipeline)); + } + } + + @ParameterizedTest + @ValueSource(strings = { + "EmptyString", "emptystring", "EMPTY_STRING", "empty_string"}) + public void emptyStringSpellings_GiveAnEmptyString(String spelling) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options("Wales", spelling))) { + assertEquals("", translate(pipeline)); + } + } + + @ParameterizedTest + @ValueSource(strings = { + "FlowError", "flowerror", "FLOW_ERROR", "flow_error"}) + public void flowErrorSpellings_AddAFlowError(String spelling) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options("Wales", spelling))) { + AggregateException thrown = assertThrows( + AggregateException.class, () -> translate(pipeline)); + // The errors processing added are the suppressed exceptions. + StringBuilder errors = new StringBuilder(); + for (Throwable error : thrown.getSuppressed()) { + errors.append(error.getMessage()).append(' '); + } + assertTrue(errors.toString().contains( + "There was no translation found for the value 'Wales'"), + errors.toString()); + } + } + + @Test + public void unknownBehavior_IsRefusedNamingTheAcceptedValues() + throws Exception { + PipelineOptions options = options("England", "Sometimes"); + PipelineConfigurationException thrown = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options)); + assertTrue(thrown.getMessage().contains( + "'Sometimes' is not a missing translation behavior. " + + "Use Original, EmptyString or FlowError."), + thrown.getMessage()); + } + + @Test + public void noPreparedBuilder_SaysSourcesAndTranslationsComeFromCode() + throws Exception { + PipelineOptions options = options("England", "Original"); + PipelineConfigurationException thrown = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + assertTrue(thrown.getMessage().contains( + "which are added in code with addSource and addTranslation"), + thrown.getMessage()); + } +} From 5caede01dd457f3b164eb5d9186196fefc9e2c72 Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Mon, 9 Nov 2026 19:13:05 +0000 Subject: [PATCH 31/95] Add every component's drift for Drift, as the specification says (#60) Co-authored-by: James Rosewell --- .../flowelements/DeviceDataHashDefault.java | 8 ++---- .../flowelements/DeviceDataTests.java | 28 ++++++++++++++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index 081205d3d..d78e6b443 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -176,13 +176,11 @@ private AspectPropertyValue getDifferenceInternal() { * @return drift */ private AspectPropertyValue getDriftInternal() { - int result = Integer.MAX_VALUE; + int total = 0; for (ResultsHashSwig results : resultsList) { - if (results.getDrift() < result) { - result = results.getDrift(); - } + total += results.getDrift(); } - return new AspectPropertyValueDefault<>(result); + return new AspectPropertyValueDefault<>(total); } /** diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java index 0c5b36069..64dc19f0f 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java @@ -39,6 +39,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.Mockito.*; @@ -84,7 +85,32 @@ public void DeviceData_ResultsClose() throws Exception { arg -> data); flowData.close(); verify(results, times(1)).close(); - } + } + } + + /** + * Check that drift adds the drift of every native result, one per + * component, as the specification describes and the native results do. + * @throws Exception + */ + @Test + public void DeviceData_Drift_IsTheSumOfTheResults() throws Exception { + try (FlowData flowData = getWrapper().getPipeline().createFlowData()) { + DeviceDataHashDefault data = new DeviceDataHashDefault( + logger, + flowData, + getWrapper().getEngine(), + missingPropertyService); + + ResultsHashSwig hardware = mock(ResultsHashSwig.class); + when(hardware.getDrift()).thenReturn(3); + ResultsHashSwig browser = mock(ResultsHashSwig.class); + when(browser.getDrift()).thenReturn(5); + data.setResults(hardware); + data.setResults(browser); + + assertEquals(8, data.getDrift().getValue().intValue()); + } } /** From 7299688fcec249f763af4dc5d86731cac86ca761 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 19:22:23 +0000 Subject: [PATCH 32/95] Accept no Evidence for usage sharing, and give it the whole web request (#68) * Test that usage sharing accepts no Evidence and reads the web request The Pipeline with usage sharing should accept only its other elements' Evidence, the web integration should add the web request as server.http-request, usage sharing should share a header only that request holds under its rules, the tracker should key on what would be shared, a revalidation's ETag should be the one it sends, and the cloud request engine should not send the web request. * Accept no Evidence for usage sharing, and give it the whole web request The usage sharing element's filter accepted every header except Cookie, and the Pipeline's filter is the union of its elements' filters, so it accepted every header too. The client-side ETag is a hash of the Evidence in that filter, so a revalidation carrying If-None-Match hashed differently and never got a 304, and the Vary header named every header. - The element accepts no Evidence. EvidenceKeyFilterShareUsage stays as the rules for which of the values it finds are shared. - ShareUsageEvidence.find gives it every Evidence value and every value of the web request when the Evidence holds server.http-request, keeping the Evidence value for a key in both. The tracker keys on the same values, so the session tracking hash is of what would be shared. - The web integration records every value it offers, accepted or not, in a read-only WebRequestEvidence map, added as server.http-request on every request. It is a map rather than the HttpServletRequest because the usage sharing element does not depend on the servlet API. - The cloud request engine sent every Evidence value whose prefix is not header, cookie or query, so it would have sent the web request. It leaves that key out. * Test that a callback's session id and sequence still reach the sequence element With usage sharing accepting no Evidence, the session id and sequence of a client-side callback reach Evidence only if the sequence element accepts them, as its specification lists. The revalidation test now sends them, as a callback does, and the usage sharing element's session cookie test expects the element to accept nothing. * Accept the session id and sequence in the sequence element, and read a sequence sent as text The sequence element accepted no Evidence and relied on usage sharing accepting query.session-id and query.sequence, so without it every callback started a new session. It now accepts both, as the specification lists, and reads a sequence that arrives from the query string as text, which the cast to int failed on. * Test that a web pipeline accepts only what its elements use A pipeline of the sequence, set headers, JSON builder, JavaScript builder and usage sharing elements should accept neither If-None-Match nor a header no element uses, and still accept the keys its elements read. * Give the web request tracker tests an interval no runner reaches They track two requests that should count as one repeat, so the class's 50 millisecond interval let a slow runner track both. * Test that the web request holds the method and path, and the Cookie header is never shared after a prefix * Add the request line's method and path to the web request, and never share the Cookie header after a prefix --- .../CloudRequestEngineDefault.java | 13 +- .../flowelements/CloudRequestEngineTests.java | 30 ++ .../data/EvidenceKeyFilterShareUsage.java | 10 +- .../fiftyone/data/ShareUsageEvidence.java | 78 +++++ .../flowelements/SequenceElement.java | 37 ++- .../fiftyone/flowelements/ShareUsageBase.java | 39 ++- .../fiftyone/trackers/ShareUsageTracker.java | 56 ++++ .../EvidenceKeyFilterShareUsageTests.java | 16 + .../ShareUsageAcceptedEvidenceTests.java | 74 +++++ .../flowelements/ShareUsageElementTests.java | 95 +++++- .../trackers/ShareUsageTrackerTests.java | 51 +++ .../web/services/WebRequestEvidence.java | 101 ++++++ .../WebRequestEvidenceServiceCore.java | 23 +- .../web/UsageSharingEvidenceTests.java | 294 ++++++++++++++++++ 14 files changed, 884 insertions(+), 33 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index f3e00e7e7..12ccc17f2 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -63,6 +63,15 @@ public class CloudRequestEngineDefault extends AspectEngineBase implements CloudRequestEngine { + /** + * The Evidence key of the whole web request, which the web integration + * adds for usage sharing to read. It holds every header and cookie of + * the request, so it is never sent to the cloud service. The usage + * sharing package defines the same key, and this package does not + * depend on that one. + */ + private static final String HTTP_REQUEST_KEY = "server.http-request"; + private HttpClient httpClient; private String endPoint; @@ -266,7 +275,9 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { } Map getFormData(FlowData flowData) { - Map evidence = flowData.getEvidence().asKeyMap(); + Map evidence = new HashMap<>( + flowData.getEvidence().asKeyMap()); + evidence.keySet().removeIf(HTTP_REQUEST_KEY::equalsIgnoreCase); Map formData = new HashMap<>(); // Add evidence in reverse alphabetical order, excluding special keys. diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 01b82a180..80ebde301 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -464,6 +464,36 @@ public void getFormData(String name, Map evidence, String expect } } + /** + * The web integration adds the whole web request as server.http-request + * for usage sharing to read. It holds every header and cookie, so it is + * never sent to the cloud service. + */ + @Test + public void getFormData_LeavesOutTheWebRequest() throws Exception { + configureMockedClient(); + CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + + Map request = new HashMap<>(); + request.put("header.Cookie", "session=secret"); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.User-Agent", "iPhone"); + data.addEvidence("server.client-ip", "1.2.3.4"); + data.addEvidence("server.http-request", request); + + Map result = engine.getFormData(data); + assertEquals("iPhone", result.get("User-Agent")); + assertEquals("1.2.3.4", result.get("client-ip")); + assertFalse(result.containsKey("http-request"), + "The web request was sent to the cloud service."); + } + } + /** * An evidence key whose name itself holds a dot, such as * {@code query.id.usage}, must reach the cloud service under its whole diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 996694614..07a21856e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -34,7 +34,9 @@ import static fiftyone.pipeline.core.Constants.*; /** - * This filter is used by the {@link ShareUsageElement}. + * This filter is used by the {@link ShareUsageElement} to decide which of the + * values {@link ShareUsageEvidence#find} finds are shared. It is not the + * element's accepted Evidence, which is empty. * It will include anything that is: * 1) An HTTP header that is not blocked by the constructor parameter, other * than Cookie. @@ -48,10 +50,6 @@ * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email * identifiers, the query string and the path are never included. - * - * As this filter is generally inclusive, it will often cause far more - * evidence to be passed into a pipeline than the engine-specific - * filters, which tend to be based on a white list */ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { @@ -85,6 +83,7 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { * Evidence keys that are never shared, matched without regard to case as * the whole key or after a '.'. 'id.email' and 'id.salt' are a raw email * address and the salt used to make hashed email identifiers. + * 'header.cookie' is the Cookie header, which carries every cookie. * 'request-query' is the whole query string, which would undo the rules * for its parameters. 'request-path' would make each page a visitor opens * look like a new session to the tracker. @@ -92,6 +91,7 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { private static final String[] NEVER_SHARED_EVIDENCE_KEYS = { "id.email", "id.salt", + "header.cookie", "request-query", "request-path" }; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java new file mode 100644 index 000000000..a48438286 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.Evidence; + +import java.util.Map; +import java.util.TreeMap; + +/** + * The values usage sharing finds for a request. The usage sharing element + * accepts no Evidence, so that cache keys, the ETag and the Vary header never + * depend on it, and finds what to share here instead. The sharing rules of + * {@link EvidenceKeyFilterShareUsage} then decide which of these are shared. + * @see Specification + */ +public final class ShareUsageEvidence { + + /** + * The Evidence key under which the web integration adds the web request, + * as a read-only map keyed the way Evidence is keyed, such as + * header.user-agent or cookie.51D_Profile. No element accepts it. + */ + public static final String HTTP_REQUEST_KEY = "server.http-request"; + + private ShareUsageEvidence() { + } + + /** + * Every Evidence value, and every value of the web request when the + * Evidence holds it, keyed without regard to case. A key in both keeps + * the Evidence value. The web request itself is not one of the values. + * @param evidence the Evidence of the request + * @return the values found + */ + public static Map find(Evidence evidence) { + Map values = evidence.asKeyMap(); + Map found = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + Object request = null; + for (Map.Entry entry : values.entrySet()) { + if (HTTP_REQUEST_KEY.equalsIgnoreCase(entry.getKey())) { + request = entry.getValue(); + } else if (entry.getValue() != null) { + found.put(entry.getKey(), entry.getValue()); + } + } + if (request instanceof Map) { + for (Map.Entry entry : ((Map) request).entrySet()) { + if (entry.getKey() instanceof String && + entry.getValue() != null) { + found.putIfAbsent((String) entry.getKey(), entry.getValue()); + } + } + } + return found; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java index 39ca21a6f..5542e99f9 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java @@ -26,6 +26,7 @@ import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.UUID; @@ -38,6 +39,18 @@ public class SequenceElement extends FlowElementBase { + /** + * The session id and sequence of a callback, which the specification + * lists as this element's accepted Evidence. Without them the web + * integration would not add them, and every callback would start a new + * session. + */ + private final EvidenceKeyFilter acceptedEvidence = + new EvidenceKeyFilterWhitelist(Arrays.asList( + Constants.EVIDENCE_SESSIONID, + Constants.EVIDENCE_SEQUENCE), + String.CASE_INSENSITIVE_ORDER); + /** * Construct a new instance of the {@link FlowElement}. * @param logger logger instance to use for logging @@ -59,9 +72,25 @@ protected void processInternal(FlowData data) throws Exception { if (evidence.containsKey(Constants.EVIDENCE_SEQUENCE) == false) { data.addEvidence(Constants.EVIDENCE_SEQUENCE, 1); } else { - Object sequence = evidence.get(Constants.EVIDENCE_SEQUENCE); - int seq = (int) sequence; - data.addEvidence(Constants.EVIDENCE_SEQUENCE, seq + 1); + data.addEvidence( + Constants.EVIDENCE_SEQUENCE, + getSequence(evidence.get(Constants.EVIDENCE_SEQUENCE)) + 1); + } + } + + /** + * The sequence in Evidence, which is a number when an element set it and + * text when it came from the query string of a callback. Anything that is + * not a number counts as no sequence. + */ + private static int getSequence(Object sequence) { + if (sequence instanceof Number) { + return ((Number) sequence).intValue(); + } + try { + return Integer.parseInt(String.valueOf(sequence).trim()); + } catch (NumberFormatException e) { + return 0; } } @@ -72,7 +101,7 @@ public String getElementDataKey() { @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - return new EvidenceKeyFilterWhitelist(new ArrayList()); + return acceptedEvidence; } @Override diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index ddb8809ba..96b240773 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -30,6 +30,7 @@ import fiftyone.pipeline.engines.configuration.CacheConfiguration; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsageTracker; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.fiftyone.exceptions.HttpException; import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; import fiftyone.pipeline.engines.trackers.Tracker; @@ -106,12 +107,15 @@ public abstract class ShareUsageBase protected final String shareUsageUrl; /** - * Evidence key filter including the session id. + * The Evidence this element accepts, which is none, so that cache keys, + * the ETag and the Vary header never depend on it. It finds what to share + * with {@link ShareUsageEvidence#find} instead. */ - private final EvidenceKeyFilter evidenceKeyFilter; + private final EvidenceKeyFilter acceptedEvidence = + new EvidenceKeyFilterWhitelist(new ArrayList()); /** - * Evidence key filter excluding the session id + * Which of the values found are shared, leaving out the session cookie. */ private final EvidenceKeyFilter evidenceKeyFilterExclSession; @@ -425,8 +429,6 @@ protected ShareUsageBase( includedQueryStringParameters.add(Constants.EVIDENCE_SESSIONID_SUFFIX); includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX); - evidenceKeyFilter = new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, true, sessionCookieName); evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage( blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName); @@ -518,15 +520,16 @@ public void addPipeline(Pipeline pipeline) { @Override protected void processInternal(FlowData flowData){ boolean ignoreData = false; - Map evidence = flowData.getEvidence().asKeyMap(); + Map found = + ShareUsageEvidence.find(flowData.getEvidence()); logger.debug("processInternal()"); if (ignoreDataEvidenceFilter != null) { for (Map.Entry entry : ignoreDataEvidenceFilter) { String key = entry.getKey(); - if (evidence.containsKey(key)) { - if (evidence.get(key).toString().equals(entry.getValue())) { + if (found.containsKey(key)) { + if (found.get(key).toString().equals(entry.getValue())) { ignoreData = true; break; } @@ -535,13 +538,13 @@ protected void processInternal(FlowData flowData){ } if (isCanceled() == false && ignoreData == false) { - processData(flowData); + processData(flowData, found); } } @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - return evidenceKeyFilter; + return acceptedEvidence; } @Override @@ -653,8 +656,9 @@ private boolean isLocalHost(String address) throws UnknownHostException { /** * Process the supplied request data * @param data the {@link FlowData} instance that provides the evidence + * @param found the values {@link ShareUsageEvidence#find} found for it */ - private void processData(FlowData data) { + private void processData(FlowData data, Map found) { logger.debug("processData()"); if (random.nextDouble() <= sharePercentage) { // Check if the tracker will allow sharing of this data @@ -663,7 +667,7 @@ private void processData(FlowData data) { // it to the collection. try { if (evidenceCollection.offer( - getDataFromEvidence(data.getEvidence()), + getDataFromEvidence(found), addTimeout, TimeUnit.MILLISECONDS)) { logger.debug("Queued {} entries", evidenceCollection.size()); @@ -689,18 +693,19 @@ private void processData(FlowData data) { } /** - * Extract the desired data from the evidence. + * Extract the desired data from the values found. * In order to avoid problems with the evidence data being disposed before * it is sent, the data placed into a new object rather than being a * reference to the existing evidence instance. * - * @param evidence an {@link Evidence} instance that contains the data to be extracted - * @return a {@link ShareUsageData} instance populated with data from the evidence + * @param found the values {@link ShareUsageEvidence#find} found + * @return a {@link ShareUsageData} instance populated with the values + * the sharing rules allow */ - private ShareUsageData getDataFromEvidence(Evidence evidence) { + private ShareUsageData getDataFromEvidence(Map found) { ShareUsageData shareUsageData = new ShareUsageData(); - for (Map.Entry entry : evidence.asKeyMap().entrySet()) { + for (Map.Entry entry : found.entrySet()) { // Check if we can send this piece of evidence if (evidenceKeyFilterExclSession.include(entry.getKey()) == false) { continue; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index 33514287d..c6082300e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -22,12 +22,19 @@ package fiftyone.pipeline.engines.fiftyone.trackers; +import fiftyone.caching.PutCache; +import fiftyone.pipeline.core.data.DataKey; +import fiftyone.pipeline.core.data.DataKeyBuilder; +import fiftyone.pipeline.core.data.DataKeyBuilderDefault; import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.trackers.TrackerBase; +import java.io.IOException; import java.util.Date; +import java.util.Map; /** * A tracker used by share usage to attempt to avoid repeatedly sending data @@ -46,6 +53,13 @@ public class ShareUsageTracker extends TrackerBase { */ private final long interval; + /** + * The times keyed by the values usage sharing would share. The cache of + * the base class is keyed by the Evidence alone, which no longer holds + * the web request's values. + */ + private final PutCache sharedValuesCache; + /** * Construct a new instance. * @param configuration the cache configuration to use when building the @@ -61,6 +75,10 @@ public ShareUsageTracker( super(configuration); this.interval = intervalMillis; this.filter = filter; + // The base class has checked that the builder makes a PutCache. + this.sharedValuesCache = (PutCache) + configuration.getCacheBuilder().build( + null, configuration.getSize()); } @Override @@ -79,4 +97,42 @@ protected boolean match(FlowData data, Date value) { protected EvidenceKeyFilter getFilter() { return filter; } + + @Override + public Date get(FlowData data) { + return sharedValuesCache.get(getSharedValuesKey(data)); + } + + @Override + public void put(FlowData data, Date value) { + sharedValuesCache.put(getSharedValuesKey(data), value); + } + + @Override + public void close() throws IOException { + try { + sharedValuesCache.close(); + } finally { + super.close(); + } + } + + /** + * The key of the values {@link ShareUsageEvidence#find} finds that the + * filter includes, which take in the web request's values, as the + * session tracking hash is of the values that would be shared. + */ + private DataKey getSharedValuesKey(FlowData data) { + DataKeyBuilder builder = new DataKeyBuilderDefault(); + for (Map.Entry entry : + ShareUsageEvidence.find(data.getEvidence()).entrySet()) { + if (filter.include(entry.getKey())) { + builder.add( + filter.order(entry.getKey()), + entry.getKey(), + entry.getValue()); + } + } + return builder.build(); + } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java index 6e6e17147..c0c97c288 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -159,6 +159,22 @@ public void neverShared() { } } + /** + * The Cookie header carries every cookie, so it is never shared, whether + * bare or after a prefix, and without regard to case. + */ + @Test + public void cookieHeaderNeverShared() { + EvidenceKeyFilterShareUsage filter = defaults(); + for (String key : Arrays.asList( + "header.cookie", "HEADER.COOKIE", "x.header.cookie", + "server.header.cookie")) { + assertFalse(filter.include(key), key); + } + assertTrue(filter.include("header.cookies-allowed")); + assertTrue(filter.include("server.request-method")); + } + /** * A key that only ends with the same letters as a key never shared is * treated as an ordinary key. diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java new file mode 100644 index 000000000..6fae6d5ae --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The usage sharing element accepts no Evidence, so the Pipeline's accepted + * Evidence, from which cache keys, the ETag and the Vary header are made, is + * that of its other elements alone. + */ +public class ShareUsageAcceptedEvidenceTests { + + @Test + public void Pipeline_WithShareUsage_AcceptsOnlyTheOtherElementsEvidence() + throws Exception { + ShareUsageElement shareUsage = new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build(); + EmptyEngine engine = new EmptyEngineBuilder( + LoggerFactory.getILoggerFactory()) + .build(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(engine) + .addFlowElement(shareUsage) + .setAutoCloseElements(true) + .build()) { + EvidenceKeyFilter accepted = pipeline.getEvidenceKeyFilter(); + + assertTrue(accepted.include("test.value"), + "The other element's Evidence is not accepted."); + assertFalse(accepted.include("header.if-none-match"), + "If-None-Match is accepted, so a revalidation's ETag differs " + + "from the one it sends."); + assertFalse(accepted.include("header.x-test"), + "A header no element uses is accepted."); + assertFalse(accepted.include("server.http-request"), + "The web request is accepted."); + assertFalse(shareUsage.getEvidenceKeyFilter().include("header.user-agent"), + "The usage sharing element accepts Evidence."); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 260118fe6..5a7a62563 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -487,6 +487,91 @@ public void ShareUsageElement_RestrictedHeaders() throws Exception { } } + /** + * The element accepts no Evidence, so a header no other element accepts + * reaches it only through the web request the web integration adds as + * server.http-request. It shares that header under the key it would have + * as Evidence, keeps the Evidence value for a key in both places, and + * still leaves out the cookie header, a blocked header and the web + * request itself. + */ + @Test + public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() + throws Exception { + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(Collections.singletonList("x-blocked")), + new ArrayList<>(), + new ArrayList<>()); + + Map request = new HashMap<>(); + request.put("header.User-Agent", "request-ua"); + request.put("header.X-Test", "shared"); + request.put("header.Cookie", "a=1; 51D_Profile=123456"); + request.put("header.X-Blocked", "blocked"); + request.put("cookie.51D_Profile", "123456"); + request.put("cookie.other", "not-shared"); + request.put("query.secret", "not-shared"); + request.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + request.put("server.request-method", "GET"); + request.put("server.request-path", "/a%2Fb/c"); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence( + EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "user-agent", + "evidence-ua"); + data.addEvidence("server.http-request", + Collections.unmodifiableMap(request)); + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + + assertEquals("shared", xpath.evaluate( + "//header[@Name='X-Test']", doc), + "A header only the web request holds was not shared."); + NodeList userAgents = (NodeList) xpath.evaluate( + "//header[translate(@Name,'USERAGNT','useragnt')='user-agent']", + doc, XPathConstants.NODESET); + assertEquals(1, userAgents.getLength(), + "The User-Agent in both places was not shared once."); + assertEquals("evidence-ua", userAgents.item(0).getTextContent(), + "The User-Agent in both places did not keep the Evidence value."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//header[@Name='Cookie' or @Name='cookie']", doc, + XPathConstants.NODESET)).getLength(), + "The cookie header was shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//header[@Name='X-Blocked']", doc, + XPathConstants.NODESET)).getLength(), + "A blocked header was shared."); + assertEquals("123456", xpath.evaluate( + "//cookie[@Name='51D_Profile']", doc), + "A 51D_ cookie from the web request was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//cookie[@Name='other'] | //query[@Name='secret']", doc, + XPathConstants.NODESET)).getLength(), + "A cookie or query string parameter the rules leave out was shared."); + assertEquals("1.2.3.4", xpath.evaluate("//ClientIP", doc), + "The client IP from the web request was not shared."); + assertEquals("GET", xpath.evaluate( + "//server[@Name='request-method']", doc), + "The method of the request line was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//*[@Name='request-path']", doc, + XPathConstants.NODESET)).getLength(), + "The path of the request line was shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//*[@Name='http-request']", doc, + XPathConstants.NODESET)).getLength(), + "The web request itself was shared."); + } + } + @Test public void ShareUsageElement_LowPercentage() throws Exception { // Arrange @@ -1172,9 +1257,11 @@ public void ShareUsageElement_RepeatWithNewSequence_NotShared() throws Exception } /** - * The session cookie the element names is kept in evidence for the - * tracker, whatever the case of its name, and never sent. With sessions - * tracked, the same evidence from a new session is shared again. + * The session cookie the element names is read by the tracker, whatever + * the case of its name, and never sent. With sessions tracked, the same + * evidence from a new session is shared again. The element accepts no + * Evidence, so the cookie reaches the tracker from what it finds rather + * than through its accepted Evidence. */ @Test public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { @@ -1204,7 +1291,7 @@ public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { shareUsageElement.close(); // Assert - assertTrue(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); + assertFalse(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); byte[] output = connector.getBaos().toByteArray(); assertFalse(new String(output, StandardCharsets.UTF_8).contains( "session cookie"), "the session cookie is never sent"); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java index 32550b140..dff24cbae 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java @@ -184,6 +184,57 @@ public DataKey answer(InvocationOnMock invocationOnMock) throws Throwable { assertEquals(1, trackedEvents); } + /** + * The tracker's key is of the values usage sharing would share, which + * take in those of the web request. Two requests that differ only in the + * session cookie the web request holds are both tracked. + */ + @Test + public void ShareUsageTracker_WebRequest_SessionCookieTracked() { + assertEquals(2, trackWebRequests( + "cookie." + DEFAULT_SESSION_COOKIE_NAME, "session-0", "session-1")); + } + + /** + * Two requests whose web requests differ only in a value usage sharing + * leaves out, here a cookie other than the session or 51Degrees ones, + * have the same key, so only the first is tracked. + */ + @Test + public void ShareUsageTracker_WebRequest_UnsharedValueIgnored() { + assertEquals(1, trackWebRequests( + "cookie.other", "value-0", "value-1")); + } + + private int trackWebRequests(String key, String... values) { + // An interval no run of these tests comes near, so a slow runner + // cannot make the second request count as new. + long noExpiryIntervalMillis = 60 * 60 * 1000; + ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + noExpiryIntervalMillis, + evidenceKeyFilter); + int trackedEvents = 0; + for (String value : values) { + Map request = new HashMap<>(); + request.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + + "user-agent", "iPhone"); + request.put(key, value); + final Map evidenceData = new HashMap<>(); + evidenceData.put("server.http-request", request); + FlowData data = MockFlowData.createFromEvidence(evidenceData, false); + // A real FlowData makes its key from its Evidence and a filter. + when(data.generateKey(any(EvidenceKeyFilter.class))).then( + invocation -> generateKey( + (EvidenceKeyFilter) invocation.getArgument(0), + evidenceData)); + if (tracker.track(data)) { + trackedEvents++; + } + } + return trackedEvents; + } + public DataKey generateKey( EvidenceKeyFilter filter, Map evidence) { diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java new file mode 100644 index 000000000..6206d41e5 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web.services; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * A read-only view of every value the web integration offered as Evidence + * for a request, accepted or not, keyed the way Evidence is keyed, such as + * header.user-agent. It is added to Evidence as + * {@link ShareUsageEvidence#HTTP_REQUEST_KEY} for the usage sharing element, + * which accepts no Evidence and reads the request here instead. + *

    + * It is a map rather than the HttpServletRequest because the usage sharing + * element does not depend on the servlet API, and the values are copied so + * that they stay the same once the request is recycled. + */ +public class WebRequestEvidence extends AbstractMap { + + private final Map values = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private final Set> entries = + Collections.unmodifiableMap(values).entrySet(); + + WebRequestEvidence() { + } + + /** + * The view in the Evidence of the {@link FlowData}, which is added to it + * the first time. + */ + static WebRequestEvidence of(FlowData flowData) { + Object existing = flowData.getEvidence().asKeyMap() + .get(ShareUsageEvidence.HTTP_REQUEST_KEY); + if (existing instanceof WebRequestEvidence) { + return (WebRequestEvidence) existing; + } + WebRequestEvidence created = new WebRequestEvidence(); + flowData.addEvidence(ShareUsageEvidence.HTTP_REQUEST_KEY, created); + return created; + } + + void record(String key, Object value) { + if (value != null) { + values.put(key, value); + } + } + + @Override + public Set> entrySet() { + return entries; + } + + @Override + public Object get(Object key) { + return key instanceof String ? values.get(key) : null; + } + + @Override + public boolean containsKey(Object key) { + return key instanceof String && values.containsKey(key); + } + + /** + * Says how many values the view holds rather than listing them, so that + * code which prints every Evidence value, such as a debug log or an + * example page, does not print every header and cookie of the request. + */ + @Override + public String toString() { + return "web request (" + values.size() + " values)"; + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java index a12e080f5..03f853e17 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java @@ -37,7 +37,9 @@ * Service used to populate the {@link Evidence} from a * {@link HttpServletRequest} ready for it to be processed by the Pipeline. * The Pipeline's {@link EvidenceKeyFilter} is used to determine what should be - * collected from the request. + * collected from the request. Every value offered, accepted or not, is also + * added in one {@link WebRequestEvidence} for the usage sharing element, + * which accepts no Evidence. * @see Specification */ public interface WebRequestEvidenceServiceCore { @@ -57,10 +59,18 @@ public interface WebRequestEvidenceServiceCore { */ class Default implements WebRequestEvidenceServiceCore { + private static final String EVIDENCE_REQUEST_METHOD = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-method"; + + private static final String EVIDENCE_REQUEST_PATH = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-path"; + @Override public void addEvidenceFromRequest( FlowData flowData, HttpServletRequest request) { + // Added on every request, whether or not any element accepts it. + WebRequestEvidence.of(flowData); Enumeration headerNames = request.getHeaderNames(); if (headerNames != null) { @@ -116,11 +126,19 @@ public void addEvidenceFromRequest( checkAndAdd(flowData, EVIDENCE_CLIENTIP_KEY, request.getRemoteAddr()); checkAndAdd(flowData, EVIDENCE_PROTOCOL, request.getScheme()); + + // The method and the path of the request line, as sent. + checkAndAdd(flowData, EVIDENCE_REQUEST_METHOD, request.getMethod()); + checkAndAdd( + flowData, + EVIDENCE_REQUEST_PATH, + request.getRequestURI()); } /** * Check if the {@link FlowData} requires this item of evidence, and add - * it if it does. + * it if it does. It is recorded in the {@link WebRequestEvidence} + * either way. * @param flowData to check and add to * @param key the key to check and add * @param value the value to add if the check passes @@ -129,6 +147,7 @@ protected void checkAndAdd(FlowData flowData, String key, Object value) { if (flowData.getEvidenceKeyFilter().include(key)) { flowData.addEvidence(key, value); } + WebRequestEvidence.of(flowData).record(key, value); } } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java new file mode 100644 index 000000000..b06d5e8b2 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -0,0 +1,294 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The web integration gives usage sharing the whole web request, and the + * usage sharing element accepts no Evidence, so the client-side endpoints' + * ETag does not change with headers no element uses, If-None-Match among + * them. + */ +public class UsageSharingEvidenceTests { + + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + private static final String SESSION_ID = + "1f2e3d4c-5b6a-4987-8a1b-2c3d4e5f6a7b"; + + private Pipeline pipeline; + + private final WebRequestEvidenceServiceCore evidenceService = + new WebRequestEvidenceServiceCore.Default(); + + @Before + public void init() throws Exception { + pipeline = new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build()) + .setAutoCloseElements(true) + .build(); + } + + @After + public void cleanUp() throws Exception { + pipeline.close(); + } + + private static HttpServletRequest request(Map headers) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/a%2Fb/c"); + when(request.getHeaderNames()).thenReturn( + Collections.enumeration(headers.keySet())); + for (Map.Entry header : headers.entrySet()) { + when(request.getHeader(header.getKey())) + .thenReturn(header.getValue()); + } + when(request.getCookies()).thenReturn(new Cookie[]{ + new Cookie("51D_Profile", "123456")}); + // A callback from the client-side script carries its session id and + // sequence in the query string. + when(request.getParameterNames()).thenReturn( + Collections.enumeration( + Arrays.asList("q", "session-id", "sequence"))); + when(request.getParameter("q")).thenReturn("search"); + when(request.getParameter("session-id")).thenReturn(SESSION_ID); + when(request.getParameter("sequence")).thenReturn("2"); + when(request.getRemoteAddr()).thenReturn("1.2.3.4"); + when(request.getScheme()).thenReturn("https"); + return request; + } + + private static Map headers() { + Map headers = new LinkedHashMap<>(); + headers.put("User-Agent", USER_AGENT); + headers.put("X-Test", "shared"); + headers.put("Cookie", "51D_Profile=123456"); + return headers; + } + + /** + * The web request is added as server.http-request on every request, + * although no element accepts it, keyed the way Evidence is keyed. A + * header no element accepts is not added as Evidence of its own. + */ + @Test + public void addEvidenceFromRequest_AddsTheWebRequest() throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + + Map evidence = flowData.getEvidence().asKeyMap(); + assertTrue("The web request was not added.", + evidence.get("server.http-request") instanceof Map); + Map webRequest = (Map) evidence.get("server.http-request"); + assertEquals("shared", webRequest.get("header.X-Test")); + assertEquals(USER_AGENT, webRequest.get("header.User-Agent")); + assertEquals("123456", webRequest.get("cookie.51D_Profile")); + assertEquals("search", webRequest.get("query.q")); + assertEquals("1.2.3.4", webRequest.get("server.client-ip")); + assertFalse("A header no element accepts was added as Evidence.", + evidence.containsKey("header.X-Test")); + } + } + + /** + * The web request holds the method and the path of the request line as + * sent, with no decoding, beside the other parts the specification names. + */ + @Test + public void addEvidenceFromRequest_WebRequestHoldsMethodAndPath() + throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + + Map webRequest = (Map) flowData.getEvidence() + .asKeyMap().get("server.http-request"); + assertEquals("The method is missing.", + "GET", webRequest.get("server.request-method")); + assertEquals("The path is missing or was decoded.", + "/a%2Fb/c", webRequest.get("server.request-path")); + assertEquals("https", webRequest.get("header.protocol")); + } + } + + /** + * Every element a web pipeline of this repository holds accepts only the + * Evidence that can change its results, so the Pipeline accepts neither + * If-None-Match nor a header no element uses, while it still accepts the + * keys its elements read. + */ + @Test + public void WebPipeline_AcceptsOnlyWhatItsElementsUse() throws Exception { + try (Pipeline webPipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new SetHeadersElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JavaScriptBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build()) + .setAutoCloseElements(true) + .build()) { + EvidenceKeyFilter accepted = webPipeline.getEvidenceKeyFilter(); + + assertFalse("If-None-Match is accepted.", + accepted.include("header.if-none-match")); + assertFalse("A header no element uses is accepted.", + accepted.include("header.x-test")); + assertFalse("The web request is accepted.", + accepted.include("server.http-request")); + assertTrue("The JavaScript builder's host is not accepted.", + accepted.include("header.host")); + assertTrue("The sequence element's session id is not accepted.", + accepted.include("query.session-id")); + } + } + + /** + * The sequence element accepts the session id and sequence of a + * callback, so they reach Evidence although usage sharing accepts + * nothing, and the sequence counts on from the callback's. + */ + @Test + public void Callback_SessionAndSequence_CarriedOn() throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + flowData.process(); + + Map evidence = flowData.getEvidence().asKeyMap(); + assertEquals("The callback's session id was not kept.", + SESSION_ID, evidence.get("query.session-id")); + assertEquals("The sequence did not count on from the callback's.", + 3, evidence.get("query.sequence")); + assertTrue("Processing recorded errors: " + flowData.getErrors(), + flowData.getErrors().isEmpty()); + } + } + + /** + * A browser revalidates with the ETag it was given in If-None-Match. + * That header changes no result, so the ETag of the revalidation is the + * one it sends, and it gets a 304. + */ + @Test + public void Revalidation_SameETag_Gets304() throws Exception { + Map revalidationHeaders = headers(); + HttpServletRequest first = request(headers()); + try (FlowData firstData = pipeline.createFlowData(); + FlowData revalidationData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest(firstData, first); + firstData.process(); + + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(first)).thenReturn(firstData); + ClientsidePropertyServiceCore service = + new ClientsidePropertyServiceCore.Default(provider, pipeline); + + HttpServletResponse firstResponse = response(); + service.serveJson(first, firstResponse); + ArgumentCaptor etag = ArgumentCaptor.forClass(String.class); + verify(firstResponse).setHeader(eq("ETag"), etag.capture()); + + revalidationHeaders.put("If-None-Match", etag.getValue()); + HttpServletRequest revalidation = request(revalidationHeaders); + evidenceService.addEvidenceFromRequest( + revalidationData, revalidation); + revalidationData.process(); + when(provider.getFlowData(revalidation)) + .thenReturn(revalidationData); + + assertEquals("The revalidation's ETag differs from the one it sends.", + etag.getValue(), + Integer.toString(revalidationData.generateKey( + pipeline.getEvidenceKeyFilter()).hashCode())); + + HttpServletResponse revalidationResponse = response(); + service.serveJson(revalidation, revalidationResponse); + verify(revalidationResponse).setStatus(304); + verify(revalidationResponse, never()).setStatus(200); + verify(revalidationResponse, never()).setContentType(anyString()); + } + } + + private static HttpServletResponse response() throws Exception { + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getWriter()).thenReturn( + new PrintWriter(new StringWriter())); + return response; + } +} From 65bbf25704058ba673a7c5906b4cd0ced604cc45 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:23:27 +0000 Subject: [PATCH 33/95] Test that a cloud engine whose product the resource key lacks builds and logs once A resource key can be valid and include no product that holds a cloud engine's properties. The Device Detection, IP Intelligence and Location cloud engines are tested to build with empty property metadata, to log the missing product once, to log and throw nothing when processing responses that hold nothing for them, and to report a property read as missing with ProductNotAccessibleWithResourceKey. The base class tests now expect a missing product to build with empty metadata and log once, keep the failure when the accessible properties are not loaded, and check that a property keyed engine gives no profiles. The Location cloud module gets Mockito as a test dependency for its first tests. --- .../flowelements/ProductMissingTests.java | 181 +++++++++++++++++ .../flowelements/ProductMissingTests.java | 182 +++++++++++++++++ location/geo-location.cloud/pom.xml | 4 + .../GeoLocationCloudEngineTests.java | 184 ++++++++++++++++++ .../CloudAspectEngineBaseTests.java | 151 ++++++++++++-- 5 files changed, 683 insertions(+), 19 deletions(-) create mode 100644 device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java create mode 100644 ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java create mode 100644 location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java new file mode 100644 index 000000000..37c1ec6d5 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -0,0 +1,181 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A resource key can be valid and include no product that holds the + * Device Detection properties. The engine then builds with empty property + * metadata, logs the missing product once, and logs or throws nothing on + * each request. + */ +public class ProductMissingTests { + + private Logger logger; + private ILoggerFactory loggerFactory; + + @Before + public void init() { + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + assertNotNull(engine.getProperties()); + assertEquals(0, engine.getProperties().size()); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).contains("'device'")); + } + } + + @Test + public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + for (int i = 0; i < 3; i++) { + process(engine); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + } + } + + /** + * Reading a property behaves as for any property the response does not + * carry. + */ + @Test + public void ProductMissing_PropertyIsMissing() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud device = process(engine); + try { + device.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + MissingPropertyReason.ProductNotAccessibleWithResourceKey, + e.getReason()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the accessible products + * of a resource key that includes IP Intelligence alone. + */ + private DeviceDetectionCloudEngine buildEngine() throws Exception { + DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("ip", product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process a cloud response that holds nothing for the Device Detection + * product. + */ + private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn( + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, device); + return device; + } + + /** + * Every warning and error logged, at any overload. + */ + private List warningsAndErrors() { + List logged = new ArrayList<>(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + String level = invocation.getMethod().getName(); + if (level.equals("warn") || level.equals("error")) { + logged.add(level + " " + + Arrays.toString(invocation.getArguments())); + } + } + return Collections.unmodifiableList(logged); + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java new file mode 100644 index 000000000..df07df779 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -0,0 +1,182 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A resource key can be valid and include no product that holds the + * IP Intelligence properties. The engine then builds with empty property + * metadata, logs the missing product once, and logs or throws nothing on + * each request. + */ +public class ProductMissingTests { + + private Logger logger; + private ILoggerFactory loggerFactory; + + @Before + public void init() { + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + assertNotNull(engine.getProperties()); + assertEquals(0, engine.getProperties().size()); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).contains("'ip'")); + } + } + + @Test + public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + for (int i = 0; i < 3; i++) { + process(engine); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + } + } + + /** + * Reading a property behaves as for any property the response does not + * carry. + */ + @Test + public void ProductMissing_PropertyIsMissing() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + IPIntelligenceDataCloud data = process(engine); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + MissingPropertyReason.ProductNotAccessibleWithResourceKey, + e.getReason()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the accessible products + * of a resource key that includes Device Detection alone. + */ + private IPIntelligenceCloudEngine buildEngine() throws Exception { + IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("hardwarevendor", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("device", product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process a cloud response that holds nothing for the IP Intelligence + * product. + */ + private IPIntelligenceDataCloud process(IPIntelligenceCloudEngine engine) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn( + "{ 'device': { 'hardwarevendor': 'Example' }, " + + "'javascriptProperties': [] }"); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + loggerFactory.getLogger( + IPIntelligenceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, data); + return data; + } + + /** + * Every warning and error logged, at any overload. + */ + private List warningsAndErrors() { + List logged = new ArrayList<>(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + String level = invocation.getMethod().getName(); + if (level.equals("warn") || level.equals("error")) { + logged.add(level + " " + + Arrays.toString(invocation.getArguments())); + } + } + return Collections.unmodifiableList(logged); + } +} diff --git a/location/geo-location.cloud/pom.xml b/location/geo-location.cloud/pom.xml index 4195dafd5..17679707e 100644 --- a/location/geo-location.cloud/pom.xml +++ b/location/geo-location.cloud/pom.xml @@ -55,5 +55,9 @@ junit junit + + org.mockito + mockito-core + diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java new file mode 100644 index 000000000..66f40b96e --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -0,0 +1,184 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.geolocation.data.CloudGeoDataDefault; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A resource key can be valid and include no product that holds the + * Location properties. The engine then builds with empty property + * metadata, logs the missing product once, and logs or throws nothing on + * each request. + */ +public class GeoLocationCloudEngineTests { + + private Logger logger; + private ILoggerFactory loggerFactory; + + @Before + public void init() { + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + assertNotNull(engine.getProperties()); + assertEquals(0, engine.getProperties().size()); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).contains("'location'")); + } + } + + @Test + public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + for (int i = 0; i < 3; i++) { + process(engine); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + } + } + + /** + * Reading a property behaves as for any property the response does not + * carry. + */ + @Test + public void ProductMissing_PropertyIsMissing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine); + try { + data.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + MissingPropertyReason.ProductNotAccessibleWithResourceKey, + e.getReason()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the accessible products + * of a resource key that includes IP Intelligence alone. + */ + private GeoLocationCloudEngine buildEngine() throws Exception { + GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("ip", product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process a cloud response that holds nothing for the Location + * product. + */ + private CloudGeoData process(GeoLocationCloudEngine engine) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn( + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + CloudGeoData data = new CloudGeoDataDefault( + loggerFactory.getLogger(CloudGeoData.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, data); + return data; + } + + /** + * Every warning and error logged, at any overload. + */ + private List warningsAndErrors() { + List logged = new ArrayList<>(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + String level = invocation.getMethod().getName(); + if (level.equals("warn") || level.equals("error")) { + logged.add(level + " " + + Arrays.toString(invocation.getArguments())); + } + } + return Collections.unmodifiableList(logged); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 514a9369e..f0f4ac5bb 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Map; +import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -57,6 +58,7 @@ import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectDataBase; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.MultiProfileData; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.flowelements.AspectEngineBase; @@ -107,6 +109,55 @@ protected void processCloudEngine(FlowData data, TestData aspectData, String jso } } + public class TestProfiles extends AspectDataBase + implements MultiProfileData { + private final List profiles = new ArrayList<>(); + + public TestProfiles( + Logger logger, + FlowData flowData, + AspectEngine engine) { + super(logger, flowData, engine); + } + + @Override + public List getProfiles() { + return profiles; + } + + @Override + public void addProfile(TestData profile) { + profiles.add(profile); + } + } + + private class TestKeyedInstance + extends PropertyKeyedCloudEngineBase { + public TestKeyedInstance() { + super(loggerFactory.getLogger("TestKeyedInstance"), new ElementDataFactory() { + @Override + public TestProfiles create(FlowData flowData, FlowElement flowElement) { + return new TestProfiles(null, flowData, (AspectEngine) flowElement); + } + }); + } + + @Override + public String getElementDataKey() { + return "test"; + } + + @Override + protected TestData createProfileData(FlowData flowData) { + return new TestData(null, flowData, this); + } + + @Override + protected void unmanagedResourcesCleanup() { + + } + } + private class TestRequestEngine extends AspectEngineBase implements CloudRequestEngine { @@ -227,31 +278,26 @@ public void loadProperties() throws Exception { } /** - * Test that an exception is thrown by the Properties auto property if - * the cloud request engine returns no properties. - * @throws Exception + * A resource key that includes no products at all includes none for this + * engine, so the Pipeline builds, the engine has empty property metadata + * and the missing product is logged once. + * @throws Exception */ @Test public void loadProperties_noProperties() throws Exception { - try { - createPipeline(); - fail("RuntimeException should be thrown"); - } catch (RuntimeException ex){ - assertEquals( - String.format( - ExceptionFailedToLoadProperties, - "test", - "test"), - ex.getMessage() - ); - } + createPipeline(); + + assertEquals(0, engine.getProperties().size()); + assertProductMissingLoggedOnce(); maxErrors = 1; } - + /** - * Test that an exception is thrown by the Properties auto property if - * the cloud engine only returns properties for other engines. - * @throws Exception + * A resource key whose products hold only other engines' properties + * does not include this engine's product. The Pipeline builds, the engine + * has empty property metadata, and the missing product is logged once + * however often the metadata is read. + * @throws Exception */ @Test public void loadProperties_wrongProperties() throws Exception { @@ -263,6 +309,24 @@ public void loadProperties_wrongProperties() throws Exception { devicePropertyData.properties = properties; propertiesReturnedByRequestEngine.put("test2", devicePropertyData); + createPipeline(); + + for (int i = 0; i < 3; i++) { + assertEquals(0, engine.getProperties().size()); + } + assertProductMissingLoggedOnce(); + maxErrors = 1; + } + + /** + * Accessible properties that have not been loaded are not a missing + * product, so reading the property metadata still fails rather than + * keeping an empty result. + * @throws Exception + */ + @Test + public void loadProperties_notLoaded() throws Exception { + propertiesReturnedByRequestEngine = null; try { createPipeline(); fail("RuntimeException should be thrown"); @@ -275,10 +339,59 @@ public void loadProperties_wrongProperties() throws Exception { ex.getMessage() ); } + maxErrors = 1; + } + /** + * A property keyed engine, such as the hardware profile engine, whose + * product the resource key does not include builds, gives no profiles + * for a response that holds nothing for it, and logs nothing on each + * request. + * @throws Exception + */ + @Test + public void propertyKeyed_productMissing_noProfiles() throws Exception { + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData otherProduct = new AccessiblePropertyMetaData.ProductMetaData(); + otherProduct.properties = properties; + propertiesReturnedByRequestEngine.put("test2", otherProduct); + TestKeyedInstance keyed = new TestKeyedInstance(); + requestEngine = new TestRequestEngine(); + requestEngine.setPublicProperties(propertiesReturnedByRequestEngine); + pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(requestEngine) + .addFlowElement(keyed) + .build(); + + for (int i = 0; i < 3; i++) { + TestProfiles profiles = new TestProfiles( + null, + Mockito.mock(FlowData.class), + keyed); + keyed.processCloudEngine( + Mockito.mock(FlowData.class), + profiles, + "{ \"test2\": { \"ismobile\": true } }"); + assertEquals(0, profiles.getProfiles().size()); + } + assertEquals(0, keyed.getProperties().size()); + assertProductMissingLoggedOnce(); maxErrors = 1; } + private void assertProductMissingLoggedOnce() { + List errors = new ArrayList<>(); + List warnings = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + errors.addAll(logger.errorsLogged); + warnings.addAll(logger.warningsLogged); + } + assertEquals(1, errors.size(), errors.toString()); + assertTrue(errors.get(0).contains("'test'"), errors.get(0)); + assertEquals(0, warnings.size(), warnings.toString()); + } + /** * Test loading sub-property meta data where a cloud aspect engine * has nested properties. E.g. the cloud property keyed engine. From 50c4c42562d2516e3b95a206cd8d28fd8e2aa05e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:25:51 +0000 Subject: [PATCH 34/95] Build a cloud engine whose product the resource key lacks, log that once, and process quietly A resource key can be valid and include no product that holds a cloud engine's properties. The Device Detection, IP Intelligence and Location cloud engines then left their property metadata null, logged two errors, and threw on every request when getJSONObject found no part of the response for them. The engines built on CloudAspectEngineBase, such as the hardware profile engines, threw when their metadata was read. CloudAspectEngineBase now has findProduct, which returns the engine's product from the accessible products, or an empty product when the resource key does not include it, logging that once for each engine instance. getElementJson returns the engine's part of the response, or an empty object when the product is missing. The three engines, the base class's own metadata loading and the property keyed base use them, so each builds with empty property metadata, logs one error, and processes a response with nothing for it without logging or throwing. Reading a property then throws PropertyMissingException with ProductNotAccessibleWithResourceKey, as for any property the response does not carry. Accessible products that were never loaded still fail as before. The Location engine's failure message named the Device Detection engine, and now names the Location engine. --- .../DeviceDetectionCloudEngine.java | 14 ++-- .../IPIntelligenceCloudEngine.java | 14 ++-- .../flowelements/GeoLocationCloudEngine.java | 16 ++-- .../cloudrequestengine/Constants.java | 5 ++ .../flowelements/CloudAspectEngineBase.java | 79 +++++++++++++++++-- .../PropertyKeyedCloudEngineBase.java | 17 +++- 6 files changed, 110 insertions(+), 35 deletions(-) diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index b6ea1aa59..2e6e2f979 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -111,7 +111,7 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); - JSONObject deviceObj = jsonObj.getJSONObject("device"); + JSONObject deviceObj = getElementJson(jsonObj); Map deviceMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); @@ -205,17 +205,15 @@ public void addPipeline(Pipeline pipeline) { * {@link #aspectProperties} */ private boolean loadAspectProperties(CloudRequestEngine engine) { - Map map = - engine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(engine.getPublicProperties()); - if (map != null && - map.size() > 0 && - map.containsKey(getElementDataKey())) { + if (product != null) { aspectProperties = new ArrayList<>(); - dataSourceTier = map.get(getElementDataKey()).dataTier; + dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : - map.get(getElementDataKey()).properties) { + product.properties) { AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, this, diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index ce84df4d8..da5039234 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -123,7 +123,7 @@ protected void processEngine(FlowData data, IPIntelligenceDataCloud aspectData) // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); - JSONObject ipiObj = jsonObj.getJSONObject("ip"); + JSONObject ipiObj = getElementJson(jsonObj); Map propertyMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); @@ -239,18 +239,16 @@ public void addPipeline(Pipeline pipeline) { * {@link #aspectProperties} */ private boolean loadAspectProperties(CloudRequestEngine engine) { - Map map = - engine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(engine.getPublicProperties()); - if (map != null && - map.size() > 0 && - map.containsKey(getElementDataKey())) { + if (product != null) { aspectProperties = new ArrayList<>(); weightedPropertyNames.clear(); - dataSourceTier = map.get(getElementDataKey()).dataTier; + dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : - map.get(getElementDataKey()).properties) { + product.properties) { if (item.type != null && item.type.startsWith("Weighted")) { weightedPropertyNames.add(item.name.toLowerCase()); } diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 098faac53..7fb48e1cd 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -107,7 +107,7 @@ protected void processEngine(FlowData data, CloudGeoData aspectData) { String json = requestData.getJsonResponse(); JSONObject jsonObj = new JSONObject(json); - JSONObject geoObj = jsonObj.getJSONObject(getElementDataKey()); + JSONObject geoObj = getElementJson(jsonObj); Map methodData = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); @@ -177,16 +177,14 @@ public void addPipeline(Pipeline pipeline) { } private boolean loadAspectProperties(CloudRequestEngine engine) { - Map map = - engine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(engine.getPublicProperties()); - if (map != null && - map.size() > 0 && - map.containsKey(getElementDataKey())) { + if (product != null) { properties = new ArrayList<>(); - dataSourceTier = map.get(getElementDataKey()).dataTier; + dataSourceTier = product.dataTier; - for (AccessiblePropertyMetaData.PropertyMetaData item : map.get(getElementDataKey()).properties) + for (AccessiblePropertyMetaData.PropertyMetaData item : product.properties) { AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, @@ -205,7 +203,7 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { } else { logger.error("Aspect properties could not be loaded for" + - " the Device Detection cloud engine", this); + " the Location cloud engine", this); return false; } } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index 593c618d6..613dcff5a 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -48,6 +48,11 @@ public class Messages { + "see our explainer: " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; + public static final String MessageProductNotInResourceKey = + "The resource key does not include the '%s' product, so " + + "%s has no properties and its results hold no values. " + + "Use a resource key that includes the product to get them."; + public static final String ProcessCloudEngineNotImplemented = "This method should be overridden in the class derived from " + "CloudAspectEngineBase class. The implementation should use " diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index 70f10ea2a..a828a9485 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionFailedToLoadProperties; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.MessageProductNotInResourceKey; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ProcessCloudEngineNotImplemented; import fiftyone.pipeline.cloudrequestengine.NotImplementedException; @@ -43,12 +44,14 @@ import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; import fiftyone.pipeline.util.Types; +import org.json.JSONObject; import org.slf4j.Logger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** * Base class for 51Degrees Cloud Aspect Engines @@ -109,6 +112,8 @@ public CloudRequestEngine getInstance() throws PipelineConfigurationException { private volatile List aspectProperties; private String dataSourceTier; private RequestEngineAccessor requestEngine; + private volatile boolean productMissing = false; + private final AtomicBoolean productMissingLogged = new AtomicBoolean(false); /** * Construct a new instance of the {@link CloudAspectEngineBase}. @@ -192,17 +197,15 @@ public TypedKey getTypedDataKey() { */ private boolean loadAspectProperties() { CloudRequestEngine requestEngine = getRequestEngine().getInstance(); - Map map = - requestEngine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(requestEngine.getPublicProperties()); - if(map != null && - map.size() > 0 && - map.containsKey(this.getElementDataKey())) { + if(product != null) { List properties = new ArrayList<>(); - dataSourceTier = map.get(this.getElementDataKey()).dataTier; + dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : - map.get(this.getElementDataKey()).properties) { + product.properties) { AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, this, @@ -224,6 +227,68 @@ private boolean loadAspectProperties() { } } + /** + * Find this engine's product among the products the Cloud Request Engine + * reports for the resource key. A valid resource key can include no + * product that holds this engine's properties. That is logged once for + * this engine, and an empty product is returned, so the engine builds + * with empty property metadata and logs nothing on each request. + * @param products the products from + * {@link CloudRequestEngine#getPublicProperties()}, or null if they have + * not been loaded + * @return the product, an empty product if the resource key does not + * include it, or null if the products have not been loaded + */ + protected AccessiblePropertyMetaData.ProductMetaData findProduct( + Map products) { + if (products == null) { + return null; + } + AccessiblePropertyMetaData.ProductMetaData product = + products.get(getElementDataKey()); + if (product != null) { + productMissing = false; + return product; + } + productMissing = true; + if (productMissingLogged.compareAndSet(false, true)) { + logger.error(String.format( + MessageProductNotInResourceKey, + getElementDataKey(), + getClass().getSimpleName())); + } + AccessiblePropertyMetaData.ProductMetaData empty = + new AccessiblePropertyMetaData.ProductMetaData(); + empty.properties = new ArrayList<>(); + return empty; + } + + /** + * True when the resource key does not include this engine's product, so + * the cloud response holds nothing for this engine. + * @return true if the product is missing + */ + protected boolean isProductMissing() { + return productMissing; + } + + /** + * This engine's part of the cloud response. When the resource key does + * not include this engine's product the response has no such part, and + * an empty object is returned, so the element data holds no values. + * @param response the whole cloud response + * @return this engine's part of the response + * @throws org.json.JSONException if the product is included and the + * response has no part for this engine + */ + protected JSONObject getElementJson(JSONObject response) { + if (productMissing) { + JSONObject element = response.optJSONObject(getElementDataKey()); + return element == null ? new JSONObject() : element; + } + return response.getJSONObject(getElementDataKey()); + } + private List loadElementProperties( List itemProperties) { List result = null; diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java index 414743370..2f8dbfe08 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java @@ -33,6 +33,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.MultiProfileData; import fiftyone.pipeline.util.Types; +import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -71,10 +72,20 @@ protected void processCloudEngine(FlowData data, TData aspectData, String json) // Extract data from json to the aspectData instance. JSONObject map = new JSONObject(json); + // Load the property meta-data, which finds whether the resource key + // includes this engine's product. + getProperties(); // Access the data relating to this engine. - JSONObject propertyKeyed = map.getJSONObject(getElementDataKey()); - // Access the 'Profiles' property - for (Object entry : propertyKeyed.getJSONArray("profiles")) { + JSONObject propertyKeyed = getElementJson(map); + // Access the 'Profiles' property. The response holds none for a + // product the resource key does not include. + JSONArray profiles = isProductMissing() ? + propertyKeyed.optJSONArray("profiles") : + propertyKeyed.getJSONArray("profiles"); + if (profiles == null) { + return; + } + for (Object entry : profiles) { // Iterate through the devices, parsing each one and // adding it to the result. From c72baaceff8b8eb50ed51afc5ffca2f31d7b1871 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:59:31 +0000 Subject: [PATCH 35/95] Test that the Cloud Request Engine discovers when it is built, and retries only after a failure that can pass CloudRequestEngineDiscoveryTests uses a stand-in for the HTTP client that counts the requests to each endpoint. A build fetches the accessible properties and the evidence keys once each and processing fetches neither again. A 4xx answer to either fails the build with a CloudRequestException carrying the status, once both requests are made, also when the other request failed in a way that can pass. A 5xx answer, a body that cannot be read, a refused connection and a timeout each complete the build with one warning. A list that arrived is kept, processing fetches only what is missing and sends no data request until both are known, the Pipeline's Evidence filter sees the keys a retry fetched, and the configured timeout applies to the discovery requests. The existing tests that expected getPublicProperties and getEvidenceKeyFilter to fetch and throw when the server was unavailable, and a 200 answer holding errors to throw, now expect the build to complete with one warning. The cloud aspect engines are tested to build without logging when the accessible products are not known yet, and to read their metadata once they are. --- .../flowelements/ProductMissingTests.java | 31 ++ .../flowelements/ProductMissingTests.java | 31 ++ .../GeoLocationCloudEngineTests.java | 32 ++ .../CloudAspectEngineBaseTests.java | 36 +- .../CloudRequestEngineDiscoveryTests.java | 402 ++++++++++++++++++ .../flowelements/CloudRequestEngineTests.java | 98 +++-- 6 files changed, 568 insertions(+), 62 deletions(-) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java index 37c1ec6d5..32eb7356a 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -114,6 +114,37 @@ public void ProductMissing_PropertyIsMissing() throws Exception { } } + /** + * After a failure at build that can pass, the Cloud Request Engine does + * not know the accessible products yet. The engine builds without + * logging anything, and reads its metadata once the products are known + * rather than keeping none. + */ + @Test + public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("hardwarevendor", "String", "cat", null)); + Map products = new HashMap<>(); + products.put("device", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(null, products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + try (DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build()) { + engine.addPipeline(pipeline); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 0, logged.size()); + + assertEquals(1, engine.getProperties().size()); + assertEquals("hardwarevendor", engine.getProperties().get(0).getName()); + } + } + /** * An engine whose Cloud Request Engine reports the accessible products * of a resource key that includes IP Intelligence alone. diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index df07df779..3249c2ab1 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -114,6 +114,37 @@ public void ProductMissing_PropertyIsMissing() throws Exception { } } + /** + * After a failure at build that can pass, the Cloud Request Engine does + * not know the accessible products yet. The engine builds without + * logging anything, and reads its metadata once the products are known + * rather than keeping none. + */ + @Test + public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map products = new HashMap<>(); + products.put("ip", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(null, products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + try (IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build()) { + engine.addPipeline(pipeline); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 0, logged.size()); + + assertEquals(1, engine.getProperties().size()); + assertEquals("registeredname", engine.getProperties().get(0).getName()); + } + } + /** * An engine whose Cloud Request Engine reports the accessible products * of a resource key that includes Device Detection alone. diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java index 66f40b96e..1520a46e9 100644 --- a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -116,6 +116,38 @@ public void ProductMissing_PropertyIsMissing() throws Exception { } } + /** + * After a failure at build that can pass, the Cloud Request Engine does + * not know the accessible products yet. The engine builds without + * logging anything, and reads its metadata once the products are known + * rather than keeping none. + */ + @Test + public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("town", "String", "cat", null)); + Map products = new HashMap<>(); + products.put("location", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(null, products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + try (GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees)) { + engine.addPipeline(pipeline); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 0, logged.size()); + + assertEquals(1, engine.getProperties().size()); + assertEquals("town", engine.getProperties().get(0).getName()); + } + } + /** * An engine whose Cloud Request Engine reports the accessible products * of a resource key that includes IP Intelligence alone. diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index f0f4ac5bb..3ec05bb44 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -22,7 +22,6 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; -import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionFailedToLoadProperties; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -319,27 +318,30 @@ public void loadProperties_wrongProperties() throws Exception { } /** - * Accessible properties that have not been loaded are not a missing - * product, so reading the property metadata still fails rather than - * keeping an empty result. + * Accessible properties the Cloud Request Engine has not fetched yet, + * after a failure at build that can pass, are not a missing product. + * The Pipeline builds, the metadata is empty and nothing is logged, and + * the metadata is read again once the properties are known rather than + * kept empty. * @throws Exception */ @Test public void loadProperties_notLoaded() throws Exception { propertiesReturnedByRequestEngine = null; - try { - createPipeline(); - fail("RuntimeException should be thrown"); - } catch (RuntimeException ex){ - assertEquals( - String.format( - ExceptionFailedToLoadProperties, - "test", - "test"), - ex.getMessage() - ); - } - maxErrors = 1; + createPipeline(); + + assertEquals(0, engine.getProperties().size()); + + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + Map products = new HashMap<>(); + products.put("test", product); + requestEngine.setPublicProperties(products); + + assertEquals(1, engine.getProperties().size()); + assertTrue(propertiesContainName(engine.getProperties(), "ismobile")); } /** diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java new file mode 100644 index 000000000..1ae9bb3e6 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -0,0 +1,402 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.common.testhelpers.TestLogger; +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The Cloud Request Engine fetches its accessible properties and accepted + * Evidence keys when it is built. A 4xx answer to either fails the build. + * Any other failure is logged as one warning, the build completes, and + * processing fetches what is missing before it sends a data request. + */ +public class CloudRequestEngineDiscoveryTests { + + private static final String PROPERTIES = "accessibleproperties"; + private static final String EVIDENCE_KEYS = "evidencekeys"; + private static final String DATA = "json"; + + private static final String PROPERTIES_BODY = + "{\"Products\": {\"device\": {\"DataTier\": \"tier\", " + + "\"Properties\": [{\"Name\": \"IsMobile\", \"Type\": \"Boolean\"," + + " \"Category\": \"Device\"}]}}}"; + private static final String EVIDENCE_KEYS_BODY = + "[\"query.User-Agent\", \"header.User-Agent\"]"; + private static final String DATA_BODY = "{\"device\": {\"ismobile\": true}}"; + + /** + * A failure that is not an HTTP status, such as a refused connection. + */ + private static final int UNREACHABLE = -1; + + /** + * A failure where the request hangs until the timeout. + */ + private static final int TIMES_OUT = -2; + + /** + * A 200 answer whose body cannot be read. + */ + private static final int UNREADABLE = -3; + + private static final int TIMEOUT_SECONDS = 3; + + private TestLoggerFactory loggerFactory; + private HttpClient httpClient; + private final Map requests = new HashMap<>(); + private final Map answers = new HashMap<>(); + private final List connections = new ArrayList<>(); + + @BeforeEach + public void init() throws IOException { + ILoggerFactory internal = mock(ILoggerFactory.class); + when(internal.getLogger(anyString())).thenReturn(mock(Logger.class)); + loggerFactory = new TestLoggerFactory(internal); + answers.put(PROPERTIES, 200); + answers.put(EVIDENCE_KEYS, 200); + answers.put(DATA, 200); + + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenAnswer(invocation -> { + URL url = invocation.getArgument(0); + String name = endpoint(url); + requests.merge(name, 1, Integer::sum); + int answer = answers.get(name); + if (answer == UNREACHABLE) { + throw new ConnectException("Connection refused"); + } + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getURL()).thenReturn(url); + when(connection.getResponseCode()) + .thenReturn(answer < 0 ? 200 : answer); + when(connection.getHeaderFields()) + .thenReturn(new HashMap>()); + connections.add(connection); + return connection; + }); + when(httpClient.getResponseString( + any(HttpURLConnection.class), anyMap())) + .thenAnswer(invocation -> body(invocation.getArgument(0))); + when(httpClient.getResponseString(any(HttpURLConnection.class))) + .thenAnswer(invocation -> body(invocation.getArgument(0))); + when(httpClient.postData( + any(HttpURLConnection.class), anyMap(), any(byte[].class))) + .thenAnswer(invocation -> body(invocation.getArgument(0))); + } + + /** + * In the normal case the built engine is ready. Both lists are fetched + * once, at build, and processing sends only the data request. + */ + @Test + public void Build_FetchesBothLists() throws Exception { + CloudRequestEngine engine = build(); + + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(0, requests(DATA), "json"); + assertNotNull(engine.getPublicProperties()); + assertTrue(engine.getPublicProperties().containsKey("device")); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + assertEquals(0, warnings().size(), warnings().toString()); + + process(engine, false); + + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, requests(DATA), "json"); + } + + private static Stream definitiveFailures() { + List arguments = new ArrayList<>(); + for (int status : new int[] { 400, 401, 403, 404, 429 }) { + arguments.add(Arguments.of(PROPERTIES, status)); + arguments.add(Arguments.of(EVIDENCE_KEYS, status)); + } + return arguments.stream(); + } + + /** + * A 4xx answer to either request is a definitive configuration error, + * which fails the build once both requests have been made. + */ + @ParameterizedTest(name = "{1} from {0}") + @MethodSource("definitiveFailures") + public void Build_4xx_FailsTheBuild(String endpoint, int status) { + answers.put(endpoint, status); + + CloudRequestException error = + assertThrows(CloudRequestException.class, this::build); + + assertEquals(status, error.getHttpStatusCode()); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + } + + /** + * A 4xx answer fails the build even when the other request failed in a + * way that can pass. + */ + @Test + public void Build_4xxAndTransient_FailsTheBuild() { + answers.put(PROPERTIES, 503); + answers.put(EVIDENCE_KEYS, 403); + + CloudRequestException error = + assertThrows(CloudRequestException.class, this::build); + + assertEquals(403, error.getHttpStatusCode()); + } + + private static Stream transientFailures() { + return Stream.of( + Arguments.of("500 from accessibleproperties", 500, 200), + Arguments.of("503 from evidencekeys", 200, 503), + Arguments.of("502 from both", 502, 502), + Arguments.of("unreadable accessibleproperties", UNREADABLE, 200), + Arguments.of("unreadable evidencekeys", 200, UNREADABLE), + Arguments.of("connection refused to both", UNREACHABLE, UNREACHABLE), + Arguments.of("timeout on evidencekeys", 200, TIMES_OUT)); + } + + /** + * Any other failure, the service unreachable, a timeout, a 5xx answer or + * a body that cannot be read, is logged as one warning and the build + * completes. + */ + @ParameterizedTest(name = "{0}") + @MethodSource("transientFailures") + public void Build_TransientFailure_CompletesWithOneWarning( + String name, + int propertiesAnswer, + int evidenceKeysAnswer) throws Exception { + answers.put(PROPERTIES, propertiesAnswer); + answers.put(EVIDENCE_KEYS, evidenceKeysAnswer); + + CloudRequestEngine engine = build(); + + assertNotNull(engine); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, warnings().size(), warnings().toString()); + } + + /** + * A list that arrived at build is kept, and the other is empty until it + * is fetched. Reading them makes no request. + */ + @Test + public void Build_PartialFailure_KeepsWhatArrived() throws Exception { + answers.put(EVIDENCE_KEYS, 503); + + CloudRequestEngine engine = build(); + + assertNotNull(engine.getPublicProperties()); + assertTrue(engine.getPublicProperties().containsKey("device")); + assertFalse(engine.getEvidenceKeyFilter().include("header.user-agent")); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + } + + /** + * Processing fetches only what the build could not, then sends the data + * request. The Pipeline's Evidence filter, built before the retry, sees + * the keys the retry fetched. + */ + @Test + public void Process_FetchesOnlyWhatIsMissing() throws Exception { + answers.put(EVIDENCE_KEYS, 503); + CloudRequestEngine engine = build(); + answers.put(EVIDENCE_KEYS, 200); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + assertFalse(pipeline.getEvidenceKeyFilter() + .include("header.user-agent")); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + data.process(); + } + assertTrue(pipeline.getEvidenceKeyFilter() + .include("header.user-agent")); + } + + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(2, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, requests(DATA), "json"); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + } + + /** + * While discovery still fails no data request is sent, and the failure + * is recorded as SuppressProcessExceptions says. + */ + @Test + public void Process_DiscoveryStillFailing_Suppressed() throws Exception { + answers.put(PROPERTIES, 503); + CloudRequestEngine engine = build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + data.process(); + + assertFalse(data.getErrors().isEmpty()); + } + + assertEquals(2, requests(PROPERTIES), "accessibleproperties"); + assertEquals(0, requests(DATA), "json"); + } + + @Test + public void Process_DiscoveryStillFailing_Throws() throws Exception { + answers.put(PROPERTIES, UNREACHABLE); + answers.put(EVIDENCE_KEYS, UNREACHABLE); + CloudRequestEngine engine = build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + assertThrows(RuntimeException.class, data::process); + } + + assertEquals(0, requests(DATA), "json"); + } + + /** + * The configured timeout applies to the discovery requests, so a + * service that does not answer delays the build by the timeout alone. + */ + @Test + public void Build_AppliesTheTimeoutToDiscovery() throws Exception { + build(); + + assertEquals(2, connections.size()); + for (HttpURLConnection connection : connections) { + verify(connection).setConnectTimeout(TIMEOUT_SECONDS * 1000); + verify(connection).setReadTimeout(TIMEOUT_SECONDS * 1000); + } + } + + private CloudRequestEngine build() throws Exception { + return new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .setTimeOutSeconds(TIMEOUT_SECONDS) + .build(); + } + + private void process(CloudRequestEngine engine, boolean suppress) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(suppress) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + data.process(); + } + } + + private int requests(String endpoint) { + return requests.getOrDefault(endpoint, 0); + } + + private List warnings() { + List warnings = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + warnings.addAll(logger.warningsLogged); + } + return warnings; + } + + private static String endpoint(URL url) { + String path = url.getPath().toLowerCase(); + if (path.endsWith(PROPERTIES)) { + return PROPERTIES; + } + if (path.endsWith(EVIDENCE_KEYS)) { + return EVIDENCE_KEYS; + } + return DATA; + } + + private String body(HttpURLConnection connection) throws IOException { + String name = endpoint(connection.getURL()); + int answer = answers.get(name); + if (answer == TIMES_OUT) { + throw new SocketTimeoutException("Read timed out"); + } + if (answer == UNREADABLE) { + return "proxy error"; + } + if (answer != 200) { + return "{\"errors\": [\"" + name + " answered " + answer + "\"]}"; + } + switch (name) { + case PROPERTIES: + return PROPERTIES_BODY; + case EVIDENCE_KEYS: + return EVIDENCE_KEYS_BODY; + default: + return DATA_BODY; + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 80ebde301..100de4cff 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -263,14 +263,11 @@ private static Stream getValidateErrorArgs() { } /** - * Test cloud request engine handles errors from the cloud service - * as expected. - * An AggregateException should be thrown by the cloud request engine - * containing the errors from the cloud service - * and the pipeline is configured to throw any exceptions up - * the stack in an AggregateException. - * We also check that the exception message includes the content - * from the JSON response. + * Errors from the cloud service when the engine is built. A 4xx answer + * is a definitive configuration error, which fails the build with a + * CloudRequestException that carries the cloud's message. Any other + * answer that holds errors is logged as one warning, the build + * completes, and the accessible properties are unknown until a retry. */ @ParameterizedTest(name = "response code {1} - {0}") @MethodSource("getValidateErrorArgs") @@ -284,24 +281,33 @@ public void validateErrorHandling(String name, int responseCode, boolean include configureMockedClient(); - Exception exception = null; - - try { - new CloudRequestEngineBuilder(loggerFactory, httpClient) - .setResourceKey(resourceKey) - .build() - .getPublicProperties(); - } catch (Exception ex) { - exception = ex; + if (responseCode >= 400 && responseCode < 500) { + CloudRequestException cloudEx = assertThrows( + CloudRequestException.class, + () -> new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build()); + assertEquals(responseCode, cloudEx.getHttpStatusCode()); + if (includeMessage) { + assertTrue(cloudEx.getMessage().contains(errorMessage), + "Exception message did not contain the expected text."); + } + } else { + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build(); + assertNull(engine.getPublicProperties()); + assertEquals(1, warningCount()); } + } - assertNotNull(exception, "Expected exception to occur"); - assertTrue(exception instanceof CloudRequestException); - CloudRequestException cloudEx = (CloudRequestException) exception; - if (includeMessage) { - assertTrue(cloudEx.getMessage().contains(errorMessage), - "Exception message did not contain the expected text."); + private int warningCount() { + int count = 0; + for (TestLogger logger : loggerFactory.loggers) { + count += logger.warningsLogged.size(); } + return count; } private static Stream getPrecidenceArgs() { @@ -656,22 +662,22 @@ public void getPublicProperties() throws Exception { } /** - * Verify that PropertyNotLoadedException is thrown on getPublicProperties if remote server is unavailable + * An unreachable server when the engine is built is a failure that can + * pass. The build completes with one warning, and the accessible + * properties are unknown until processing fetches them. */ @Test - public void getPublicProperties_Throw_Exception_When_Server_Unavailable() throws IOException { + public void getPublicProperties_Server_Unavailable_At_Build() throws Exception { final String resourceKey = "resource_key"; configureFailingMockClient(); - try { - new CloudRequestEngineBuilder(loggerFactory, httpClient) - .setResourceKey(resourceKey) - .build() - .getPublicProperties(); - fail("Expected exception was not thrown"); - } catch (Exception e) { - assertInstanceOf(PropertyNotLoadedException.class, e); - } + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build(); + + assertNull(engine.getPublicProperties()); + assertEquals(1, warningCount()); } /** @@ -689,21 +695,23 @@ public void getEvidenceKeyFilter() throws Exception { } /** - * Verify that PropertyNotLoadedException is thrown on getEvidenceKeyFilter if remote server is unavailable + * An unreachable server when the engine is built leaves the accepted + * Evidence empty, rather than failing whatever reads it, until + * processing fetches it. */ @Test - public void getEvidenceKeyFilter_Throw_Exception_When_Server_Unavailable() throws IOException { + public void getEvidenceKeyFilter_Server_Unavailable_At_Build() throws Exception { final String resourceKey = "resource_key"; configureFailingMockClient(); - try { - new CloudRequestEngineBuilder(loggerFactory, httpClient) - .setResourceKey(resourceKey) - .build() - .getEvidenceKeyFilter(); - fail("Expected exception was not thrown"); - } catch (Exception e) { - assertInstanceOf(PropertyNotLoadedException.class, e); - } + EvidenceKeyFilter filter = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build() + .getEvidenceKeyFilter(); + + assertNotNull(filter); + assertFalse(filter.include("query.User-Agent")); + assertEquals(1, warningCount()); } } From b92cfcfc3b6955a4bc4a0794d2a5b1330ae7d3ed Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Mon, 9 Nov 2026 21:01:04 +0000 Subject: [PATCH 36/95] Let common-ci set the versions, find the bundle and collect the unit test reports (#78) common-ci's release of 3 November takes a list of folders to set the version in, finds the Central publishing bundle under any module, and collects Surefire reports from modules at any depth. The versions:set loop and the link at target/central-publishing in ci/build-package.ps1 go, and ci/run-unit-tests.ps1 calls common-ci's helper instead of copying it. ci/copy-test-results.ps1 stays for the integration and performance tests, which make several Maven runs that report into one folder. Its comment now says so. --- ci/README.md | 6 ++--- ci/build-package.ps1 | 32 +++++++-------------------- ci/copy-test-results.ps1 | 7 +++--- ci/run-unit-tests.ps1 | 48 +++++++++++++--------------------------- 4 files changed, 30 insertions(+), 63 deletions(-) diff --git a/ci/README.md b/ci/README.md index bf6b2a62b..b80c3659e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,9 +12,9 @@ repositories used to do for one package tree each. generated accessors. The other scripts loop over it rather than naming folders themselves. - The root `pom.xml` builds every tree in one reactor but is not their parent, - so [build-package.ps1](build-package.ps1) sets the version in each tree - before it deploys from the root. The examples are built and tested but left - out of the deploy. + so [build-package.ps1](build-package.ps1) has common-ci set the version in + each tree before it deploys from the root. The examples are built and tested + but left out of the deploy. - [build-project.ps1](build-project.ps1) and [run-unit-tests.ps1](run-unit-tests.ps1) use the `dev` profile, so a pull request tests the examples against the libraries it changes. The publish diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index 38de86fe9..75464f09f 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -21,19 +21,6 @@ foreach ($package in $NativePackages) { } } -# The root pom.xml is not the parent of the package trees, so setting the -# version at the root does not reach them. Set it in each tree, the examples -# included, so the whole repository carries the version being released. -foreach ($package in $Packages) { - Write-Host "Setting the version of '$($package.Path)' to '$Version'" - Push-Location "$RepoName/$($package.Path)" - try { - mvn versions:set @MavenOptions "-DnewVersion=$Version" -DgenerateBackupPoms=false - } finally { - Pop-Location - } -} - # The examples are never published. They are left out of the deploy so that # the last project in the reactor is one the publishing plugin is configured # for. Maven 3 does not exclude a folder's modules with the folder, so each of @@ -45,21 +32,18 @@ $examples = foreach ($package in $Packages | Where-Object { -not $_.Publish }) { ForEach-Object { '!' + [IO.Path]::GetRelativePath($repoPath, $_.DirectoryName).Replace('\', '/') } } -# The publishing plugin writes the one bundle for the whole reactor under the -# first tree that publishes. common-ci copies the bundle from where a single -# tree puts it, below target at the root, so a link at that path leads to the -# bundle. The link is made before the deploy, which Linux allows. -$firstPublishing = ($Packages | Where-Object { $_.Publish } | Select-Object -First 1).Path -New-Item -ItemType Directory -Force -Path "$RepoName/target" | Out-Null -New-Item -ItemType SymbolicLink -Force -Path "$RepoName/target/central-publishing" -Target "../$firstPublishing/target/central-publishing" | Out-Null - -# common-ci sets the version of the root pom.xml, deploys from the root to a -# local staging folder with signing, and copies the result and the publishing -# bundle into the package folder. +# common-ci sets the version, deploys from the root to a local staging folder +# with signing, and copies the result and the publishing bundle into the +# package folder. The root pom.xml is not the parent of the package trees, so +# setting the version at the root would not reach them. VersionPaths names each +# tree, the examples included, so the whole repository carries the version +# being released. The publishing plugin writes the one bundle for the whole +# reactor under the first tree that publishes, and common-ci finds it there. ./java/build-package.ps1 ` -RepoName $RepoName ` -Name $Name ` -Version $Version ` + -VersionPaths $Packages.Path ` -ExtraArgs "-DskipNativeBuild=true", "-pl", ($examples -join ',') ` -JavaGpgKeyPassphrase $Keys['JavaGpgKeyPassphrase'] ` -JavaPGP $Keys['JavaPGP'] ` diff --git a/ci/copy-test-results.ps1 b/ci/copy-test-results.ps1 index 712c1539c..543342159 100644 --- a/ci/copy-test-results.ps1 +++ b/ci/copy-test-results.ps1 @@ -1,8 +1,9 @@ # Copies the Surefire reports of every module below Path into Destination. # -# common-ci's Java scripts only look one folder down for modules, which finds -# the modules of a single package tree but not those of this repository, where -# each tree is a folder of its own and the examples are a folder further down. +# common-ci's Java scripts copy the reports of the one Maven run they make. The +# integration and performance tests here make several runs, each in a tree of +# its own and each reporting into the same folder, so those two scripts copy +# the reports themselves. The unit tests are one run, which common-ci makes. param ( [Parameter(Mandatory)][string]$Path, [Parameter(Mandatory)][string]$Destination, diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index 562f29486..a54755337 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -4,39 +4,21 @@ param( [string]$TestResourceKey, [string]$DeviceDetection ) -$PSNativeCommandUseErrorActionPreference = $false - -. "$PSScriptRoot/packages.ps1" +$ErrorActionPreference = "Stop" # One run of every test in the reactor that build-project.ps1 built, the -# examples included. SuperResourceKey is read by the Location and Device -# Detection examples tests, and LicenseKey by the Device Detection data file -# update example. The IP Intelligence update example's test is ignored, so it -# does not need a key of its own. -Write-Host "Entering '$RepoName'" -Push-Location $RepoName -try { - Write-Host "Testing '$Name'" - mvn surefire:test @MavenOptions -Pdev -DfailIfNoTests=false ` - "-DTestResourceKey=$TestResourceKey" ` - "-DSuperResourceKey=$TestResourceKey" ` - "-DLicenseKey=$DeviceDetection" - # Kept here because the commands below run commands of their own, which - # would replace Maven's exit code. - $MavenExitCode = $LASTEXITCODE -} finally { - Write-Host "Leaving '$RepoName'" - Pop-Location -} - -& "$PSScriptRoot/copy-test-results.ps1" -Path $RepoName -Destination "$RepoName/test-results/unit" -Exclude '*ExampleTests*' - -# A forked JVM that dies leaves why it died only on the runner, so common-ci -# logs it. The collector is only in branches of common-ci that have it, so it -# is called only when present. It does not change the outcome. -$Collector = [IO.Path]::Combine($pwd, "java", "collect-jvm-crash-evidence.ps1") -if ($MavenExitCode -ne 0 -and (Test-Path $Collector) -eq $true) { - & $Collector -RepoName $RepoName -} +# examples included. common-ci runs Surefire from the root, copies the reports +# of every module at any depth into test-results/unit, leaving out the example +# tests the integration tests report, and logs why a forked JVM died when one +# does. The dev profile points the examples at the reactor. SuperResourceKey is +# read by the Location and Device Detection examples tests, and LicenseKey by +# the Device Detection data file update example. The IP Intelligence update +# example's test is ignored, so it does not need a key of its own. +./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs @( + '-Pdev' + "-DTestResourceKey=$TestResourceKey" + "-DSuperResourceKey=$TestResourceKey" + "-DLicenseKey=$DeviceDetection" +) -exit $MavenExitCode +exit $LASTEXITCODE From a4b6fd5e24e6812a702244536af8c8a721eded5d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:05:43 +0000 Subject: [PATCH 37/95] Discover when the Cloud Request Engine is built, and retry on first use only after a failure that can pass The engine fetched its accessible properties and accepted evidence keys only when something first asked for them, and a failure surfaced wherever that was. The builder now calls discover, which makes both requests. A 4xx answer to either is a definitive configuration error, and its CloudRequestException fails the build once both requests are made. Any other failure, the service unreachable, a timeout, a 5xx answer or a body that cannot be read, is logged as one warning, the build completes and the list that arrived is kept. The configured timeout now applies to both requests. getPublicProperties and getEvidenceKeyFilter make no request. The products are null until they are known. The evidence filter is one CloudEvidenceKeyFilter that includes nothing until the keys are known and reads them on each call, and it derives from EvidenceKeyFilterWhitelist so the Pipeline's filter keeps it rather than copying its keys once. Processing fetches what is missing, evidence keys first, before it sends the data request, and a request that could not be made is reported as its IOException, as for the data request. The Device Detection, IP Intelligence and Location cloud engines, and CloudAspectEngineBase, read their metadata when the products are known rather than logging errors at build and keeping none. A missing product still builds and logs once. The web integration reads a filter that derives from EvidenceKeyFilterWhitelist for the Vary header, as it reads the exact class. --- .../DeviceDetectionCloudEngine.java | 36 +++- .../IPIntelligenceCloudEngine.java | 36 +++- .../flowelements/GeoLocationCloudEngine.java | 36 +++- .../cloudrequestengine/Constants.java | 7 + .../flowelements/CloudAspectEngineBase.java | 22 +- .../flowelements/CloudEvidenceKeyFilter.java | 94 +++++++++ .../flowelements/CloudRequestEngine.java | 3 +- .../CloudRequestEngineBuilder.java | 7 +- .../CloudRequestEngineDefault.java | 188 +++++++++++++++--- .../ClientsidePropertyServiceCore.java | 10 +- 10 files changed, 370 insertions(+), 69 deletions(-) create mode 100644 pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index 2e6e2f979..e43fc4b24 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -55,9 +55,10 @@ */ public class DeviceDetectionCloudEngine extends CloudAspectEngineBase { - List aspectProperties; + volatile List aspectProperties; private String dataSourceTier; CloudRequestEngine cloudRequestEngine; + private final Object metadataLock = new Object(); /** * Construct a new instance of the {@link DeviceDetectionCloudEngine}. @@ -72,9 +73,26 @@ public DeviceDetectionCloudEngine( this.cloudRequestEngine = null; } + /** + * The property metadata, from the products the Cloud Request Engine + * reports. Until they are known, after a failure at build that can + * pass, this is empty and is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { - return aspectProperties; + List loaded = aspectProperties; + if (loaded == null && cloudRequestEngine != null) { + synchronized (metadataLock) { + if (aspectProperties == null) { + loadAspectProperties(cloudRequestEngine); + } + loaded = aspectProperties; + } + } + return loaded == null ? + new ArrayList() : + loaded; } @Override @@ -188,9 +206,10 @@ public void addPipeline(Pipeline pipeline) { if (cloudRequestEngine == null) { cloudRequestEngine = pipeline.getElement(CloudRequestEngine.class); if (cloudRequestEngine != null) { - if (loadAspectProperties(cloudRequestEngine) == false) { - logger.error("Failed to load aspect properties"); - } + // The products are not known after a failure at build that + // can pass, which the Cloud Request Engine has logged. + // getProperties reads them once they are. + loadAspectProperties(cloudRequestEngine); } } @@ -209,7 +228,7 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { findProduct(engine.getPublicProperties()); if (product != null) { - aspectProperties = new ArrayList<>(); + List properties = new ArrayList<>(); dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : @@ -221,13 +240,12 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { item.getPropertyType(), new ArrayList(), true); - aspectProperties.add(property); + properties.add(property); } + aspectProperties = properties; return true; } else { - logger.error("Aspect properties could not be loaded for" + - " the Device Detection cloud engine", this); return false; } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index da5039234..bec4510d2 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -63,8 +63,9 @@ */ public class IPIntelligenceCloudEngine extends CloudAspectEngineBase { - List aspectProperties; + volatile List aspectProperties; private String dataSourceTier; + private final Object metadataLock = new Object(); // Lower-cased names of properties the cloud returns as weighted lists // (cloud type "Weighted..."), so processEngine can read their value/weight // pairs rather than treating them as a plain list. @@ -84,9 +85,26 @@ public IPIntelligenceCloudEngine( this.cloudRequestEngine = null; } + /** + * The property metadata, from the products the Cloud Request Engine + * reports. Until they are known, after a failure at build that can + * pass, this is empty and is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { - return aspectProperties; + List loaded = aspectProperties; + if (loaded == null && cloudRequestEngine != null) { + synchronized (metadataLock) { + if (aspectProperties == null) { + loadAspectProperties(cloudRequestEngine); + } + loaded = aspectProperties; + } + } + return loaded == null ? + new ArrayList() : + loaded; } @Override @@ -222,9 +240,10 @@ public void addPipeline(Pipeline pipeline) { if (cloudRequestEngine == null) { cloudRequestEngine = pipeline.getElement(CloudRequestEngine.class); if (cloudRequestEngine != null) { - if (loadAspectProperties(cloudRequestEngine) == false) { - logger.error("Failed to load aspect properties"); - } + // The products are not known after a failure at build that + // can pass, which the Cloud Request Engine has logged. + // getProperties reads them once they are. + loadAspectProperties(cloudRequestEngine); } } @@ -243,7 +262,7 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { findProduct(engine.getPublicProperties()); if (product != null) { - aspectProperties = new ArrayList<>(); + List properties = new ArrayList<>(); weightedPropertyNames.clear(); dataSourceTier = product.dataTier; @@ -259,13 +278,12 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { resolvePropertyType(item), new ArrayList(), true); - aspectProperties.add(property); + properties.add(property); } + aspectProperties = properties; return true; } else { - logger.error("Aspect properties could not be loaded for" + - " the IP Intelligence cloud engine", this); return false; } } diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 7fb48e1cd..243abac97 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -47,15 +47,33 @@ public class GeoLocationCloudEngine extends CloudAspectEngineBase { private CloudRequestEngine cloudRequestEngine = null; - private List properties; + private volatile List properties; + private final Object metadataLock = new Object(); private String dataSourceTier; private String elementDataKey; private EvidenceKeyFilter evidenceKeyFilter = new EvidenceKeyFilterWhitelist(Collections.emptyList()); private String dataProviderPrefix; + /** + * The property metadata, from the products the Cloud Request Engine + * reports. Until they are known, after a failure at build that can + * pass, this is empty and is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { - return properties; + List loaded = properties; + if (loaded == null && cloudRequestEngine != null) { + synchronized (metadataLock) { + if (properties == null) { + loadAspectProperties(cloudRequestEngine); + } + loaded = properties; + } + } + return loaded == null ? + new ArrayList() : + loaded; } @Override @@ -167,9 +185,10 @@ public void addPipeline(Pipeline pipeline) { if (cloudRequestEngine == null) { cloudRequestEngine = pipeline.getElement(CloudRequestEngine.class); if (cloudRequestEngine != null) { - if (loadAspectProperties(cloudRequestEngine) == false) { - logger.error("Failed to load aspect properties"); - } + // The products are not known after a failure at build that + // can pass, which the Cloud Request Engine has logged. + // getProperties reads them once they are. + loadAspectProperties(cloudRequestEngine); } } @@ -181,7 +200,7 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { findProduct(engine.getPublicProperties()); if (product != null) { - properties = new ArrayList<>(); + List loaded = new ArrayList<>(); dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : product.properties) @@ -197,13 +216,12 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { item.delayExecution, item.evidenceProperties); - properties.add(property); + loaded.add(property); } + properties = loaded; return true; } else { - logger.error("Aspect properties could not be loaded for" + - " the Location cloud engine", this); return false; } } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index 613dcff5a..4ac6fbbbb 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -48,6 +48,13 @@ public class Messages { + "see our explainer: " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; + public static final String MessageDiscoveryDeferred = + "The cloud request engine could not fetch its accessible " + + "properties or accepted evidence keys when it was built, " + + "because of a failure that can pass. It will fetch them " + + "again when it is first used, and sends no data request " + + "until it has both."; + public static final String MessageProductNotInResourceKey = "The resource key does not include the '%s' product, so " + "%s has no properties and its results hold no values. " diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index a828a9485..ee605233e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -22,7 +22,6 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; -import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionFailedToLoadProperties; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.MessageProductNotInResourceKey; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ProcessCloudEngineNotImplemented; @@ -146,6 +145,13 @@ protected void setRequestEngine(RequestEngineAccessor requestEngine) { this.requestEngine = requestEngine; } + /** + * The property metadata of this engine, from the products the Cloud + * Request Engine reports. After a failure at build that can pass the + * products are not known yet, and an empty list is returned without + * being kept, so the metadata is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { List localRef = aspectProperties; @@ -154,17 +160,13 @@ public List getProperties() { localRef = aspectProperties; if (localRef == null) { if(loadAspectProperties() == false) { - throw new RuntimeException( - String.format( - ExceptionFailedToLoadProperties, - this.getElementDataKey(), - this.getElementDataKey()) - ); + return new ArrayList<>(); } + localRef = aspectProperties; } } } - return aspectProperties; + return localRef; } @Override @@ -221,8 +223,8 @@ private boolean loadAspectProperties() { aspectProperties = properties; return true; } else { - logger.error("Aspect properties could not be loaded for " + - this.getClass().getName(), this); + // The Cloud Request Engine has logged why the products are not + // known yet. return false; } } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java new file mode 100644 index 000000000..bfcac3997 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java @@ -0,0 +1,94 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The Evidence keys the cloud service accepts, from its evidencekeys + * request. It includes nothing until the keys are known, and reads them on + * each call, so a filter that holds this one sees the keys as soon as they + * arrive. It derives from {@link EvidenceKeyFilterWhitelist} so that the + * Pipeline's filter keeps it rather than copying its keys once, and so + * that the web integration lists its headers in the Vary header. + */ +class CloudEvidenceKeyFilter extends EvidenceKeyFilterWhitelist { + + private volatile EvidenceKeyFilterWhitelist keys = null; + + CloudEvidenceKeyFilter() { + super(new ArrayList(), String.CASE_INSENSITIVE_ORDER); + } + + /** + * @return true once the keys are known + */ + boolean isKnown() { + return keys != null; + } + + /** + * Set the keys the cloud service accepts. + * @param evidenceKeys the keys + */ + void setKeys(List evidenceKeys) { + keys = new EvidenceKeyFilterWhitelist( + evidenceKeys, + String.CASE_INSENSITIVE_ORDER); + } + + /** + * @return the keys, or null if they are not known + */ + List getKeys() { + EvidenceKeyFilterWhitelist current = keys; + return current == null ? + null : + new ArrayList<>(current.getWhitelist().keySet()); + } + + @Override + public boolean include(String key) { + EvidenceKeyFilterWhitelist current = keys; + return current != null && current.include(key); + } + + @Override + public Integer order(String key) { + EvidenceKeyFilterWhitelist current = keys; + return current == null ? null : current.order(key); + } + + @Override + public Map getWhitelist() { + EvidenceKeyFilterWhitelist current = keys; + return current == null ? + Collections.emptyMap() : + current.getWhitelist(); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java index 1025b5799..d3027f2d0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java @@ -41,7 +41,8 @@ public interface CloudRequestEngine /** * A collection of the properties that the cloud service can populate in the * JSON response. Keyed on property name. - * @return public properties + * @return public properties, or null if they are not known yet, after a + * failure when the engine was built that can pass */ Map getPublicProperties(); } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index bc72731bf..f2da2735e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -98,7 +98,7 @@ protected CloudRequestEngine newEngine(List properties) throws Exception } } - return new CloudRequestEngineDefault( + CloudRequestEngineDefault engine = new CloudRequestEngineDefault( loggerFactory.getLogger(CloudRequestEngine.class.getName()), new CloudRequestDataFactory(loggerFactory), httpClient, @@ -109,6 +109,11 @@ protected CloudRequestEngine newEngine(List properties) throws Exception evidenceKeysEndpoint, timeoutMillis, cloudRequestOrigin); + // Fetch the accessible properties and the accepted evidence keys + // now, so that the built engine is ready. A 4xx answer fails the + // build, and any other failure is retried on first use. + engine.discover(); + return engine; } private String getEnvironmentVariable(String name) { diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 12ccc17f2..65181de0c 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -82,12 +82,38 @@ public class CloudRequestEngineDefault private String cloudRequestOrigin; private Integer timeoutMillis; + /** + * The lowest HTTP status of a client error. A discovery request answered + * with a status from this up to {@link #HTTP_SERVER_ERROR_MIN} is a + * definitive configuration error. + */ + private static final int HTTP_CLIENT_ERROR_MIN = 400; + + /** + * The lowest HTTP status of a server error. + */ + private static final int HTTP_SERVER_ERROR_MIN = 500; + private List propertyMetaData; + + /** + * The products the resource key gives access to, from the + * accessibleproperties request, or null until that request succeeds. + */ private volatile Map publicProperties; - private final Object publicPropertiesLock = new Object(); - private volatile EvidenceKeyFilter evidenceKeyFilter; - private final Object evidenceKeyFilterLock = new Object(); + /** + * The accepted Evidence, from the evidencekeys request. It is the same + * instance before and after the request succeeds. + */ + private final CloudEvidenceKeyFilter evidenceKeyFilter = + new CloudEvidenceKeyFilter(); + + /** + * Held while the discovery requests are made, so that concurrent first + * uses make them once. + */ + private final Object discoveryLock = new Object(); public CloudRequestEngineDefault( Logger logger, @@ -193,40 +219,141 @@ public String getElementDataKey() { return "cloud-response"; } + /** + * The accepted Evidence keys from the evidencekeys request. Until that + * request succeeds the filter includes nothing, and it makes no request + * itself. A filter that holds this one, such as the Pipeline's, sees + * the keys as soon as a retry fetches them. + * @return the accepted Evidence + */ @Override - public EvidenceKeyFilter getEvidenceKeyFilter() throws CloudRequestException, AggregateException, PropertyNotLoadedException { - if (evidenceKeyFilter == null) { - synchronized (evidenceKeyFilterLock) { - if (evidenceKeyFilter == null) { - getCloudEvidenceKeys(); - } - } - } + public EvidenceKeyFilter getEvidenceKeyFilter() { return evidenceKeyFilter; } + /** + * The products the resource key gives access to, from the + * accessibleproperties request, which the builder makes when it builds + * the engine. After a failure at build that can pass this is null until + * processing fetches them. It makes no request itself. + * @return the products, or null if they are not known yet + */ @Override public Map - getPublicProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { - if (publicProperties == null) { - synchronized (publicPropertiesLock) { + getPublicProperties() { + return publicProperties; + } + + /** + * Fetch the accepted Evidence keys and the accessible properties that + * are not known yet. The builder calls this when it builds the engine, + * so in the normal case a built engine is ready to process. Both + * requests are made. A 4xx answer to either is a definitive + * configuration error, and its {@link CloudRequestException} is thrown, + * because retrying with the same configuration can never succeed. Any + * other failure, such as the service being unreachable, a timeout, a + * 5xx answer or a body that cannot be read, is logged as one warning + * and the list that did arrive is kept. Processing fetches what is + * still missing. + * @throws CloudRequestException for a 4xx answer to either request + */ + public void discover() { + List failures = new ArrayList<>(); + synchronized (discoveryLock) { + if (evidenceKeyFilter.isKnown() == false) { + try { + fetchEvidenceKeys(); + } catch (RuntimeException e) { + failures.add(e); + } + } + if (publicProperties == null) { + try { + fetchPublicProperties(); + } catch (RuntimeException e) { + failures.add(e); + } + } + } + if (failures.isEmpty()) { + return; + } + for (RuntimeException failure : failures) { + CloudRequestException definitive = definitiveError(failure); + if (definitive != null) { + throw definitive; + } + } + RuntimeException first = failures.get(0); + for (int i = 1; i < failures.size(); i++) { + first.addSuppressed(failures.get(i)); + } + logger.warn(MessageDiscoveryDeferred, first); + } + + /** + * The failure of a discovery request that the cloud service received + * and rejected with a 4xx status, so that retrying with the same + * configuration can never succeed. + * @param failure the failure of a discovery request + * @return the exception with the 4xx status, or null if there is none + */ + private static CloudRequestException definitiveError(Throwable failure) { + if (failure instanceof CloudRequestException) { + int status = ((CloudRequestException) failure).getHttpStatusCode(); + return status >= HTTP_CLIENT_ERROR_MIN && + status < HTTP_SERVER_ERROR_MIN ? + (CloudRequestException) failure : + null; + } + if (failure instanceof AggregateException) { + for (Throwable inner : failure.getSuppressed()) { + CloudRequestException definitive = definitiveError(inner); + if (definitive != null) { + return definitive; + } + } + } + return null; + } + + /** + * Fetch what discovery could not when the engine was built, evidence + * keys first, stopping at the first failure, which is a process-time + * failure. No data request is sent until both are known. + * @throws IOException if a discovery request could not be made + */ + private void completeDiscovery() throws IOException { + if (evidenceKeyFilter.isKnown() && publicProperties != null) { + return; + } + synchronized (discoveryLock) { + try { + if (evidenceKeyFilter.isKnown() == false) { + fetchEvidenceKeys(); + } if (publicProperties == null) { - getCloudProperties(); + fetchPublicProperties(); + } + } catch (PropertyNotLoadedException e) { + // A request that could not be made is reported as the + // IOException it was, as for the data request. + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); } + throw e; } } - return publicProperties; } @Override protected void processEngine(FlowData data, CloudRequestData aspectData) throws IOException { + ((CloudRequestDataInternal)aspectData).setProcessStarted(true); + completeDiscovery(); + byte[] content = getContent(data); HttpURLConnection connection = httpClient.connect(toUrl(endPoint.trim())); - if (timeoutMillis != null) { - connection.setConnectTimeout(timeoutMillis); - connection.setReadTimeout(timeoutMillis); - } - ((CloudRequestDataInternal)aspectData).setProcessStarted(true); + setTimeouts(connection); Map headers = new HashMap<>(); setCommonHeaders(headers); @@ -425,7 +552,17 @@ private static URL toUrl(String spec) throws MalformedURLException { } } - private void getCloudProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { + /** + * Apply the configured timeout to a request. + */ + private void setTimeouts(HttpURLConnection connection) { + if (timeoutMillis != null) { + connection.setConnectTimeout(timeoutMillis); + connection.setReadTimeout(timeoutMillis); + } + } + + private void fetchPublicProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; @@ -433,6 +570,7 @@ private void getCloudProperties() throws CloudRequestException, AggregateExcepti setCommonHeaders(headers); HttpURLConnection connection = httpClient.connect(toUrl(propertiesEndpoint.trim() + (resourceKey != null ? "?Resource=" + resourceKey : ""))); + setTimeouts(connection); jsonResult = httpClient.getResponseString(connection, headers); validateResponse(jsonResult, connection); @@ -455,7 +593,7 @@ private void getCloudProperties() throws CloudRequestException, AggregateExcepti } } - private void getCloudEvidenceKeys() throws CloudRequestException, AggregateException, PropertyNotLoadedException { + private void fetchEvidenceKeys() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; @@ -463,6 +601,7 @@ private void getCloudEvidenceKeys() throws CloudRequestException, AggregateExcep setCommonHeaders(headers); HttpURLConnection connection = httpClient.connect(toUrl(evidenceKeysEndpoint.trim())); + setTimeouts(connection); jsonResult = httpClient.getResponseString(connection, headers); validateResponse(jsonResult, connection, false); @@ -472,8 +611,7 @@ private void getCloudEvidenceKeys() throws CloudRequestException, AggregateExcep for (int i = 0; i < jsonArray.length(); i++) { keys.add(jsonArray.get(i).toString()); } - evidenceKeyFilter = new EvidenceKeyFilterWhitelist(keys, - String.CASE_INSENSITIVE_ORDER); + evidenceKeyFilter.setKeys(keys); } } catch (IOException e) { throw new PropertyNotLoadedException(e); diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java index 67439d7cc..630ac847a 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java @@ -150,11 +150,11 @@ protected final void init() { } for (EvidenceKeyFilter filter : filters) { - // If the filter is a white list or derived type then - // get all HTTP header evidence keys from white list - // and add them to the headers that could affect the - // generated JavaScript. - if (filter.getClass().equals(EvidenceKeyFilterWhitelist.class)) { + // If the filter is a white list or derived type, such as + // the filter of a parallel group, then get all HTTP header + // evidence keys from white list and add them to the + // headers that could affect the generated JavaScript. + if (filter instanceof EvidenceKeyFilterWhitelist) { EvidenceKeyFilterWhitelist whitelist = (EvidenceKeyFilterWhitelist) filter; for (String key : whitelist.getWhitelist().keySet()) { if (key.startsWith(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR) && From 4defa2ab87dd6bde5808e9c2a6f0e2898dc69ac6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:07:42 +0000 Subject: [PATCH 38/95] Let a Cloud Request Engine be built from a kept copy of what an earlier one discovered The specification's Resilience section says an engine SHOULD be buildable from a kept copy of the discovery results, the accepted evidence keys and the accessible properties, which depend only on the resource key, so that a Pipeline can be built offline and a host that starts often need not call the cloud service each time. CloudDiscoveryState holds the evidence keys and the accessibleproperties answer, and writes and reads them as one JSON document. CloudRequestEngineBuilder.getDiscoveryState returns the copy after a build that knows both lists, and setDiscoveryState makes a later build use it with no discovery request. The tests are in this commit because they use the new class and methods. --- .../CloudDiscoveryState.java | 115 ++++++++++++++++++ .../CloudRequestEngineBuilder.java | 43 ++++++- .../CloudRequestEngineDefault.java | 56 +++++++-- .../CloudRequestEngineDiscoveryTests.java | 52 ++++++++ 4 files changed, 253 insertions(+), 13 deletions(-) create mode 100644 pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java new file mode 100644 index 000000000..0731e46d8 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java @@ -0,0 +1,115 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A copy of what the Cloud Request Engine discovered for a resource key, + * being the accepted Evidence keys and the accessible properties, which + * depend only on the resource key. Read it from the builder after a + * successful build, keep it, for example as {@link #toJson()}, and give it + * to a later builder, which then makes no discovery request. A Pipeline can + * so be built offline, and a host that starts often need not call the + * cloud service each time. + * @see Specification + */ +public final class CloudDiscoveryState { + + private static final String EVIDENCE_KEYS = "evidenceKeys"; + + private static final String ACCESSIBLE_PROPERTIES = "accessibleProperties"; + + private final List evidenceKeys; + + private final String accessibleProperties; + + /** + * Construct a new instance. + * @param evidenceKeys the evidence keys the cloud service accepts + * @param accessibleProperties the answer to the accessibleproperties + * request, as JSON + * @throws org.json.JSONException if the accessible properties are not + * a JSON object with a Products member + */ + public CloudDiscoveryState( + List evidenceKeys, + String accessibleProperties) { + Objects.requireNonNull(evidenceKeys, "evidenceKeys"); + Objects.requireNonNull(accessibleProperties, "accessibleProperties"); + new JSONObject(accessibleProperties).getJSONObject("Products"); + this.evidenceKeys = Collections.unmodifiableList( + new ArrayList<>(evidenceKeys)); + this.accessibleProperties = accessibleProperties; + } + + /** + * @return the evidence keys the cloud service accepts + */ + public List getEvidenceKeys() { + return evidenceKeys; + } + + /** + * @return the answer to the accessibleproperties request, as JSON + */ + public String getAccessibleProperties() { + return accessibleProperties; + } + + /** + * @return this copy as one JSON document, which {@link #fromJson} + * reads back + */ + public String toJson() { + JSONObject json = new JSONObject(); + json.put(EVIDENCE_KEYS, new JSONArray(evidenceKeys)); + json.put(ACCESSIBLE_PROPERTIES, new JSONObject(accessibleProperties)); + return json.toString(); + } + + /** + * Read a copy written by {@link #toJson()}. + * @param json the JSON document + * @return the copy + * @throws org.json.JSONException if the document is not one + * {@link #toJson()} writes + */ + public static CloudDiscoveryState fromJson(String json) { + JSONObject document = new JSONObject(json); + JSONArray keys = document.getJSONArray(EVIDENCE_KEYS); + List evidenceKeys = new ArrayList<>(); + for (int i = 0; i < keys.length(); i++) { + evidenceKeys.add(keys.getString(i)); + } + return new CloudDiscoveryState( + evidenceKeys, + document.getJSONObject(ACCESSIBLE_PROPERTIES).toString()); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index f2da2735e..5e23b4ea4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -25,6 +25,7 @@ import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; @@ -56,6 +57,8 @@ public class CloudRequestEngineBuilder extends private String licenseKey = null; private String cloudRequestOrigin = null; private int timeoutMillis = Constants.DEFAULT_TIMEOUT_MILLIS; + private CloudDiscoveryState discoveryState = null; + private CloudRequestEngineDefault built = null; // Function to get environment variable value. This enable testable code. private Function getEnvVar = (name) -> { @@ -109,12 +112,44 @@ protected CloudRequestEngine newEngine(List properties) throws Exception evidenceKeysEndpoint, timeoutMillis, cloudRequestOrigin); - // Fetch the accessible properties and the accepted evidence keys - // now, so that the built engine is ready. A 4xx answer fails the - // build, and any other failure is retried on first use. - engine.discover(); + if (discoveryState != null) { + engine.useDiscoveryState(discoveryState); + } + else { + // Fetch the accessible properties and the accepted evidence + // keys now, so that the built engine is ready. A 4xx answer + // fails the build, and any other failure is retried on first + // use. + engine.discover(); + } + built = engine; return engine; } + + /** + * Build the engine from a copy of what an earlier engine discovered for + * the same resource key, so that the build makes no discovery request. + * The copy comes from {@link #getDiscoveryState()} after a successful + * build, and can be kept with {@link CloudDiscoveryState#toJson()}. + * @param state the copy, or null to discover when building + * @return this builder + */ + public CloudRequestEngineBuilder setDiscoveryState( + CloudDiscoveryState state) { + this.discoveryState = state; + return this; + } + + /** + * A copy of what the engine this builder last built discovered, to give + * to a later build with {@link #setDiscoveryState(CloudDiscoveryState)}. + * @return the copy, or null if no engine has been built or it does not + * yet know both the accepted evidence keys and the accessible properties + */ + public CloudDiscoveryState getDiscoveryState() { + CloudRequestEngineDefault engine = built; + return engine == null ? null : engine.getDiscoveryState(); + } private String getEnvironmentVariable(String name) { return System.getenv(name); diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 65181de0c..3f1be59b5 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; +import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; import fiftyone.pipeline.cloudrequestengine.CloudRequestException; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.Constants; @@ -102,6 +103,12 @@ public class CloudRequestEngineDefault */ private volatile Map publicProperties; + /** + * The answer to the accessibleproperties request that gave + * {@link #publicProperties}, kept for {@link #getDiscoveryState()}. + */ + private volatile String accessibleProperties; + /** * The accepted Evidence, from the evidencekeys request. It is the same * instance before and after the request succeeds. @@ -291,6 +298,33 @@ public void discover() { logger.warn(MessageDiscoveryDeferred, first); } + /** + * A copy of what discovery fetched, which a later builder can use in + * place of the discovery requests. + * @return the copy, or null unless both the accepted evidence keys and + * the accessible properties are known + */ + public CloudDiscoveryState getDiscoveryState() { + synchronized (discoveryLock) { + List keys = evidenceKeyFilter.getKeys(); + return keys == null || accessibleProperties == null ? + null : + new CloudDiscoveryState(keys, accessibleProperties); + } + } + + /** + * Use a copy of what an earlier discovery fetched, in place of the + * discovery requests. + * @param state the copy + */ + public void useDiscoveryState(CloudDiscoveryState state) { + synchronized (discoveryLock) { + evidenceKeyFilter.setKeys(state.getEvidenceKeys()); + setPublicProperties(state.getAccessibleProperties()); + } + } + /** * The failure of a discovery request that the cloud service received * and rejected with a 4xx status, so that retrying with the same @@ -574,16 +608,8 @@ private void fetchPublicProperties() throws CloudRequestException, AggregateExce jsonResult = httpClient.getResponseString(connection, headers); validateResponse(jsonResult, connection); - JSONObject jsonObj = null; if (jsonResult.isEmpty() == false) { - jsonObj = new JSONObject(jsonResult); - } - - if (jsonObj != null) { - AccessiblePropertyMetaData.LicencedProducts accessiblePropertyData = - new AccessiblePropertyMetaData.LicencedProducts(jsonObj.getJSONObject("Products")); - - publicProperties = accessiblePropertyData.products; + setPublicProperties(jsonResult); } else { throw new RuntimeException("Failed to retrieve available properties " + "from cloud service at " + propertiesEndpoint + "."); @@ -593,6 +619,18 @@ private void fetchPublicProperties() throws CloudRequestException, AggregateExce } } + /** + * Read the answer to the accessibleproperties request. + * @param json the answer + */ + private void setPublicProperties(String json) { + AccessiblePropertyMetaData.LicencedProducts accessiblePropertyData = + new AccessiblePropertyMetaData.LicencedProducts( + new JSONObject(json).getJSONObject("Products")); + accessibleProperties = json; + publicProperties = accessiblePropertyData.products; + } + private void fetchEvidenceKeys() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java index 1ae9bb3e6..742adaffc 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -24,6 +24,7 @@ import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; import fiftyone.pipeline.cloudrequestengine.CloudRequestException; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -336,6 +337,57 @@ public void Build_AppliesTheTimeoutToDiscovery() throws Exception { } } + /** + * A copy of what discovery fetched is read from the builder after a + * build, kept as JSON, and given to a later build, which makes no + * discovery request and is ready to process. + */ + @Test + public void DiscoveryState_KeptAndUsedWithNoRequest() throws Exception { + CloudRequestEngineBuilder first = builder(); + first.build(); + CloudDiscoveryState state = first.getDiscoveryState(); + assertNotNull(state); + String kept = state.toJson(); + + answers.put(PROPERTIES, UNREACHABLE); + answers.put(EVIDENCE_KEYS, UNREACHABLE); + requests.clear(); + CloudRequestEngine engine = builder() + .setDiscoveryState(CloudDiscoveryState.fromJson(kept)) + .build(); + + assertEquals(0, requests(PROPERTIES), "accessibleproperties"); + assertEquals(0, requests(EVIDENCE_KEYS), "evidencekeys"); + assertTrue(engine.getPublicProperties().containsKey("device")); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + assertEquals(0, warnings().size(), warnings().toString()); + + process(engine, false); + + assertEquals(0, requests(PROPERTIES), "accessibleproperties"); + assertEquals(0, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, requests(DATA), "json"); + } + + /** + * A build that does not know both lists gives no copy. + */ + @Test + public void DiscoveryState_NoneWithoutBothLists() throws Exception { + answers.put(EVIDENCE_KEYS, 503); + CloudRequestEngineBuilder builder = builder(); + builder.build(); + + assertNull(builder.getDiscoveryState()); + } + + private CloudRequestEngineBuilder builder() { + return new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .setTimeOutSeconds(TIMEOUT_SECONDS); + } + private CloudRequestEngine build() throws Exception { return new CloudRequestEngineBuilder(loggerFactory, httpClient) .setResourceKey("resource_key") From 4f62134b6d12ed6044fb630abe7aecbde07d5132 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:21:48 +0000 Subject: [PATCH 39/95] Test that discovery makes both requests at once, times out after 2 seconds, and that the Pipeline reads metadata again once it is known Each discovery request in the new test waits for the other to start, so they only succeed if they are made at once. A build with no timeout set applies 2 seconds to both, as the page's Configuration options table and .NET give. The Pipeline's map of available properties, built before the Cloud Request Engine knows the accessible properties, holds the cloud aspect engine's properties once they are known. The shared test client now gives each request a connection of its own, as the requests can be made at the same time. --- .../CloudAspectEngineBaseTests.java | 27 ++++++++ .../CloudRequestEngineDiscoveryTests.java | 67 ++++++++++++++++++- .../CloudRequestEngineTestsBase.java | 30 +++++---- 3 files changed, 108 insertions(+), 16 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 3ec05bb44..06f0b8505 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -344,6 +344,33 @@ public void loadProperties_notLoaded() throws Exception { assertTrue(propertiesContainName(engine.getProperties(), "ismobile")); } + /** + * The Pipeline's map of available properties, built before the Cloud + * Request Engine knows the accessible properties, holds the engine's + * properties once they are known rather than keeping what it first saw. + * @throws Exception + */ + @Test + public void elementAvailableProperties_ReadAgainOnceKnown() throws Exception { + propertiesReturnedByRequestEngine = null; + createPipeline(); + + assertTrue( + pipeline.getElementAvailableProperties().get("test") == null || + pipeline.getElementAvailableProperties().get("test").isEmpty()); + + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + Map products = new HashMap<>(); + products.put("test", product); + requestEngine.setPublicProperties(products); + + assertTrue(pipeline.getElementAvailableProperties().get("test") + .containsKey("ismobile")); + } + /** * A property keyed engine, such as the hardware profile engine, whose * product the resource key does not include builds, gives no profiles diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java index 742adaffc..a5bbc81fa 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -44,9 +44,13 @@ import java.net.SocketTimeoutException; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; @@ -94,11 +98,28 @@ public class CloudRequestEngineDiscoveryTests { private static final int TIMEOUT_SECONDS = 3; + /** + * How long a discovery request waits for the other one to start. + */ + private static final int WAIT_SECONDS = 5; + + /** + * The timeout the page's Configuration options table and .NET give. + */ + private static final int DEFAULT_TIMEOUT_SECONDS = 2; + private TestLoggerFactory loggerFactory; private HttpClient httpClient; - private final Map requests = new HashMap<>(); + private final Map requests = new ConcurrentHashMap<>(); private final Map answers = new HashMap<>(); - private final List connections = new ArrayList<>(); + private final List connections = + Collections.synchronizedList(new ArrayList()); + + /** + * When set, each discovery request waits until the other has started, + * so the two only complete if they are made at once. + */ + private volatile CountDownLatch bothDiscoveryRequests = null; @BeforeEach public void init() throws IOException { @@ -114,6 +135,14 @@ public void init() throws IOException { URL url = invocation.getArgument(0); String name = endpoint(url); requests.merge(name, 1, Integer::sum); + CountDownLatch latch = bothDiscoveryRequests; + if (latch != null && name.equals(DATA) == false) { + latch.countDown(); + if (latch.await(WAIT_SECONDS, TimeUnit.SECONDS) == false) { + throw new SocketTimeoutException( + "The other discovery request was not made"); + } + } int answer = answers.get(name); if (answer == UNREACHABLE) { throw new ConnectException("Connection refused"); @@ -388,6 +417,40 @@ private CloudRequestEngineBuilder builder() { .setTimeOutSeconds(TIMEOUT_SECONDS); } + /** + * The two discovery requests are made at once, as the page allows, so + * a service that does not answer delays the build by one timeout. Each + * request here waits for the other to start, so made one after the + * other they would both fail. + */ + @Test + public void Build_MakesBothRequestsAtOnce() throws Exception { + bothDiscoveryRequests = new CountDownLatch(2); + + CloudRequestEngine engine = build(); + + assertEquals(0, warnings().size(), warnings().toString()); + assertNotNull(engine.getPublicProperties()); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + } + + /** + * With no timeout set, the requests time out after 2 seconds. + */ + @Test + public void Build_DefaultTimeoutIsTwoSeconds() throws Exception { + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + + assertEquals(2, connections.size()); + for (HttpURLConnection connection : connections) { + verify(connection).setConnectTimeout( + DEFAULT_TIMEOUT_SECONDS * 1000); + verify(connection).setReadTimeout(DEFAULT_TIMEOUT_SECONDS * 1000); + } + } + private CloudRequestEngine build() throws Exception { return new CloudRequestEngineBuilder(loggerFactory, httpClient) .setResourceKey("resource_key") diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java index 0320d94e5..98d854117 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java @@ -69,24 +69,26 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { invocationOnMock.getMethod().getName() + "' should not have been called."); } }); - final HttpURLConnection connection = mock(HttpURLConnection.class, new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - throw new Exception("The method '" + - invocationOnMock.getMethod().getName() + "' should not have been called."); - } - }); - doNothing().when(connection).setConnectTimeout(anyInt()); - doNothing().when(connection).setReadTimeout(anyInt()); - Map> responseHeaders = new HashMap>(); - doReturn(responseHeaders).when(connection).getHeaderFields(); + // Each request gets a connection of its own, because the engine + // makes its two discovery requests at once. doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - URL url = (URL)invocationOnMock.getArgument(0); - doReturn(url).when(connection).getURL(); + URL url = (URL)invocationOnMock.getArgument(0); + HttpURLConnection connection = mock(HttpURLConnection.class, new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + throw new Exception("The method '" + + invocation.getMethod().getName() + "' should not have been called."); + } + }); + doNothing().when(connection).setConnectTimeout(anyInt()); + doNothing().when(connection).setReadTimeout(anyInt()); + Map> responseHeaders = new HashMap>(); + doReturn(responseHeaders).when(connection).getHeaderFields(); + doReturn(url).when(connection).getURL(); if (url.getPath().endsWith("properties")) { - propertiesEndPoint = url.getQuery(); + propertiesEndPoint = url.getQuery(); doReturn(accessiblePropertiesResponseCode).when(connection).getResponseCode(); } else { From bef79b0b0a66ee0d80e268230ac78483b8240231 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:26:00 +0000 Subject: [PATCH 40/95] Make the discovery requests at once, time out after 2 seconds by default, and read metadata again until it is loaded The two discovery requests are now made at once, as the specification allows, so a service that does not answer delays the build by one timeout rather than two. The evidence keys request runs on another thread while the accessible properties request runs on the building thread, and the build waits for both. The default timeout for each request to the cloud service is now 2 seconds, as the specification's Configuration options table gives and as .NET's CLOUD_REQUEST_TIMEOUT_DEFAULT_SECONDS is. It was 100 seconds. FlowElement gains hasLoadedProperties, true by default, as .NET's aspect engines have HasLoadedProperties. The Cloud Request Engine returns true once both discovery lists are known, the cloud aspect engines once their metadata is loaded, and a parallel group once all its elements have. PipelineDefault keeps its map of available properties only once every element has loaded its properties, and builds it again on each call until then, as .NET's Pipeline.ElementAvailableProperties does after a PropertiesNotYetLoadedException. The web integration works out the Vary headers again on each response until every element has loaded, as .NET's ClientsidePropertyService.HeadersAffectingJavaScript does after an exception from an element's EvidenceKeyFilter. --- .../DeviceDetectionCloudEngine.java | 11 +++++ .../IPIntelligenceCloudEngine.java | 11 +++++ .../flowelements/GeoLocationCloudEngine.java | 11 +++++ .../flowelements/CloudAspectEngineBase.java | 11 +++++ .../CloudRequestEngineBuilder.java | 5 ++- .../CloudRequestEngineDefault.java | 33 +++++++++++--- .../flowelements/Constants.java | 4 +- .../core/flowelements/FlowElement.java | 13 ++++++ .../core/flowelements/ParallelElements.java | 13 ++++++ .../core/flowelements/PipelineDefault.java | 44 +++++++++++++++++-- .../ClientsidePropertyServiceCore.java | 40 +++++++++++++++-- .../web/ClientsidePropertyServiceTests.java | 22 ++++++++++ 12 files changed, 200 insertions(+), 18 deletions(-) diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index e43fc4b24..400ca1dad 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -95,6 +95,17 @@ public List getProperties() { loaded; } + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return aspectProperties != null; + } + @Override public String getDataSourceTier() { return dataSourceTier; diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index bec4510d2..99b84cbb3 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -107,6 +107,17 @@ public List getProperties() { loaded; } + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return aspectProperties != null; + } + @Override public String getDataSourceTier() { return dataSourceTier; diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 243abac97..cde6a4932 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -76,6 +76,17 @@ public List getProperties() { loaded; } + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return properties != null; + } + @Override public String getDataSourceTier() { return dataSourceTier; diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index ee605233e..0d83d2b1a 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -169,6 +169,17 @@ public List getProperties() { return localRef; } + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return aspectProperties != null; + } + @Override public TypedKey getTypedDataKey() { if (typedKey == null) { diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index 5e23b4ea4..b0d420a0b 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -247,9 +247,10 @@ public CloudRequestEngineBuilder setLicenseKey(String licenseKey) { } /** - * Timeout in seconds for the request to the endpoint. + * Timeout in seconds for each request to the cloud service, the two + * discovery requests when the engine is built and each data request. *

    - * Default value is 100 seconds + * Default value is 2 seconds * @param timeout in seconds * @return this builder */ diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 3f1be59b5..0bf8338f9 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -51,6 +51,8 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -267,13 +269,12 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { public void discover() { List failures = new ArrayList<>(); synchronized (discoveryLock) { - if (evidenceKeyFilter.isKnown() == false) { - try { - fetchEvidenceKeys(); - } catch (RuntimeException e) { - failures.add(e); - } - } + // The two requests are made at once, so a service that does not + // answer delays the build by one timeout rather than two. + CompletableFuture evidenceKeys = + evidenceKeyFilter.isKnown() ? + CompletableFuture.completedFuture(null) : + CompletableFuture.runAsync(this::fetchEvidenceKeys); if (publicProperties == null) { try { fetchPublicProperties(); @@ -281,6 +282,13 @@ public void discover() { failures.add(e); } } + try { + evidenceKeys.join(); + } catch (CompletionException e) { + failures.add(e.getCause() instanceof RuntimeException ? + (RuntimeException) e.getCause() : + e); + } } if (failures.isEmpty()) { return; @@ -298,6 +306,17 @@ public void discover() { logger.warn(MessageDiscoveryDeferred, first); } + /** + * Whether discovery has fetched both the accepted evidence keys and the + * accessible properties, so that what reads them, such as the + * Pipeline's map of available properties, can keep what it reads. + * @return true once both are known + */ + @Override + public boolean hasLoadedProperties() { + return evidenceKeyFilter.isKnown() && publicProperties != null; + } + /** * A copy of what discovery fetched, which a later builder can use in * place of the discovery requests. diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java index f6300acfb..b9b26c6ae 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java @@ -28,6 +28,8 @@ public class Constants { "https://cloud.51degrees.com/api/v4"; // Environment variable name to set end point public static final String FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL"; - public static final int DEFAULT_TIMEOUT_MILLIS = 100000; + // Default timeout for each request to the cloud service, as the + // specification's Configuration options table gives. + public static final int DEFAULT_TIMEOUT_MILLIS = 2000; } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java index c27716d21..b03878a83 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java @@ -103,6 +103,19 @@ public interface FlowElement< */ boolean isConcurrent(); + /** + * Whether the element knows all the property metadata and accepted + * Evidence it will expose. An element that gets them from a remote + * service returns false until it has them, and anything that keeps what + * it reads from the element, such as the Pipeline's map of available + * properties, reads it again until then. + * @return true if the property metadata and accepted Evidence are + * complete + */ + default boolean hasLoadedProperties() { + return true; + } + /** * Indicates whether the element has been closed using the {@link #close()} * method, either explicitly or as a result of a 'try with resource'. diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java index 79b8e4148..b1a479b75 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java @@ -117,6 +117,19 @@ public boolean isConcurrent() { return true; } + /** + * @return true if every child element has loaded its properties + */ + @Override + public boolean hasLoadedProperties() { + for (FlowElement element : flowElements) { + if (element.hasLoadedProperties() == false) { + return false; + } + } + return true; + } + @Override protected void managedResourcesCleanup() { for (FlowElement element : flowElements) { diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java index f53cda4ad..302b51d94 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java @@ -71,9 +71,15 @@ class PipelineDefault implements PipelineInternal { * Map of {@link ElementPropertyMetaData}s keyed first on the data key of * the element which they belong to, then by the property name. */ - private final Map> + private volatile Map> elementAvailableProperties; + /** + * Held while the map of available properties is built after an element + * had not loaded its properties when the Pipeline was built. + */ + private final Object availablePropertiesLock = new Object(); + /** * Lock used for synchronised blocks. */ @@ -155,8 +161,7 @@ class PipelineDefault implements PipelineInternal { for (FlowElement element : this.flowElements) { isConcurrent = isConcurrent || element.isConcurrent(); } - this.elementAvailableProperties = - getElementAvailableProperties(flowElements); + getElementAvailableProperties(); logger.debug("Pipeline '" + hashCode() + "' created."); @@ -360,10 +365,41 @@ public List getFlowElements() { return Collections.unmodifiableList(flowElements); } + /** + * The available properties of each element. The map is kept once every + * element has loaded its properties. Until then, as after a cloud + * element could not reach its service when it was built, it is built + * again on each call, so it gains the properties once they are known. + * @return the available properties, keyed on element data key then + * property name + */ @Override public Map> getElementAvailableProperties() { - return Collections.unmodifiableMap(elementAvailableProperties); + Map> properties = + elementAvailableProperties; + if (properties == null) { + synchronized (availablePropertiesLock) { + properties = elementAvailableProperties; + if (properties == null) { + boolean loaded = allPropertiesLoaded(); + properties = getElementAvailableProperties(flowElements); + if (loaded) { + elementAvailableProperties = properties; + } + } + } + } + return Collections.unmodifiableMap(properties); + } + + private boolean allPropertiesLoaded() { + for (FlowElement element : flowElements) { + if (element.hasLoadedProperties() == false) { + return false; + } + } + return true; } public void close(boolean closing) throws Exception { diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java index 630ac847a..d51d2cfe9 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java @@ -108,6 +108,17 @@ private enum ContentTypes { */ protected Pipeline pipeline; + /** + * True once the Vary headers were worked out while every element + * had loaded its properties, so they can be kept. + */ + private volatile boolean headersComplete = false; + + /** + * Held while the Vary headers are worked out again. + */ + private final Object headersLock = new Object(); + /** * A list of all the HTTP headers that are requested evidence for * elements that populate JavaScript properties. @@ -142,10 +153,12 @@ public Default( */ @SuppressWarnings("rawtypes") protected final void init() { - headersAffectingJavaScript = new ArrayList<>(); + List headers = new ArrayList<>(); + boolean complete = true; // Get evidence filters for all elements. List filters = new ArrayList<>(); for (FlowElement element : pipeline.getFlowElements()) { + complete = complete && element.hasLoadedProperties(); filters.add(element.getEvidenceKeyFilter()); } @@ -160,13 +173,32 @@ protected final void init() { if (key.startsWith(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR) && hasControlChar(key) == false) { String header = key.substring(key.indexOf(EVIDENCE_SEPERATOR) + 1); - if(headersAffectingJavaScript.contains(header) == false){ - headersAffectingJavaScript.add(header); + if(headers.contains(header) == false){ + headers.add(header); } } } } } + headersAffectingJavaScript = headers; + headersComplete = complete; + } + + /** + * The headers for the Vary header. Until every element has loaded its + * properties, as after a cloud element could not reach its service + * when it was built, they are worked out again on each call, so they + * gain the cloud's headers once those are known. + */ + private List getHeadersAffectingJavaScript() { + if (headersComplete == false && pipeline != null) { + synchronized (headersLock) { + if (headersComplete == false) { + init(); + } + } + } + return headersAffectingJavaScript; } private boolean hasControlChar(String str) { @@ -269,7 +301,7 @@ private void setHeaders( response.setContentLength(contentLength); response.setStatus(200); response.setHeader("Cache-Control", stringJoin(cacheControl, ",")); - response.setHeader("Vary", stringJoin(headersAffectingJavaScript, ",")); + response.setHeader("Vary", stringJoin(getHeadersAffectingJavaScript(), ",")); response.setHeader("ETag", Integer.toString(hash)); response.setHeader("Access-Control-Allow-Origin", "*"); } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java index f819bd994..31c8f9d46 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java @@ -115,6 +115,28 @@ public void JavaScript_VaryHeader_UA() throws IOException { JavaScript_VaryHeader(elements, "User-Agent"); } + /** + * An element that has not loaded its properties, as a cloud element + * after a failure to reach its service when it was built, gains its + * headers in the Vary header once it has them. + */ + @Test + @SuppressWarnings("rawtypes") + public void JavaScript_VaryHeader_ReadAgainUntilLoaded() throws IOException { + FlowElement element = mock(FlowElement.class); + when(element.getEvidenceKeyFilter()).thenReturn( + new EvidenceKeyFilterWhitelist(new ArrayList()), + new EvidenceKeyFilterWhitelist(Arrays.asList("header.User-Agent"))); + when(element.hasLoadedProperties()).thenReturn(false, true); + List elements = new ArrayList<>(); + elements.add(element); + Configure(elements); + + service.serveJavascript(request, response); + + verify(response).setHeader(eq("Vary"), eq("User-Agent")); + } + @Test public void JavaScript_VaryHeader_UA_AND_SEC() throws IOException { List> elements = new ArrayList>(); From be26bcf32d6c5f7429e36a8d2b7791ef066fe542 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:32:22 +0000 Subject: [PATCH 41/95] Test that the SetHeaders element keeps its configuration once every element has loaded its properties With every element loaded, the configuration built from the Pipeline's available properties is built once and kept across requests. While an element has not loaded its properties, it is built again on each request, and kept once every element has loaded. Both fail on the code before the fix, where the configuration was built again on every request, because getOrDefault never stored it. --- .../flowelements/SetHeadersElementTests.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java index cd25d63ac..231fe1502 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java @@ -49,8 +49,11 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -234,6 +237,79 @@ public void setHeadersElement_MultipleProperties_SameHeader() throws Exception { } } + /** + * Once every element of the Pipeline has loaded its properties, the + * configuration built from the Pipeline's available properties is kept, + * so later requests do not build it again. + */ + @Test + public void setHeadersElement_ConfigKeptOnceLoaded() throws Exception { + FlowElement source = mock(FlowElement.class); + when(source.hasLoadedProperties()).thenReturn(true); + Pipeline mockPipeline = mockPipeline(source); + createElement(); + + for (int i = 0; i < 3; i++) { + element.processInternal(mockFlowData(mockPipeline)); + } + + verify(mockPipeline, times(1)).getElementAvailableProperties(); + } + + /** + * While an element has not loaded its properties, as a cloud element + * after a failure to reach its service when the Pipeline was built, the + * configuration is built again on each request, and kept once every + * element has loaded. + */ + @Test + public void setHeadersElement_ConfigBuiltAgainUntilLoaded() throws Exception { + FlowElement source = mock(FlowElement.class); + when(source.hasLoadedProperties()).thenReturn(false, true); + Pipeline mockPipeline = mockPipeline(source); + createElement(); + + for (int i = 0; i < 4; i++) { + element.processInternal(mockFlowData(mockPipeline)); + } + + verify(mockPipeline, times(2)).getElementAvailableProperties(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static Pipeline mockPipeline(FlowElement source) { + Pipeline mockPipeline = mock(Pipeline.class); + List elements = new ArrayList<>(); + elements.add(source); + when(mockPipeline.getFlowElements()).thenReturn(elements); + when(mockPipeline.getElementAvailableProperties()).thenReturn( + new HashMap>()); + return mockPipeline; + } + + private static FlowData mockFlowData(Pipeline mockPipeline) { + FlowData data = mock(FlowData.class); + when(data.getPipeline()).thenReturn(mockPipeline); + when(data.getOrAdd(anyString(), any())).thenReturn(mock(SetHeadersData.class)); + return data; + } + + private void createElement() { + element = new SetHeadersElement( + loggerFactory.getLogger(SetHeadersElement.class.getName()), + new ElementDataFactory() { + @Override + public SetHeadersData create( + FlowData flowData, + FlowElement flowElement) { + return new SetHeadersData( + loggerFactory.getLogger( + SetHeadersData.class.getName()), + flowData); + } + }); + } + private void createPipeline( Map propertyNameValues) throws Exception { sourceElement = new ActivePropertySourceElement( From 9f35ac909f4210aaa4ee9f0030e581ffbf7451fa Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:32:51 +0000 Subject: [PATCH 42/95] Keep the SetHeaders element's configuration once every element has loaded its properties The SetHeaders element built its configuration from the Pipeline's available properties on every request, because getOrDefault never stored it. It now keeps it once every element of the Pipeline has loaded its properties, and builds it again on each request until then, as after a cloud element could not reach its service when the Pipeline was built. So the configuration gains the cloud's SetHeader properties once a retry fetches them, and is then built once. This is the same rule the Pipeline's map of available properties and the web integration's Vary header follow. --- .../flowelements/SetHeadersElement.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java index 78d4c29ba..9bb05c6ea 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java @@ -38,6 +38,7 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectPropertyValue; @@ -101,8 +102,20 @@ protected void processInternal(FlowData data) throws Exception { PipelineConfig config; Pipeline pipeline = data.getPipeline(); if ((config = pipelineConfigs.get(pipeline)) == null) { + // The configuration is kept once every element has loaded its + // properties. Until then, as after a cloud element could not + // reach its service when the Pipeline was built, it is built + // again on each request, so it gains the SetHeader properties + // once they are known. + boolean loaded = allPropertiesLoaded(pipeline); config = populateConfig(pipeline); - config = pipelineConfigs.getOrDefault(pipeline, config); + if (loaded) { + PipelineConfig kept = + pipelineConfigs.putIfAbsent(pipeline, config); + if (kept != null) { + config = kept; + } + } } SetHeadersData elementData = data.getOrAdd( @@ -139,6 +152,21 @@ protected void unmanagedResourcesCleanup() { // Nothing to clean up here } + /** + * @param pipeline the Pipeline + * @return true if every element of the Pipeline has loaded its + * properties + */ + @SuppressWarnings("rawtypes") + private static boolean allPropertiesLoaded(Pipeline pipeline) { + for (FlowElement element : pipeline.getFlowElements()) { + if (element.hasLoadedProperties() == false) { + return false; + } + } + return true; + } + /** * Build the response headers from the SetHeader properties in the FlowData. * @param data flowData object From ea5fc04454c85cb11ae384b121123167e523c486 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:45:31 +0000 Subject: [PATCH 43/95] Give the SetHeaders test mocks their type parameters, as the build treats raw types as errors --- .../engines/fiftyone/flowelements/SetHeadersElementTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java index 231fe1502..93029030b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java @@ -244,7 +244,7 @@ public void setHeadersElement_MultipleProperties_SameHeader() throws Exception { */ @Test public void setHeadersElement_ConfigKeptOnceLoaded() throws Exception { - FlowElement source = mock(FlowElement.class); + FlowElement source = mock(FlowElement.class); when(source.hasLoadedProperties()).thenReturn(true); Pipeline mockPipeline = mockPipeline(source); createElement(); @@ -264,7 +264,7 @@ public void setHeadersElement_ConfigKeptOnceLoaded() throws Exception { */ @Test public void setHeadersElement_ConfigBuiltAgainUntilLoaded() throws Exception { - FlowElement source = mock(FlowElement.class); + FlowElement source = mock(FlowElement.class); when(source.hasLoadedProperties()).thenReturn(false, true); Pipeline mockPipeline = mockPipeline(source); createElement(); From f600e1df641176b69088ca3990c678dfe9ddf04f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:00:40 +0000 Subject: [PATCH 44/95] Mark the mocked product map unchecked, as the build treats the varargs warning as an error --- .../devicedetection/cloud/flowelements/ProductMissingTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java index 37c1ec6d5..3c9ef7c1f 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -118,6 +118,8 @@ public void ProductMissing_PropertyIsMissing() throws Exception { * An engine whose Cloud Request Engine reports the accessible products * of a resource key that includes IP Intelligence alone. */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") private DeviceDetectionCloudEngine buildEngine() throws Exception { DeviceDetectionCloudEngine engine = new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); From efbeed089a843905b5e4c9d318cedae67878d7bd Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:00:43 +0000 Subject: [PATCH 45/95] Mark the mocked product map unchecked, as the build treats the varargs warning as an error --- .../ipintelligence/cloud/flowelements/ProductMissingTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index df07df779..3ee13582f 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -118,6 +118,8 @@ public void ProductMissing_PropertyIsMissing() throws Exception { * An engine whose Cloud Request Engine reports the accessible products * of a resource key that includes Device Detection alone. */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") private IPIntelligenceCloudEngine buildEngine() throws Exception { IPIntelligenceCloudEngine engine = new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); From 33a13fe06418f5154b436713be7bda43e0acf2f9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:00:46 +0000 Subject: [PATCH 46/95] Mark the mocked product map unchecked, as the build treats the varargs warning as an error --- .../geolocation/flowelements/GeoLocationCloudEngineTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java index 66f40b96e..80dadab55 100644 --- a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -120,6 +120,8 @@ public void ProductMissing_PropertyIsMissing() throws Exception { * An engine whose Cloud Request Engine reports the accessible products * of a resource key that includes IP Intelligence alone. */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") private GeoLocationCloudEngine buildEngine() throws Exception { GeoLocationCloudEngine engine = new GeoLocationCloudEngineBuilder(loggerFactory) From 8f709b4bccdfa707c6a78ddf90a0637177c7561b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:24:30 +0000 Subject: [PATCH 47/95] Stub the product map once per call, so no varargs array is created thenReturn(null, products) uses the varargs overload, and a generic Map array is an unchecked warning that the build treats as an error, so each stub now chains two single-value thenReturn calls. --- .../cloud/flowelements/ProductMissingTests.java | 4 +++- .../cloud/flowelements/ProductMissingTests.java | 4 +++- .../geolocation/flowelements/GeoLocationCloudEngineTests.java | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java index 93133bd52..8528c6367 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -129,7 +129,9 @@ public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { Map products = new HashMap<>(); products.put("device", product); CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); - when(requestEngine.getPublicProperties()).thenReturn(null, products); + when(requestEngine.getPublicProperties()) + .thenReturn(null) + .thenReturn(products); Pipeline pipeline = mock(Pipeline.class); when(pipeline.getElement(CloudRequestEngine.class)) .thenReturn(requestEngine); diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index eed28895f..53be6a7bf 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -129,7 +129,9 @@ public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { Map products = new HashMap<>(); products.put("ip", product); CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); - when(requestEngine.getPublicProperties()).thenReturn(null, products); + when(requestEngine.getPublicProperties()) + .thenReturn(null) + .thenReturn(products); Pipeline pipeline = mock(Pipeline.class); when(pipeline.getElement(CloudRequestEngine.class)) .thenReturn(requestEngine); diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java index 3bd56083d..8c36cc1d3 100644 --- a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -131,7 +131,9 @@ public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { Map products = new HashMap<>(); products.put("location", product); CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); - when(requestEngine.getPublicProperties()).thenReturn(null, products); + when(requestEngine.getPublicProperties()) + .thenReturn(null) + .thenReturn(products); Pipeline pipeline = mock(Pipeline.class); when(pipeline.getElement(CloudRequestEngine.class)) .thenReturn(requestEngine); From 40452b7d5d2a5a4de52ab482ccfa56dbd73f70a3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:41:20 +0000 Subject: [PATCH 48/95] Test that a 408 or 429 at discovery is transient and does not fail the build The specification now treats a 408 or a 429 answer to a discovery request as a transient failure, because the same request can succeed later, so 429 moves from the cases that fail the build to those that do not, and 408 cases join it. --- .../CloudRequestEngineDiscoveryTests.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java index a5bbc81fa..654273c61 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -191,7 +191,7 @@ public void Build_FetchesBothLists() throws Exception { private static Stream definitiveFailures() { List arguments = new ArrayList<>(); - for (int status : new int[] { 400, 401, 403, 404, 429 }) { + for (int status : new int[] { 400, 401, 403, 404 }) { arguments.add(Arguments.of(PROPERTIES, status)); arguments.add(Arguments.of(EVIDENCE_KEYS, status)); } @@ -199,8 +199,9 @@ private static Stream definitiveFailures() { } /** - * A 4xx answer to either request is a definitive configuration error, - * which fails the build once both requests have been made. + * A 4xx answer to either request, other than 408 or 429, is a + * definitive configuration error, which fails the build once both + * requests have been made. */ @ParameterizedTest(name = "{1} from {0}") @MethodSource("definitiveFailures") @@ -235,6 +236,10 @@ private static Stream transientFailures() { Arguments.of("500 from accessibleproperties", 500, 200), Arguments.of("503 from evidencekeys", 200, 503), Arguments.of("502 from both", 502, 502), + Arguments.of("408 from accessibleproperties", 408, 200), + Arguments.of("408 from evidencekeys", 200, 408), + Arguments.of("429 from accessibleproperties", 429, 200), + Arguments.of("429 from evidencekeys", 200, 429), Arguments.of("unreadable accessibleproperties", UNREADABLE, 200), Arguments.of("unreadable evidencekeys", 200, UNREADABLE), Arguments.of("connection refused to both", UNREACHABLE, UNREACHABLE), @@ -242,9 +247,9 @@ private static Stream transientFailures() { } /** - * Any other failure, the service unreachable, a timeout, a 5xx answer or - * a body that cannot be read, is logged as one warning and the build - * completes. + * Any other failure, the service unreachable, a timeout, a 5xx, 408 or + * 429 answer or a body that cannot be read, is logged as one warning and + * the build completes. */ @ParameterizedTest(name = "{0}") @MethodSource("transientFailures") From 7bc481c8fc509ccbea0eb3dbc9f6138bbe3daede Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:42:02 +0000 Subject: [PATCH 49/95] Treat a 408 or 429 at discovery as transient, so it does not fail the build A build fails on a 4xx answer because no retry with the same configuration can succeed, which is not so for a 408 or a 429, as RFC 9110 section 15.5.9 and RFC 6585 section 4 say, so those two are logged as a warning and retried on first use like a 5xx. --- .../CloudRequestEngineBuilder.java | 4 +- .../CloudRequestEngineDefault.java | 44 +++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index b0d420a0b..5530411db 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -118,8 +118,8 @@ protected CloudRequestEngine newEngine(List properties) throws Exception else { // Fetch the accessible properties and the accepted evidence // keys now, so that the built engine is ready. A 4xx answer - // fails the build, and any other failure is retried on first - // use. + // other than 408 or 429 fails the build, and any other failure + // is retried on first use. engine.discover(); } built = engine; diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 0bf8338f9..5c95377f6 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -87,11 +87,24 @@ public class CloudRequestEngineDefault /** * The lowest HTTP status of a client error. A discovery request answered - * with a status from this up to {@link #HTTP_SERVER_ERROR_MIN} is a - * definitive configuration error. + * with a status from this up to {@link #HTTP_SERVER_ERROR_MIN}, other + * than {@link #HTTP_REQUEST_TIMEOUT} or {@link #HTTP_TOO_MANY_REQUESTS}, + * is a definitive configuration error. */ private static final int HTTP_CLIENT_ERROR_MIN = 400; + /** + * Request Timeout, a client error that a later retry can pass, as RFC + * 9110 section 15.5.9 says. + */ + private static final int HTTP_REQUEST_TIMEOUT = 408; + + /** + * Too Many Requests, a client error that a later retry can pass, as RFC + * 6585 section 4 says. + */ + private static final int HTTP_TOO_MANY_REQUESTS = 429; + /** * The lowest HTTP status of a server error. */ @@ -257,14 +270,15 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { * Fetch the accepted Evidence keys and the accessible properties that * are not known yet. The builder calls this when it builds the engine, * so in the normal case a built engine is ready to process. Both - * requests are made. A 4xx answer to either is a definitive - * configuration error, and its {@link CloudRequestException} is thrown, - * because retrying with the same configuration can never succeed. Any - * other failure, such as the service being unreachable, a timeout, a - * 5xx answer or a body that cannot be read, is logged as one warning - * and the list that did arrive is kept. Processing fetches what is - * still missing. - * @throws CloudRequestException for a 4xx answer to either request + * requests are made. A 4xx answer to either, other than 408 or 429, is + * a definitive configuration error, and its + * {@link CloudRequestException} is thrown, because retrying with the + * same configuration can never succeed. Any other failure, such as the + * service being unreachable, a timeout, a 5xx, 408 or 429 answer or a + * body that cannot be read, is logged as one warning and the list that + * did arrive is kept. Processing fetches what is still missing. + * @throws CloudRequestException for a 4xx answer to either request, + * other than 408 or 429 */ public void discover() { List failures = new ArrayList<>(); @@ -346,16 +360,18 @@ public void useDiscoveryState(CloudDiscoveryState state) { /** * The failure of a discovery request that the cloud service received - * and rejected with a 4xx status, so that retrying with the same - * configuration can never succeed. + * and rejected with a 4xx status other than 408 or 429, so that + * retrying with the same configuration can never succeed. * @param failure the failure of a discovery request - * @return the exception with the 4xx status, or null if there is none + * @return the exception with that status, or null if there is none */ private static CloudRequestException definitiveError(Throwable failure) { if (failure instanceof CloudRequestException) { int status = ((CloudRequestException) failure).getHttpStatusCode(); return status >= HTTP_CLIENT_ERROR_MIN && - status < HTTP_SERVER_ERROR_MIN ? + status < HTTP_SERVER_ERROR_MIN && + status != HTTP_REQUEST_TIMEOUT && + status != HTTP_TOO_MANY_REQUESTS ? (CloudRequestException) failure : null; } From 059c0dfc25c29b6c2f256b2bbe47f8d4f79a979e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 22:42:17 +0000 Subject: [PATCH 50/95] Check the callback URL as the specification's six rules say (#77) * Test the callback URL against the specification's six rules Adds the refused, accepted and logged cases from the JavaScript builder specification's callback URL section, and removes from the accepted hosts every host the section now refuses. The new cases fail until the check follows the rules. * Check the callback URL as the specification's six rules say The host value is checked as written. A host holding '?', '#' or '@' is refused, as is an IPv6 zone identifier and a name whose last label is a number but which is not an IPv4 address in RFC 3986 form. The URL kept for the log shows *** in place of everything before the host value's last '@'. * Test that a port that is present but empty is refused Adds example.com:, example.com:/path, [::1]:, https://example.com and http://example.com to the refused hosts, and the log case for the last protocol-in-host mistake, and removes example.com: from the accepted hosts. * Refuse a port that is present but empty A host such as https://example.com otherwise reads as the host https with an empty port, and gives https://https://example.com/... with nothing logged. RFC 3986 section 3.2.3 allows an empty port in the syntax, and section 6.2.3 notes that a URI with an empty port is equivalent to one with none, so the host is written without the colon. * Test hosts that end in a line break and more IPv6 forms A trailing line break after a host, a port or a bracketed address is refused, and 1:2:3:4:5:6:7:8 and ::1.2.3.4 are accepted whilst 1::2::3 and 12345::1 are refused. * Say the callback URL comment covers text after the authority --- .../flowelements/CallbackUrl.java | 124 ++++++------- .../javascriptbuilder/JavaScriptUrlTests.java | 32 +++- .../flowelements/CallbackUrlTests.java | 164 ++++++++++++++---- 3 files changed, 224 insertions(+), 96 deletions(-) diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java index c6e76aa65..c9118d980 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java @@ -31,9 +31,11 @@ * and host usually come from the request, and the script holds the URL in a * single-quoted string, so the URL is checked before it is written into the * script. A URL that is not valid, including one whose protocol is not http - * or https, is refused. In the user information, in anything after the host - * and port, and in the endpoint, the characters a URL cannot hold or that - * could end the string or the script are percent-encoded. + * or https, is refused. In anything after the authority, and in the + * endpoint, the characters a URL cannot hold or that could end the string or + * the script are percent-encoded. The rules are applied to the text given, + * and not to the result of parsing it, because a parser can accept hosts the + * rules refuse and rewrite others, such as 127.1. * @see Specification */ final class CallbackUrl { @@ -78,20 +80,21 @@ final class CallbackUrl { private static final String[] SCHEMES = {"http", "https"}; /** - * A label of a host name, being ASCII letters, digits, '-' and '_'. + * The characters the host value cannot hold. A '?' or '#' would make the + * endpoint part of the query or fragment, and an '@' means user + * information, which fetch refuses and XMLHttpRequest sends. */ - private static final Pattern LABEL = Pattern.compile("[A-Za-z0-9_-]+"); + private static final String NOT_IN_HOST = "?#@"; /** - * An IPv6 zone identifier, being unreserved characters as RFC 3986 - * section 2.3 defines them. + * A label of a host name, being ASCII letters, digits, '-' and '_'. */ - private static final Pattern ZONE = Pattern.compile("[A-Za-z0-9._~-]+"); + private static final Pattern LABEL = Pattern.compile("[A-Za-z0-9_-]+"); /** - * A port as it follows the host, being ':' and any number of digits. + * A port as it follows the host, being ':' and at least one digit. */ - private static final Pattern PORT = Pattern.compile(":[0-9]*"); + private static final Pattern PORT = Pattern.compile(":[0-9]+"); /** * One group of an IPv6 address. @@ -107,6 +110,13 @@ final class CallbackUrl { "(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])" + "(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}"); + /** + * A label that the URL Standard reads as a number, being decimal digits + * or "0x" followed by hexadecimal digits. + */ + private static final Pattern NUMBER = + Pattern.compile("[0-9]+|0[xX][0-9A-Fa-f]*"); + /** * Whether the URL can be written into the script. */ @@ -114,7 +124,8 @@ final class CallbackUrl { /** * The URL to write into the script when it is valid, and otherwise the - * URL as it was built, for the log. + * URL as it was built, for the log, with *** in place of everything in + * the host value before its last '@'. */ final String url; @@ -129,8 +140,7 @@ private CallbackUrl(boolean valid, String url) { * and removed when both do. The protocol and the host are written in * lower case and the rest as given, apart from the percent-encoding. * @param protocol the protocol, http or https in any case - * @param host the host, which can start with user information and end - * with a port, a path, a query or a fragment + * @param host the host, which can end with a port and a path * @param endpoint the endpoint * @return the callback URL, or null when the protocol, host or endpoint * is not set @@ -146,14 +156,14 @@ static CallbackUrl of(String protocol, String host, String endpoint) { } else if (hostSlash && endpointSlash) { endpoint = endpoint.substring(1); } - String built = protocol + "://" + host + endpoint; - if (isScheme(protocol) == false) { - return new CallbackUrl(false, built); - } int end = authorityEnd(host); - String authority = checkedAuthority(host.substring(0, end)); + String authority = isScheme(protocol) && hasNoForbiddenCharacter(host) + ? checkedHostPort(host.substring(0, end)) + : null; if (authority == null) { - return new CallbackUrl(false, built); + return new CallbackUrl( + false, + protocol + "://" + withoutUserInformation(host) + endpoint); } return new CallbackUrl( true, @@ -161,6 +171,28 @@ static CallbackUrl of(String protocol, String host, String endpoint) { encoded(host.substring(end)) + encoded(endpoint)); } + /** + * Whether the host value holds none of '?', '#' and '@'. + */ + private static boolean hasNoForbiddenCharacter(String host) { + for (int i = 0; i < host.length(); i++) { + if (NOT_IN_HOST.indexOf(host.charAt(i)) >= 0) { + return false; + } + } + return true; + } + + /** + * The host value with everything before its last '@' replaced by ***. + * The last '@' of the whole value is used, and not that of the + * authority, because a password can hold a '/', '?' or '#'. + */ + private static String withoutUserInformation(String host) { + int at = host.lastIndexOf('@'); + return at < 0 ? host : "***" + host.substring(at); + } + /** * Whether the protocol is http or https, ignoring the case of its ASCII * letters only. String.equalsIgnoreCase also matches a letter outside @@ -208,41 +240,20 @@ static boolean isBlank(String text) { } /** - * Where the authority ends, which is at the first '/', '?' or '#' of the - * host, or at its end. + * Where the authority ends, which is at the first '/' of the host, or at + * its end. */ private static int authorityEnd(String host) { - for (int i = 0; i < host.length(); i++) { - char c = host.charAt(i); - if (c == '/' || c == '?' || c == '#') { - return i; - } - } - return host.length(); - } - - /** - * The user information, host and port as they are written into the URL, - * or null when they are not valid. The user information ends at the - * first '@', so a second '@' is part of the host and refused. - */ - private static String checkedAuthority(String authority) { - int at = authority.indexOf('@'); - if (at < 0) { - return checkedHostPort(authority); - } - String hostPort = checkedHostPort(authority.substring(at + 1)); - return hostPort == null - ? null - : encoded(authority.substring(0, at)) + "@" + hostPort; + int end = host.indexOf('/'); + return end < 0 ? host.length() : end; } /** * The host and optional port in lower case, or null when they are not - * valid. The host is an IPv6 address in brackets with an optional zone - * identifier, or a name of labels separated by dots with a dot allowed - * at the end. The port is ':' followed by digits up to 65535, or by - * nothing. + * valid. The host is an IPv6 address in brackets, or a name of labels + * separated by dots with a dot allowed at the end, which must be an IPv4 + * address when its last label is a number. The port is ':' followed by + * digits up to 65535. */ private static String checkedHostPort(String hostPort) { String host; @@ -253,14 +264,6 @@ private static String checkedHostPort(String hostPort) { return null; } String address = hostPort.substring(1, close); - int zone = address.indexOf("%25"); - if (zone >= 0) { - if (ZONE.matcher(address.substring(zone + 3)).matches() == - false) { - return null; - } - address = address.substring(0, zone); - } if (isIpv6(address) == false) { return null; } @@ -274,11 +277,16 @@ private static String checkedHostPort(String hostPort) { String name = host.endsWith(".") ? host.substring(0, host.length() - 1) : host; - for (String label : name.split("\\.", -1)) { + String[] labels = name.split("\\.", -1); + for (String label : labels) { if (LABEL.matcher(label).matches() == false) { return null; } } + if (NUMBER.matcher(labels[labels.length - 1]).matches() && + IPV4.matcher(host).matches() == false) { + return null; + } } if (port.isEmpty() == false && isPort(port) == false) { return null; @@ -288,7 +296,7 @@ private static String checkedHostPort(String hostPort) { /** * Whether the text is ':' followed by digits whose value is no more than - * 65535, or by nothing. + * 65535. */ private static boolean isPort(String text) { if (PORT.matcher(text).matches() == false) { diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index 67006a697..fc8cb2f61 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -34,6 +34,7 @@ import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.ILoggerFactory; import org.slf4j.Logger; @@ -230,6 +231,34 @@ public void Url_RefusedHost_NoCallbackAndLoggedOnce() throws Exception { errors.get(0)); } + /** + * A refused host is logged with *** in place of everything in the host + * value before its last '@', so a password is never written to the log. + * Each case has its own element, because a refusal is logged once per + * element. + */ + @ParameterizedTest(name = "[{index}]") + @CsvSource(delimiter = '|', value = { + "user:secret@example.com|\"https://***@example.com/json\"", + "a@b@example.com|\"https://***@example.com/json\"", + "u@[::1]:80x|\"https://***@[::1]:80x/json\"", + "user:pa/ss@example.com|\"https://***@example.com/json\"", + "user:pa?ss@example.com|\"https://***@example.com/json\"", + "exa mple.com/a@b|\"https://***@b/json\"", + "example.123|\"https://example.123/json\"", + "https://example.com|\"https://https://example.com/json\""}) + public void Url_RefusedHost_LoggedWithoutUserInformation( + String host, String logged) throws Exception { + renderWithHosts(host); + + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + assertTrue(errors.get(0).contains(logged), errors.get(0)); + for (String secret : new String[] {"secret", "pa/", "pa?", "ss@"}) { + assertFalse(errors.get(0).contains(secret), errors.get(0)); + } + } + /** * A protocol from the request that is neither http nor https gives a * script with no callback, and is logged once like any other URL that @@ -273,7 +302,8 @@ public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { * refused too, which String.equalsIgnoreCase would match to 's'. */ @ParameterizedTest - @ValueSource(strings = {"ftp", "ws", "javascript", "https ", "http\u017f"}) + @ValueSource(strings = {"ftp", "ws", "javascript", "https ", "http\u017f", + " ", "\t", "\u00a0"}) public void Protocol_ConfiguredOtherThanHttpOrHttps_IsRefused( String protocol) { JavaScriptBuilderElementBuilder builder = diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java index b4be236ef..12b7e3c9f 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java @@ -99,8 +99,6 @@ static Stream acceptedHosts() { "https://example.com/51dpipeline/json"), Arguments.of("example.com:8080", "https://example.com:8080/51dpipeline/json"), - Arguments.of("example.com:", - "https://example.com:/51dpipeline/json"), Arguments.of("example.com:65535", "https://example.com:65535/51dpipeline/json"), Arguments.of("a_b.com", @@ -109,26 +107,38 @@ static Stream acceptedHosts() { "https://-ab.com/51dpipeline/json"), Arguments.of("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + Arguments.of("example.com", + "https://example.com/51dpipeline/json"), + Arguments.of("1.2.3.4", + "https://1.2.3.4/51dpipeline/json"), + Arguments.of("1.2.3.4:8080", + "https://1.2.3.4:8080/51dpipeline/json"), + Arguments.of("1.2.3.4/path", + "https://1.2.3.4/path/51dpipeline/json"), + Arguments.of("0.0.0.0", + "https://0.0.0.0/51dpipeline/json"), + Arguments.of("101.2.3.4", + "https://101.2.3.4/51dpipeline/json"), + Arguments.of("example.co1", + "https://example.co1/51dpipeline/json"), + Arguments.of("123.example.com", + "https://123.example.com/51dpipeline/json"), Arguments.of("[::1]", "https://[::1]/51dpipeline/json"), Arguments.of("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), - Arguments.of("[fe80::1%25eth0]", - "https://[fe80::1%25eth0]/51dpipeline/json"), - Arguments.of("user:pw@example.com", - "https://user:pw@example.com/51dpipeline/json"), - Arguments.of("u x@example.com", - "https://u%20x@example.com/51dpipeline/json"), - Arguments.of("u\tx@example.com", - "https://u%09x@example.com/51dpipeline/json"), + Arguments.of("[1:2:3:4:5:6:7:8]", + "https://[1:2:3:4:5:6:7:8]/51dpipeline/json"), + Arguments.of("[::1.2.3.4]", + "https://[::1.2.3.4]/51dpipeline/json"), Arguments.of("example.com.", "https://example.com./51dpipeline/json"), Arguments.of("example.com/path", "https://example.com/path/51dpipeline/json"), - Arguments.of("example.com?x", - "https://example.com?x/51dpipeline/json"), Arguments.of("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), + Arguments.of("example.com/a\"b", + "https://example.com/a%22b/51dpipeline/json"), Arguments.of("example.com/a%zzb", "https://example.com/a%25zzb/51dpipeline/json"), Arguments.of("example.com/a%41b", @@ -183,7 +193,63 @@ static Stream refusedHosts() { "a\nb.com", " example.com", "example.com ", - "example.com:8080');alert(1);//"); + "example.com\n", + "example.com:80\n", + "[::1]\n", + "example.com:8080');alert(1);//", + // User information, or an @ anywhere in the host value. + "user@example.com", + "user:pw@example.com", + "@example.com", + ":@example.com", + "user@example.com:8080", + "user@[::1]", + "user@example.com/path", + "u%41@example.com", + "u x@example.com", + "u\tx@example.com", + "u\"x@example.com", + "u'x@example.com", + "u\\x@example.com", + "u\nx@example.com", + "u\u0001x@example.com", + "example.com/a@b", + "user:12/ss@example.com", + "user:pa/ss@example.com", + "user:pa?ss@example.com", + "exa mple.com/a@b", + // A query or fragment. + "example.com?x", + "example.com#x", + "example.com?", + "example.com#", + "example.com:8080?x", + // A port that is present but empty. + "example.com:", + "example.com:/path", + "[::1]:", + "https://example.com", + "http://example.com", + "[::1]#x", + "example.com/path?x", + "example.com/path#x", + "example.com?');alert(1);//", + "example.com#');alert(1);//", + // A name whose last label is a number must be an IPv4 address. + "example.123", + "example.0x1f", + "example.0X", + "1.2.3.999", + "1.2.3", + "127.1", + "0x7f.0.0.1", + "01.2.3.4", + "example.123.", + "1.2.3.4.", + "1.2.3.4.5", + "1", + "300.1.1.1", + "0x7f.1"); } @ParameterizedTest(name = "[{index}]") @@ -195,20 +261,27 @@ public void CallbackUrl_RefusesHost(String host) { static Stream refusedZonesAndTextAfterTheBracket() { return Stream.of( "[fe80::1%25]", + "[fe80::1%25eth0]", + "[fe80::1%25eth0]:8080", + "[FE80::1%25ETH0]", + "[fe80::1%251]", + "[::1%25lo]", + "[fe80::1%]", + "[fe80::1%25eth0]/path", "[fe80::1%25'+alert(1)+']", "[fe80::1%eth0]", "[fe80::1%25eth%0]", "[::1]x", "[::1]80", "[::1]:80x", - "[::1]]"); + "[::1]]", + "[1::2::3]", + "[12345::1]"); } /** - * Inside the brackets only an IPv6 address, optionally followed by '%25' - * and a zone of unreserved characters, and after them only a port. The - * zone check is what keeps a quote out of the script for a bracketed - * host. + * Inside the brackets only an IPv6 address, so a zone identifier is + * refused, and after them only a port. */ @ParameterizedTest(name = "[{index}]") @MethodSource("refusedZonesAndTextAfterTheBracket") @@ -216,14 +289,6 @@ public void CallbackUrl_RefusesZoneOrTextAfterTheBracket(String host) { assertInvalid(url("https", host), "host " + host); } - @Test - public void CallbackUrl_AcceptsZoneFollowedByPort() { - assertValid( - "https://[fe80::1%25eth0]:8080/51dpipeline/json", - url("https", "[fe80::1%25eth0]:8080"), - "host [fe80::1%25eth0]:8080"); - } - /** * A port goes by its value, so leading zeros are allowed, and a port too * long to read is refused rather than read as small enough. @@ -248,17 +313,14 @@ public void CallbackUrl_ChecksThePortByItsValue() { } /** - * Each character rule 8 lists, in the user information, after the host - * and in the endpoint, and a '%' without two hexadecimal digits after - * it, even at the end. + * Each character rule 8 lists, after the host and in the endpoint, and a + * '%' without two hexadecimal digits after it, even at the end. */ @Test public void CallbackUrl_EncodesTheCharactersRule8Lists() { assertValid( - "https://u%22%5E%60%7B%7C%7D@example.com/%22%5E%60%7B%7C%7D" + - "/j%22%5E%60%7B%7C%7D", - CallbackUrl.of( - "https", "u\"^`{|}@example.com/\"^`{|}", "/j\"^`{|}"), + "https://example.com/%22%5E%60%7B%7C%7D/j%22%5E%60%7B%7C%7D", + CallbackUrl.of("https", "example.com/\"^`{|}", "/j\"^`{|}"), "rule 8 characters"); assertValid( "https://example.com/a%25/51dpipeline/json", @@ -278,6 +340,37 @@ public void CallbackUrl_EncodesTheCharactersRule8Lists() { "endpoint /json%"); } + static Stream refusedUrlsForTheLog() { + return Stream.of( + Arguments.of("user:secret@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("a@b@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("u@[::1]:80x", + "https://***@[::1]:80x/51dpipeline/json"), + Arguments.of("user:pa/ss@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("user:pa?ss@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("exa mple.com/a@b", + "https://***@b/51dpipeline/json"), + Arguments.of("@example.com", + "https://***@example.com/51dpipeline/json")); + } + + /** + * The URL kept for the log shows *** in place of everything in the host + * value before its last '@'. + */ + @ParameterizedTest(name = "[{index}]") + @MethodSource("refusedUrlsForTheLog") + public void CallbackUrl_RefusedUrlForTheLogHasNoUserInformation( + String host, String expected) { + CallbackUrl url = url("https", host); + assertInvalid(url, "host " + host); + assertEquals(expected, url.url, "host " + host); + } + @Test public void CallbackUrl_InvalidUrlIsTheOneBuilt() { CallbackUrl url = url("https", "exa mple.com"); @@ -317,10 +410,7 @@ public void CallbackUrl_RefusesScheme(String protocol) { static Stream hostsThatCouldEndTheString() { return Stream.of( "example.com/');alert(1);//", - "example.com?');alert(1);//", - "example.com#');alert(1);//", - "u'x@example.com", - "u\\x@example.com", + "example.com/a\"b", "example.com/a\\b", "example.com/"); } From 984607670449d53dca664817fc796763ae00b7a5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 23:08:30 +0000 Subject: [PATCH 51/95] Write the usage sharing record as the specification says (#81) * Test the usage sharing record against the specification The record marks invalid characters with escaped, marks a long value with truncated, follows the order of the specification's example and uses neither CDATA nor BadSchema. * Write the usage sharing record as the specification says Mark an Evidence value that had invalid characters replaced with the escaped attribute, where it had a replaced attribute, and write the elements in the order of the specification's example. * Test the usage sharing record's escaped characters and 10,000 character limit An invalid character is written as backslash, x and four lower-case hexadecimal digits, a value is cut at 10,000 characters without splitting a surrogate pair, and every character XML 1.0 allows is kept. * Write invalid characters as an escape and cut values at 10,000 characters A character XML 1.0 does not allow is written as backslash, x and its code as four lower-case hexadecimal digits, where it was replaced with U+FFFD. The limit is 10,000 characters, counted as characters so that a pair of surrogates is never split, where it was 512 UTF-16 code units. * Test that the 10,000 character limit counts the value before its characters are escaped --- .../fiftyone/flowelements/Constants.java | 6 +- .../flowelements/ShareUsageElement.java | 93 ++++++------ .../flowelements/ReplacedStringTest.java | 137 +++++++++++++++--- .../flowelements/ShareUsageElementTests.java | 101 ++++++++++++- 4 files changed, 265 insertions(+), 72 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index f53a7bf29..b098ee136 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -28,10 +28,10 @@ public class Constants { /** - * The maximum length of a piece of evidence's value which can be added to - * the usage data being sent. + * The maximum length, in characters, of a piece of evidence's value which + * can be added to the usage data being sent. */ - public static final int SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512; + public static final int SHARE_USAGE_MAX_EVIDENCE_LENGTH = 10000; /** * Session id evidence suffix. This is prefixed by the query constant. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index c56e032f8..6fb71eae7 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -40,9 +40,6 @@ import java.net.URI; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; import java.util.zip.GZIPOutputStream; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT; @@ -422,7 +419,13 @@ private void streamXml(List allData, OutputStream os) throws XML */ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws XMLStreamException { writer.writeStartElement("Device"); - // --- write invariant data + // The elements follow the order of the specification's example. + // The SessionID used to track a series of requests + writeXmlElement(writer, "SessionId", data.sessionId); + // The sequence number of the request in a series of requests. + writeXmlElement(writer, "Sequence", String.valueOf(data.sequence)); + // The UTC date/time this entry was written + writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date())); // The version number of the Pipeline API writeXmlElement(writer, "Version", coreVersion); // Write Pipeline information @@ -435,21 +438,13 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws writeXmlElement(writer, "Language", "java"); // The software language version writeXmlElement(writer, "LanguageVersion", languageVersion); + // The client IP of the request + writeXmlElement(writer, "ClientIP", data.clientIP); // The IP of this server writeXmlElement(writer, "ServerIP", getHostAddress()); // The OS name and version writeXmlElement(writer, "Platform", osVersion); - // --- write variable data - // The SessionID used to track a series of requests - writeXmlElement(writer, "SessionId", data.sessionId); - // The sequence number of the request in a series of requests. - writeXmlElement(writer, "Sequence", String.valueOf(data.sequence)); - // The UTC date/time this entry was written - writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date())); - // The client IP of the request - writeXmlElement(writer, "ClientIP", data.clientIP); - // Write all other evidence data that has been included. for (Map.Entry> category : data.evidenceData.entrySet()) { for (Map.Entry entry : category.getValue().entrySet()) { @@ -461,7 +456,7 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws writer.writeStartElement(entry.getKey()); } if (replacedString.isReplaced()) { - writer.writeAttribute("replaced", "true"); + writer.writeAttribute("escaped", "true"); } if (replacedString.isTruncated()) { writer.writeAttribute("truncated", "true"); @@ -488,45 +483,55 @@ protected void writeXmlElement(XMLStreamWriter writer, * replace characters that cause problems in XML with the "Replacement character" */ public static class ReplacedString { - // a set of valid XML character values (ignoring valid controls x09, x0a, x0d, x85) - static final Set VALID_XML_CHARS = - Stream.of( IntStream.range(0x20,0x7F), - IntStream.range(0xA0, 0x100)) - .flatMapToInt(i -> i) - .boxed() - .collect(Collectors.toCollection(HashSet::new)); - // an array describing whether a character value is valid - static final Boolean[] IS_VALID_XML_CHAR = IntStream - .range(0, 0x100) - .mapToObj(VALID_XML_CHARS::contains) - .toArray(Boolean[]::new); - - // maximum string length + // maximum string length, in characters static final int MAX_LENGTH = SHARE_USAGE_MAX_EVIDENCE_LENGTH; private boolean replaced; private boolean truncated; private final StringBuilder builder; + /** + * Whether the character is allowed in an XML 1.0 document, as the + * Char production in section 2.2 of the XML 1.0 specification says. + * @param codePoint the character to check + * @return true if the character is allowed in XML 1.0 + */ + static boolean isXmlChar(int codePoint) { + return codePoint == 0x9 || codePoint == 0xA || codePoint == 0xD || + (codePoint >= 0x20 && codePoint <= 0xD7FF) || + (codePoint >= 0xE000 && codePoint <= 0xFFFD) || + (codePoint >= 0x10000 && codePoint <= 0x10FFFF); + } + + /** + * Keep the first {@link #MAX_LENGTH} characters of the text, and write + * each character XML 1.0 does not allow as a backslash, an x and the + * four lower-case hexadecimal digits of its code. The limit counts + * characters, so a pair of surrogates is never split. + * @param text the text to prepare + */ public ReplacedString(String text) { if (Objects.isNull(text) || text.length() == 0) { builder = new StringBuilder(0); - // Using 'return' statement in a constructor throws compilation - // error in Jdk 8. - // return; } else { - builder = new StringBuilder(Math.min(text.length(), MAX_LENGTH)); - truncated = text.length() > MAX_LENGTH; - text.chars() - .limit(MAX_LENGTH) - .forEach(c -> { - if (c < IS_VALID_XML_CHAR.length && IS_VALID_XML_CHAR[c]) { - builder.append((char) c); - } else { - builder.append((char) 0xFFFD); - replaced = true; - } - }); + int end = text.length(); + if (text.codePointCount(0, end) > MAX_LENGTH) { + end = text.offsetByCodePoints(0, MAX_LENGTH); + truncated = true; + } + builder = new StringBuilder(end); + int index = 0; + while (index < end) { + int codePoint = text.codePointAt(index); + if (isXmlChar(codePoint)) { + builder.appendCodePoint(codePoint); + } else { + builder.append("\\x").append( + String.format("%04x", codePoint)); + replaced = true; + } + index += Character.charCount(codePoint); + } } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java index c98691cca..df4d534bb 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java @@ -39,7 +39,7 @@ */ class ReplacedStringTest { - // is transparent to legal characters + // is transparent to the three control characters XML 1.0 allows @Test public void testNormal(){ String aString = IntStream.of(0x9, 0xA, 0xD) @@ -47,8 +47,8 @@ public void testNormal(){ .collect(joining()); ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); - assertTrue(replacedString.isReplaced()); - assertEquals(3, replacedString.toString().length()); + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); } // is transparent to legal characters except \ @Test @@ -83,14 +83,74 @@ public void testNull(){ assertEquals(0, escapedString.toString().length()); assertEquals("", escapedString.toString()); } - // truncates longer than 512 + // truncates longer than 10,000 characters @Test public void testTruncate(){ - String aString = IntStream.range(0,513).mapToObj(i -> "a").collect(joining()); + String aString = repeat("a", 10001); + ReplacedString replacedString = new ReplacedString(aString); + assertTrue(replacedString.isTruncated()); + assertFalse(replacedString.isReplaced()); + assertEquals(10000, replacedString.toString().length()); + } + // keeps a value of exactly 10,000 characters whole + @Test + public void testNotTruncatedAtTheLimit(){ + String aString = repeat("a", 10000); + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isTruncated()); + assertEquals(aString, replacedString.toString()); + } + // the limit counts characters, so a character made of a pair of + // surrogates at the limit is kept whole and the next one is cut + @Test + public void testTruncateNeverSplitsASurrogatePair(){ + String emoji = "\uD83D\uDE00"; + String aString = repeat("a", 9999) + emoji + "b"; ReplacedString replacedString = new ReplacedString(aString); assertTrue(replacedString.isTruncated()); assertFalse(replacedString.isReplaced()); - assertEquals(512, replacedString.toString().length()); + assertEquals(repeat("a", 9999) + emoji, replacedString.toString()); + } + // a pair that would start after the limit is cut with the rest + @Test + public void testTruncateBeforeASurrogatePair(){ + String emoji = "\uD83D\uDE00"; + String aString = repeat("a", 10000) + emoji; + ReplacedString replacedString = new ReplacedString(aString); + assertTrue(replacedString.isTruncated()); + assertEquals(repeat("a", 10000), replacedString.toString()); + } + // counts characters and not UTF-16 code units + @Test + public void testLimitCountsCharactersNotCodeUnits(){ + String emoji = "\uD83D\uDE00"; + ReplacedString atLimit = new ReplacedString(repeat(emoji, 10000)); + assertFalse(atLimit.isTruncated()); + assertEquals(20000, atLimit.toString().length()); + ReplacedString over = new ReplacedString(repeat(emoji, 10001)); + assertTrue(over.isTruncated()); + assertEquals(repeat(emoji, 10000), over.toString()); + } + // the limit counts the value as supplied, so the escapes written for + // invalid characters do not count towards it + @Test + public void testEscapesDoNotCountTowardsTheLimit(){ + String aString = "a" + (char) 0x1 + repeat("b", 9998); + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isTruncated()); + assertTrue(replacedString.isReplaced()); + assertEquals(10000 - 1 + 6, replacedString.toString().length()); + // One character more is cut, whatever the escapes add. + ReplacedString over = new ReplacedString(aString + "c"); + assertTrue(over.isTruncated()); + assertEquals(10000 - 1 + 6, over.toString().length()); + } + private static String repeat(String text, int times) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < times; i++) { + builder.append(text); + } + return builder.toString(); } // escapes single digit Hex @Test @@ -99,8 +159,7 @@ public void testEscapedSmall(){ ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); assertTrue(replacedString.isReplaced()); - System.out.println(replacedString); - assertEquals(5, replacedString.toString().length()); + assertEquals(repeat("\\x0001", 5), replacedString.toString()); } // escapes 2 digit hex @Test @@ -109,8 +168,7 @@ public void testEscapedBigger(){ ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); assertTrue(replacedString.isReplaced()); - System.out.println(replacedString); - assertEquals(5, replacedString.toString().length()); + assertEquals(repeat("\\x001f", 5), replacedString.toString()); } // does not escape valid chars @Test @@ -123,15 +181,50 @@ public void testEscapedBiggerStill(){ assertEquals(5, replacedString.toString().length()); } - @Test () - public void testEscapedEvenBigger(){ - String aString = IntStream.range(0,5).mapToObj(i -> "\u0100").collect(joining()); + // does not replace characters above U+00FF, which XML 1.0 allows + @Test + public void testNonLatinKept(){ + String aString = "\u0100 \u0416 \u6F22\u5B57 \uD7FF \uE000 \uFFFD"; ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); - assertTrue(replacedString.isReplaced()); - System.out.println(replacedString); - assertEquals(5, replacedString.toString().length()); - + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); + } + // does not replace the controls from U+007F to U+009F, which XML 1.0 allows + @Test + public void testC1ControlsKept(){ + String aString = IntStream.range(0x7F, 0xA0) + .mapToObj(i -> Character.toString((char)i)) + .collect(joining()); + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); + } + // keeps a pair of surrogates, which is one character above U+FFFF + @Test + public void testSurrogatePairKept(){ + String aString = "a\uD83D\uDE00b"; + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); + } + // writes a surrogate on its own, U+FFFE and U+FFFF as \x and four + // lower-case hexadecimal digits + @Test + public void testForbiddenCharactersEscaped(){ + String[][] cases = { + {"\uD83D", "\\xd83d"}, {"\uDE00", "\\xde00"}, + {"\uFFFE", "\\xfffe"}, {"\uFFFF", "\\xffff"}, + {"\u0000", "\\x0000"}, {"\u000B", "\\x000b"}}; + for (String[] c : cases) { + ReplacedString replacedString = new ReplacedString("a" + c[0] + "b"); + assertTrue(replacedString.isReplaced()); + assertEquals("a" + c[1] + "b", replacedString.toString()); + } + // A low surrogate before a high one is two surrogates on their own. + ReplacedString reversed = new ReplacedString("\uDE00\uD83D"); + assertTrue(reversed.isReplaced()); + assertEquals("\\xde00\\xd83d", reversed.toString()); } private final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance(); @@ -140,12 +233,12 @@ public void testEscapedEvenBigger(){ @Test public void testXmlSerialise() throws XMLStreamException { xmlOutputFactory.setProperty("escapeCharacters", true); - String text = ReplacedString.VALID_XML_CHARS.stream() - .map(i->Character.toString((char)i.intValue())) - .collect(joining()); + String text = "\t\n a\u00E9\u0100\u0416\u6F22\u5B57\uD83D\uDE00" + + "\u007F\u0080\u009F\uD7FF\uE000\uFFFD <&>"; + assertEquals(text, new ReplacedString(text).toString()); ByteArrayOutputStream os = new ByteArrayOutputStream(); XMLStreamWriter writer = xmlOutputFactory.createXMLStreamWriter(os, "UTF-8"); - writer.writeStartDocument("UTF-8", "1.1"); + writer.writeStartDocument("UTF-8", "1.0"); writer.writeStartElement("test"); writer.writeCharacters(text); writer.writeEndElement(); @@ -155,8 +248,6 @@ public void testXmlSerialise() throws XMLStreamException { XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(is); reader.next(); String readText = reader.getElementText(); - System.out.println(text); - System.out.println(readText); reader.close(); assertEquals(readText, text); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 5a7a62563..a7ecfecbe 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -40,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; import javax.xml.parsers.DocumentBuilder; @@ -797,13 +798,109 @@ public void ShareUsageElement_BadSchema() throws Exception { // Check that the expected values are populated. NodeList f = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); assertEquals(1, f.getLength(), "1 user-agent header expected"); - assertEquals("", f.item(0).getTextContent()); - assertEquals("true", f.item(0).getAttributes().getNamedItem("replaced").getTextContent()); + assertEquals("", f.item(0).getTextContent()); + assertEquals("true", f.item(0).getAttributes().getNamedItem("escaped").getTextContent()); NodeList b = (NodeList) xpath.compile("//BadSchema").evaluate(doc, XPathConstants.NODESET); assertEquals(0, b.getLength()); } } + /** + * Check that the invalid characters in a value are marked with the + * attribute the specification names, and that a value that is not too + * long is not marked as truncated. + */ + @Test + public void ShareUsageElement_InvalidCharacter_EscapedAttributeOnly() + throws Exception { + Document doc = shareUserAgent(""); + + Node userAgent = doc.getElementsByTagName("header").item(0); + assertEquals("true", + userAgent.getAttributes().getNamedItem("escaped").getTextContent()); + assertNull(userAgent.getAttributes().getNamedItem("replaced")); + assertNull(userAgent.getAttributes().getNamedItem("truncated")); + assertEquals("", userAgent.getTextContent()); + } + + /** + * Check that a value that is too long is cut and marked as truncated, + * without the escaped attribute. + */ + @Test + public void ShareUsageElement_LongValue_TruncatedAttributeOnly() + throws Exception { + StringBuilder longValue = new StringBuilder(); + for (int i = 0; i < 10001; i++) { + longValue.append('a'); + } + Document doc = shareUserAgent(longValue.toString()); + + Node userAgent = doc.getElementsByTagName("header").item(0); + assertEquals("true", + userAgent.getAttributes().getNamedItem("truncated").getTextContent()); + assertNull(userAgent.getAttributes().getNamedItem("escaped")); + assertEquals(10000, userAgent.getTextContent().length()); + } + + /** + * Check that the values are written in the order the specification's + * example gives, and that no CDATA section or BadSchema element is used. + */ + @Test + public void ShareUsageElement_RecordFollowsTheSpecificationExample() + throws Exception { + Document doc = shareUserAgent(""); + + Node device = doc.getElementsByTagName("Device").item(0); + List names = new ArrayList<>(); + for (int i = 0; i < device.getChildNodes().getLength(); i++) { + Node child = device.getChildNodes().item(i); + assertNotEquals(Node.CDATA_SECTION_NODE, child.getNodeType()); + if (child.getNodeType() == Node.ELEMENT_NODE) { + names.add(child.getNodeName()); + } + } + String[] expected = {"SessionId", "Sequence", "DateSent", "Version", + "Product", "FlowElement", "Language", "LanguageVersion", + "ClientIP", "ServerIP", "Platform"}; + List required = Arrays.asList("SessionId", "Sequence", + "DateSent", "Product", "Language", "LanguageVersion"); + int last = -1; + for (String name : expected) { + int index = names.indexOf(name); + if (!required.contains(name)) { + // Only present when the Pipeline, the request and the + // platform give one. + if (index < 0) { + continue; + } + } + assertTrue(index >= 0, name + " missing from " + names); + assertTrue(index > last, name + " out of order in " + names); + last = index; + } + assertEquals(0, doc.getElementsByTagName("BadSchema").getLength()); + assertEquals(1, doc.getElementsByTagName("header").getLength()); + } + + private Document shareUserAgent(String userAgent) throws Exception { + createShareUsage(1, 1, 1, new ArrayList<>(), + new ArrayList<>(), new ArrayList<>()); + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_HEADER_USERAGENT_KEY, userAgent); + evidenceData.put("query.session-id", "session"); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + byte[] output = connector.getBaos().toByteArray(); + return docBuilderFactory.newDocumentBuilder() + .parse(new ByteArrayInputStream(output)); + } + } + /** * Test that the ShareUsageElement generates a session id if one is not * contained in the evidence and adds it to the results. From 726204bf7b2b3873584a6c604ba82828ad5d8b5e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 23:38:03 +0000 Subject: [PATCH 52/95] Revert "Treat a 408 or 429 at discovery as transient, so it does not fail the build" This reverts commit 7bc481c8fc509ccbea0eb3dbc9f6138bbe3daede. --- .../CloudRequestEngineBuilder.java | 4 +- .../CloudRequestEngineDefault.java | 44 ++++++------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index 5530411db..b0d420a0b 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -118,8 +118,8 @@ protected CloudRequestEngine newEngine(List properties) throws Exception else { // Fetch the accessible properties and the accepted evidence // keys now, so that the built engine is ready. A 4xx answer - // other than 408 or 429 fails the build, and any other failure - // is retried on first use. + // fails the build, and any other failure is retried on first + // use. engine.discover(); } built = engine; diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 5c95377f6..0bf8338f9 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -87,24 +87,11 @@ public class CloudRequestEngineDefault /** * The lowest HTTP status of a client error. A discovery request answered - * with a status from this up to {@link #HTTP_SERVER_ERROR_MIN}, other - * than {@link #HTTP_REQUEST_TIMEOUT} or {@link #HTTP_TOO_MANY_REQUESTS}, - * is a definitive configuration error. + * with a status from this up to {@link #HTTP_SERVER_ERROR_MIN} is a + * definitive configuration error. */ private static final int HTTP_CLIENT_ERROR_MIN = 400; - /** - * Request Timeout, a client error that a later retry can pass, as RFC - * 9110 section 15.5.9 says. - */ - private static final int HTTP_REQUEST_TIMEOUT = 408; - - /** - * Too Many Requests, a client error that a later retry can pass, as RFC - * 6585 section 4 says. - */ - private static final int HTTP_TOO_MANY_REQUESTS = 429; - /** * The lowest HTTP status of a server error. */ @@ -270,15 +257,14 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { * Fetch the accepted Evidence keys and the accessible properties that * are not known yet. The builder calls this when it builds the engine, * so in the normal case a built engine is ready to process. Both - * requests are made. A 4xx answer to either, other than 408 or 429, is - * a definitive configuration error, and its - * {@link CloudRequestException} is thrown, because retrying with the - * same configuration can never succeed. Any other failure, such as the - * service being unreachable, a timeout, a 5xx, 408 or 429 answer or a - * body that cannot be read, is logged as one warning and the list that - * did arrive is kept. Processing fetches what is still missing. - * @throws CloudRequestException for a 4xx answer to either request, - * other than 408 or 429 + * requests are made. A 4xx answer to either is a definitive + * configuration error, and its {@link CloudRequestException} is thrown, + * because retrying with the same configuration can never succeed. Any + * other failure, such as the service being unreachable, a timeout, a + * 5xx answer or a body that cannot be read, is logged as one warning + * and the list that did arrive is kept. Processing fetches what is + * still missing. + * @throws CloudRequestException for a 4xx answer to either request */ public void discover() { List failures = new ArrayList<>(); @@ -360,18 +346,16 @@ public void useDiscoveryState(CloudDiscoveryState state) { /** * The failure of a discovery request that the cloud service received - * and rejected with a 4xx status other than 408 or 429, so that - * retrying with the same configuration can never succeed. + * and rejected with a 4xx status, so that retrying with the same + * configuration can never succeed. * @param failure the failure of a discovery request - * @return the exception with that status, or null if there is none + * @return the exception with the 4xx status, or null if there is none */ private static CloudRequestException definitiveError(Throwable failure) { if (failure instanceof CloudRequestException) { int status = ((CloudRequestException) failure).getHttpStatusCode(); return status >= HTTP_CLIENT_ERROR_MIN && - status < HTTP_SERVER_ERROR_MIN && - status != HTTP_REQUEST_TIMEOUT && - status != HTTP_TOO_MANY_REQUESTS ? + status < HTTP_SERVER_ERROR_MIN ? (CloudRequestException) failure : null; } From 19cc0faf40f8ce0eccab0d88ddf2c999ccbb7e31 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 23:38:04 +0000 Subject: [PATCH 53/95] Revert "Test that a 408 or 429 at discovery is transient and does not fail the build" This reverts commit 40452b7d5d2a5a4de52ab482ccfa56dbd73f70a3. --- .../CloudRequestEngineDiscoveryTests.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java index 654273c61..a5bbc81fa 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -191,7 +191,7 @@ public void Build_FetchesBothLists() throws Exception { private static Stream definitiveFailures() { List arguments = new ArrayList<>(); - for (int status : new int[] { 400, 401, 403, 404 }) { + for (int status : new int[] { 400, 401, 403, 404, 429 }) { arguments.add(Arguments.of(PROPERTIES, status)); arguments.add(Arguments.of(EVIDENCE_KEYS, status)); } @@ -199,9 +199,8 @@ private static Stream definitiveFailures() { } /** - * A 4xx answer to either request, other than 408 or 429, is a - * definitive configuration error, which fails the build once both - * requests have been made. + * A 4xx answer to either request is a definitive configuration error, + * which fails the build once both requests have been made. */ @ParameterizedTest(name = "{1} from {0}") @MethodSource("definitiveFailures") @@ -236,10 +235,6 @@ private static Stream transientFailures() { Arguments.of("500 from accessibleproperties", 500, 200), Arguments.of("503 from evidencekeys", 200, 503), Arguments.of("502 from both", 502, 502), - Arguments.of("408 from accessibleproperties", 408, 200), - Arguments.of("408 from evidencekeys", 200, 408), - Arguments.of("429 from accessibleproperties", 429, 200), - Arguments.of("429 from evidencekeys", 200, 429), Arguments.of("unreadable accessibleproperties", UNREADABLE, 200), Arguments.of("unreadable evidencekeys", 200, UNREADABLE), Arguments.of("connection refused to both", UNREACHABLE, UNREACHABLE), @@ -247,9 +242,9 @@ private static Stream transientFailures() { } /** - * Any other failure, the service unreachable, a timeout, a 5xx, 408 or - * 429 answer or a body that cannot be read, is logged as one warning and - * the build completes. + * Any other failure, the service unreachable, a timeout, a 5xx answer or + * a body that cannot be read, is logged as one warning and the build + * completes. */ @ParameterizedTest(name = "{0}") @MethodSource("transientFailures") From dba6b9a038897f26dec6eb1394a67c0f8de824e6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:06:24 +0000 Subject: [PATCH 54/95] Test that a missing product is a warning and a missing response part no error The Start-up activity section of the cloud aspect engine specification on rc says the engine "MUST log the missing product once per Cloud Aspect Engine instance, at warning level, naming its Data Key". The Processing section says "Where the response JSON has no portion for the Engine's Data Key, the Cloud Aspect Engine leaves the Element Data without values and raises no error." The missing product tests now expect one warning and no error. ProductPresent_ResponseWithoutPart_NoValues, for each of the Device Detection, IP Intelligence and Location cloud engines, includes the engine's product and processes a response with no part for it, and expects no exception and a property without a value. propertyKeyed_productPresent_noProfiles does the same for a property keyed engine, with a response that has no part for it and one whose part has no profiles. All are written to fail on the previous commit, which logs an error and throws on a missing part when the product is present. --- .../flowelements/ProductMissingTests.java | 61 +++++++++++++++---- .../flowelements/ProductMissingTests.java | 61 +++++++++++++++---- .../GeoLocationCloudEngineTests.java | 61 +++++++++++++++---- .../CloudAspectEngineBaseTests.java | 50 +++++++++++++-- 4 files changed, 194 insertions(+), 39 deletions(-) diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java index 3c9ef7c1f..a9562e0fb 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -47,6 +47,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -80,6 +81,7 @@ public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { assertEquals(0, engine.getProperties().size()); List logged = warningsAndErrors(); assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).startsWith("warn ")); assertTrue(logged.get(0), logged.get(0).contains("'device'")); } } @@ -95,6 +97,24 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { } } + /** + * With the Device Detection product included, a response that holds no + * part for it leaves the element data without values and throws + * nothing, as the Processing section of the cloud aspect engine + * specification says. + */ + @Test + public void ProductPresent_ResponseWithoutPart_NoValues() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine("device", "hardwarevendor")) { + DeviceDataCloud data = process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + assertFalse(data.getHardwareVendor().hasValue()); + } + } + /** * Reading a property behaves as for any property the response does not * carry. @@ -115,21 +135,23 @@ public void ProductMissing_PropertyIsMissing() throws Exception { } /** - * An engine whose Cloud Request Engine reports the accessible products - * of a resource key that includes IP Intelligence alone. + * An engine whose Cloud Request Engine reports one product, holding + * one string property. */ // thenReturn has a varargs overload, so a generic Map is an unchecked array. @SuppressWarnings("unchecked") - private DeviceDetectionCloudEngine buildEngine() throws Exception { + private DeviceDetectionCloudEngine buildEngine( + String productKey, + String property) throws Exception { DeviceDetectionCloudEngine engine = new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); ProductMetaData product = new ProductMetaData(); product.dataTier = "test"; product.properties = Arrays.asList( - new PropertyMetaData("registeredname", "String", "cat", null)); + new PropertyMetaData(property, "String", "cat", null)); Map publicProperties = new HashMap<>(); - publicProperties.put("ip", product); + publicProperties.put(productKey, product); CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); when(requestEngine.getPublicProperties()).thenReturn(publicProperties); @@ -143,14 +165,13 @@ private DeviceDetectionCloudEngine buildEngine() throws Exception { } /** - * Process a cloud response that holds nothing for the Device Detection - * product. + * Process the given cloud response. */ - private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { + private DeviceDataCloud process( + DeviceDetectionCloudEngine engine, + String response) { CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn( - "{ 'ip': { 'registeredname': 'Example' }, " + - "'javascriptProperties': [] }"); + when(cloudData.getJsonResponse()).thenReturn(response); FlowData flowData = mock(FlowData.class); when(flowData.getFromElement(any(CloudRequestEngine.class))) @@ -165,6 +186,24 @@ private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { return device; } + /** + * An engine on a resource key that includes IP Intelligence alone. + */ + private DeviceDetectionCloudEngine buildEngine() throws Exception { + return buildEngine("ip", "registeredname"); + } + + /** + * Process a cloud response that holds nothing for the Device Detection + * product. + */ + private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { + return process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + } + /** * Every warning and error logged, at any overload. */ diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index 3ee13582f..88aa35083 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -47,6 +47,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -80,6 +81,7 @@ public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { assertEquals(0, engine.getProperties().size()); List logged = warningsAndErrors(); assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).startsWith("warn ")); assertTrue(logged.get(0), logged.get(0).contains("'ip'")); } } @@ -95,6 +97,24 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { } } + /** + * With the IP Intelligence product included, a response that holds no + * part for it leaves the element data without values and throws + * nothing, as the Processing section of the cloud aspect engine + * specification says. + */ + @Test + public void ProductPresent_ResponseWithoutPart_NoValues() + throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine("ip", "registeredname")) { + IPIntelligenceDataCloud data = process( + engine, + "{ 'device': { 'hardwarevendor': 'Example' }, " + + "'javascriptProperties': [] }"); + assertFalse(data.getRegisteredName().hasValue()); + } + } + /** * Reading a property behaves as for any property the response does not * carry. @@ -115,21 +135,23 @@ public void ProductMissing_PropertyIsMissing() throws Exception { } /** - * An engine whose Cloud Request Engine reports the accessible products - * of a resource key that includes Device Detection alone. + * An engine whose Cloud Request Engine reports one product, holding + * one string property. */ // thenReturn has a varargs overload, so a generic Map is an unchecked array. @SuppressWarnings("unchecked") - private IPIntelligenceCloudEngine buildEngine() throws Exception { + private IPIntelligenceCloudEngine buildEngine( + String productKey, + String property) throws Exception { IPIntelligenceCloudEngine engine = new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); ProductMetaData product = new ProductMetaData(); product.dataTier = "test"; product.properties = Arrays.asList( - new PropertyMetaData("hardwarevendor", "String", "cat", null)); + new PropertyMetaData(property, "String", "cat", null)); Map publicProperties = new HashMap<>(); - publicProperties.put("device", product); + publicProperties.put(productKey, product); CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); when(requestEngine.getPublicProperties()).thenReturn(publicProperties); @@ -143,14 +165,13 @@ private IPIntelligenceCloudEngine buildEngine() throws Exception { } /** - * Process a cloud response that holds nothing for the IP Intelligence - * product. + * Process the given cloud response. */ - private IPIntelligenceDataCloud process(IPIntelligenceCloudEngine engine) { + private IPIntelligenceDataCloud process( + IPIntelligenceCloudEngine engine, + String response) { CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn( - "{ 'device': { 'hardwarevendor': 'Example' }, " + - "'javascriptProperties': [] }"); + when(cloudData.getJsonResponse()).thenReturn(response); FlowData flowData = mock(FlowData.class); when(flowData.getFromElement(any(CloudRequestEngine.class))) @@ -166,6 +187,24 @@ private IPIntelligenceDataCloud process(IPIntelligenceCloudEngine engine) { return data; } + /** + * An engine on a resource key that includes Device Detection alone. + */ + private IPIntelligenceCloudEngine buildEngine() throws Exception { + return buildEngine("device", "hardwarevendor"); + } + + /** + * Process a cloud response that holds nothing for the IP Intelligence + * product. + */ + private IPIntelligenceDataCloud process(IPIntelligenceCloudEngine engine) { + return process( + engine, + "{ 'device': { 'hardwarevendor': 'Example' }, " + + "'javascriptProperties': [] }"); + } + /** * Every warning and error logged, at any overload. */ diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java index 80dadab55..4bed66b42 100644 --- a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -49,6 +49,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -82,6 +83,7 @@ public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { assertEquals(0, engine.getProperties().size()); List logged = warningsAndErrors(); assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).startsWith("warn ")); assertTrue(logged.get(0), logged.get(0).contains("'location'")); } } @@ -97,6 +99,24 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { } } + /** + * With the Location product included, a response that holds no + * part for it leaves the element data without values and throws + * nothing, as the Processing section of the cloud aspect engine + * specification says. + */ + @Test + public void ProductPresent_ResponseWithoutPart_NoValues() + throws Exception { + try (GeoLocationCloudEngine engine = buildEngine("location", "town")) { + CloudGeoData data = process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + assertFalse(data.getTown().hasValue()); + } + } + /** * Reading a property behaves as for any property the response does not * carry. @@ -117,12 +137,14 @@ public void ProductMissing_PropertyIsMissing() throws Exception { } /** - * An engine whose Cloud Request Engine reports the accessible products - * of a resource key that includes IP Intelligence alone. + * An engine whose Cloud Request Engine reports one product, holding + * one string property. */ // thenReturn has a varargs overload, so a generic Map is an unchecked array. @SuppressWarnings("unchecked") - private GeoLocationCloudEngine buildEngine() throws Exception { + private GeoLocationCloudEngine buildEngine( + String productKey, + String property) throws Exception { GeoLocationCloudEngine engine = new GeoLocationCloudEngineBuilder(loggerFactory) .build(Enums.GeoLocationProvider.FiftyOneDegrees); @@ -130,9 +152,9 @@ private GeoLocationCloudEngine buildEngine() throws Exception { ProductMetaData product = new ProductMetaData(); product.dataTier = "test"; product.properties = Arrays.asList( - new PropertyMetaData("registeredname", "String", "cat", null)); + new PropertyMetaData(property, "String", "cat", null)); Map publicProperties = new HashMap<>(); - publicProperties.put("ip", product); + publicProperties.put(productKey, product); CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); when(requestEngine.getPublicProperties()).thenReturn(publicProperties); @@ -146,14 +168,13 @@ private GeoLocationCloudEngine buildEngine() throws Exception { } /** - * Process a cloud response that holds nothing for the Location - * product. + * Process the given cloud response. */ - private CloudGeoData process(GeoLocationCloudEngine engine) { + private CloudGeoData process( + GeoLocationCloudEngine engine, + String response) { CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn( - "{ 'ip': { 'registeredname': 'Example' }, " + - "'javascriptProperties': [] }"); + when(cloudData.getJsonResponse()).thenReturn(response); FlowData flowData = mock(FlowData.class); when(flowData.getFromElement(any(CloudRequestEngine.class))) @@ -168,6 +189,24 @@ private CloudGeoData process(GeoLocationCloudEngine engine) { return data; } + /** + * An engine on a resource key that includes IP Intelligence alone. + */ + private GeoLocationCloudEngine buildEngine() throws Exception { + return buildEngine("ip", "registeredname"); + } + + /** + * Process a cloud response that holds nothing for the Location + * product. + */ + private CloudGeoData process(GeoLocationCloudEngine engine) { + return process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + } + /** * Every warning and error logged, at any overload. */ diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index f0f4ac5bb..6eba9dc3d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.verify; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -289,7 +290,7 @@ public void loadProperties_noProperties() throws Exception { assertEquals(0, engine.getProperties().size()); assertProductMissingLoggedOnce(); - maxErrors = 1; + maxWarnings = 1; } /** @@ -315,7 +316,7 @@ public void loadProperties_wrongProperties() throws Exception { assertEquals(0, engine.getProperties().size()); } assertProductMissingLoggedOnce(); - maxErrors = 1; + maxWarnings = 1; } /** @@ -377,7 +378,44 @@ public void propertyKeyed_productMissing_noProfiles() throws Exception { } assertEquals(0, keyed.getProperties().size()); assertProductMissingLoggedOnce(); - maxErrors = 1; + maxWarnings = 1; + } + + /** + * A property keyed engine whose product the resource key includes gives + * no profiles, and throws nothing, for a response that holds no part + * for it or a part without profiles, as the Processing section of the + * cloud aspect engine specification says. + * @throws Exception + */ + @Test + public void propertyKeyed_productPresent_noProfiles() throws Exception { + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + propertiesReturnedByRequestEngine.put("test", product); + TestKeyedInstance keyed = new TestKeyedInstance(); + requestEngine = new TestRequestEngine(); + requestEngine.setPublicProperties(propertiesReturnedByRequestEngine); + pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(requestEngine) + .addFlowElement(keyed) + .build(); + + for (String response : Arrays.asList( + "{ \"test2\": { \"ismobile\": true } }", + "{ \"test\": { } }")) { + TestProfiles profiles = new TestProfiles( + null, + Mockito.mock(FlowData.class), + keyed); + keyed.processCloudEngine( + Mockito.mock(FlowData.class), + profiles, + response); + assertEquals(0, profiles.getProfiles().size(), response); + } } private void assertProductMissingLoggedOnce() { @@ -387,9 +425,9 @@ private void assertProductMissingLoggedOnce() { errors.addAll(logger.errorsLogged); warnings.addAll(logger.warningsLogged); } - assertEquals(1, errors.size(), errors.toString()); - assertTrue(errors.get(0).contains("'test'"), errors.get(0)); - assertEquals(0, warnings.size(), warnings.toString()); + assertEquals(1, warnings.size(), warnings.toString()); + assertTrue(warnings.get(0).contains("'test'"), warnings.get(0)); + assertEquals(0, errors.size(), errors.toString()); } /** From 5e2553c7ccd59650010a9af294146c7161ef459a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:07:11 +0000 Subject: [PATCH 55/95] Log a missing product as a warning, and treat a missing response part as no values The Start-up activity section of the cloud aspect engine specification on rc says the missing product is logged "once per Cloud Aspect Engine instance, at warning level, naming its Data Key". findProduct now logs it with logger.warn. The Processing section says "Where the response JSON has no portion for the Engine's Data Key, the Cloud Aspect Engine leaves the Element Data without values and raises no error." getElementJson now returns an empty object whenever the response has no part for the engine, whether or not the product is included, and PropertyKeyedCloudEngineBase reads profiles with optJSONArray, so a part without profiles gives none. The productMissing flag and isProductMissing, which only chose between those two behaviours, are removed. --- .../flowelements/CloudAspectEngineBase.java | 33 +++++-------------- .../PropertyKeyedCloudEngineBase.java | 11 ++----- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index a828a9485..a448b3f10 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -112,7 +112,6 @@ public CloudRequestEngine getInstance() throws PipelineConfigurationException { private volatile List aspectProperties; private String dataSourceTier; private RequestEngineAccessor requestEngine; - private volatile boolean productMissing = false; private final AtomicBoolean productMissingLogged = new AtomicBoolean(false); /** @@ -231,7 +230,8 @@ private boolean loadAspectProperties() { * Find this engine's product among the products the Cloud Request Engine * reports for the resource key. A valid resource key can include no * product that holds this engine's properties. That is logged once for - * this engine, and an empty product is returned, so the engine builds + * this engine, as a warning naming its Data Key, and an empty product is + * returned, so the engine builds * with empty property metadata and logs nothing on each request. * @param products the products from * {@link CloudRequestEngine#getPublicProperties()}, or null if they have @@ -247,12 +247,10 @@ protected AccessiblePropertyMetaData.ProductMetaData findProduct( AccessiblePropertyMetaData.ProductMetaData product = products.get(getElementDataKey()); if (product != null) { - productMissing = false; return product; } - productMissing = true; if (productMissingLogged.compareAndSet(false, true)) { - logger.error(String.format( + logger.warn(String.format( MessageProductNotInResourceKey, getElementDataKey(), getClass().getSimpleName())); @@ -264,29 +262,16 @@ protected AccessiblePropertyMetaData.ProductMetaData findProduct( } /** - * True when the resource key does not include this engine's product, so - * the cloud response holds nothing for this engine. - * @return true if the product is missing - */ - protected boolean isProductMissing() { - return productMissing; - } - - /** - * This engine's part of the cloud response. When the resource key does - * not include this engine's product the response has no such part, and - * an empty object is returned, so the element data holds no values. + * This engine's part of the cloud response. Where the response has no + * part for this engine, as when the resource key does not include its + * product, an empty object is returned, so the element data holds no + * values and no error is raised. * @param response the whole cloud response * @return this engine's part of the response - * @throws org.json.JSONException if the product is included and the - * response has no part for this engine */ protected JSONObject getElementJson(JSONObject response) { - if (productMissing) { - JSONObject element = response.optJSONObject(getElementDataKey()); - return element == null ? new JSONObject() : element; - } - return response.getJSONObject(getElementDataKey()); + JSONObject element = response.optJSONObject(getElementDataKey()); + return element == null ? new JSONObject() : element; } private List loadElementProperties( diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java index 2f8dbfe08..7d8a19821 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java @@ -72,16 +72,11 @@ protected void processCloudEngine(FlowData data, TData aspectData, String json) // Extract data from json to the aspectData instance. JSONObject map = new JSONObject(json); - // Load the property meta-data, which finds whether the resource key - // includes this engine's product. - getProperties(); // Access the data relating to this engine. JSONObject propertyKeyed = getElementJson(map); - // Access the 'Profiles' property. The response holds none for a - // product the resource key does not include. - JSONArray profiles = isProductMissing() ? - propertyKeyed.optJSONArray("profiles") : - propertyKeyed.getJSONArray("profiles"); + // Access the 'Profiles' property. A response without it, as for a + // product the resource key does not include, gives no profiles. + JSONArray profiles = propertyKeyed.optJSONArray("profiles"); if (profiles == null) { return; } From 1e1dae8a52ff37cb8dd3f7368814c1220dfd17bc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:25:06 +0000 Subject: [PATCH 56/95] Test that a 408 at discovery fails the build, as every 4xx does The specification on rc makes every 4xx answer to a discovery request a definitive configuration error, so the cases that fail the build now include 408 beside 429. --- .../flowelements/CloudRequestEngineDiscoveryTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java index a5bbc81fa..07189b21d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -191,7 +191,7 @@ public void Build_FetchesBothLists() throws Exception { private static Stream definitiveFailures() { List arguments = new ArrayList<>(); - for (int status : new int[] { 400, 401, 403, 404, 429 }) { + for (int status : new int[] { 400, 401, 403, 404, 408, 429 }) { arguments.add(Arguments.of(PROPERTIES, status)); arguments.add(Arguments.of(EVIDENCE_KEYS, status)); } From 127b173e15c9c97744650b41ec3571bcee03d344 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:25:18 +0000 Subject: [PATCH 57/95] Test that a 4xx at discovery fails the build when its body is not JSON A 4xx answer is a definitive configuration error whatever its body holds, such as an error page from a proxy. The accessibleproperties answer is read as JSON before its status is used, so a 403 with an HTML body there fails this test, as the build completes with a warning. --- .../CloudRequestEngineDiscoveryTests.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java index 07189b21d..aeadfb1ae 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -35,6 +35,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.ILoggerFactory; import org.slf4j.Logger; @@ -112,6 +113,11 @@ public class CloudRequestEngineDiscoveryTests { private HttpClient httpClient; private final Map requests = new ConcurrentHashMap<>(); private final Map answers = new HashMap<>(); + + /** + * A body to answer with in place of the usual one, by endpoint. + */ + private final Map bodies = new HashMap<>(); private final List connections = Collections.synchronizedList(new ArrayList()); @@ -215,6 +221,25 @@ public void Build_4xx_FailsTheBuild(String endpoint, int status) { assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); } + /** + * A 4xx answer fails the build even when its body is not JSON, such as + * an error page from a proxy, because the status alone makes it + * definitive. + */ + @ParameterizedTest(name = "{0}") + @ValueSource(strings = { PROPERTIES, EVIDENCE_KEYS }) + public void Build_4xxWithBodyNotJson_FailsTheBuild(String endpoint) { + answers.put(endpoint, 403); + bodies.put(endpoint, "Forbidden"); + + CloudRequestException error = + assertThrows(CloudRequestException.class, this::build); + + assertEquals(403, error.getHttpStatusCode()); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + } + /** * A 4xx answer fails the build even when the other request failed in a * way that can pass. @@ -502,6 +527,9 @@ private String body(HttpURLConnection connection) throws IOException { if (answer == UNREADABLE) { return "proxy error"; } + if (bodies.containsKey(name)) { + return bodies.get(name); + } if (answer != 200) { return "{\"errors\": [\"" + name + " answered " + answer + "\"]}"; } From 7d4cebdd234afed486cf492a81acb88ddaf1496f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:25:24 +0000 Subject: [PATCH 58/95] Keep the status of a cloud answer whose body is not JSON An answer that cannot be read as JSON now raises a CloudRequestException carrying its HTTP status, as .NET does, so a 4xx answer to a discovery request fails the build whatever its body holds, as the specification says. --- .../pipeline/cloudrequestengine/Constants.java | 4 ++++ .../flowelements/CloudRequestEngineDefault.java | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index 4ac6fbbbb..e957273ee 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -40,6 +40,10 @@ public class Messages { public static final String ExceptionCloudError = "Error returned from 51Degrees cloud service: %s"; + + public static final String ExceptionResponseNotJson = + "Cloud service returned status code '%d' with content that is " + + "not JSON"; public static final String ExceptionFailedToLoadProperties = "Failed to load aspect properties for element '%s'. This is " diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 0bf8338f9..d0ecc75ea 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -38,6 +38,7 @@ import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.exceptions.AggregateException; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; @@ -702,7 +703,19 @@ private void validateResponse(String jsonResult, List messages = new ArrayList<>(); if (hasData && checkForErrorMessages) { - JSONObject jObj = new JSONObject(jsonResult); + JSONObject jObj; + try { + jObj = new JSONObject(jsonResult); + } catch (JSONException e) { + // The status says whether the failure is definitive, so it + // is kept when the body cannot be read. + CloudRequestException error = new CloudRequestException( + String.format(ExceptionResponseNotJson, code), + code, + connection.getHeaderFields()); + error.initCause(e); + throw error; + } boolean hasErrors = jObj.keySet().contains("errors"); hasData = hasErrors ? jObj.keySet().size() > 1 : From 97d7e45b211c2c71cf927f38a7a63ac669a669a5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 09:32:07 +0000 Subject: [PATCH 59/95] Close a projected ring by copying its first point to its last (#87) * Test a projected ring whose ends differ by one bit * Close a projected ring by copying its first point to its last --- .../examples/console/areas/Calculations.java | 10 ++++- .../console/areas/CalculationsTest.java | 43 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java index b3499f480..7f1cc8edb 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java @@ -259,7 +259,7 @@ private static Geometry transformGeometry( holes); } - private static LinearRing transformRing( + static LinearRing transformRing( LinearRing ring, CoordinateTransform transform) { Coordinate[] source = ring.getCoordinates(); @@ -276,6 +276,14 @@ private static LinearRing transformRing( transformed.y); } + // The first and last points are the same input, but compiled code can + // project them one bit apart, which JTS rejects as an open ring. The + // last point is therefore copied from the first. + if (coordinates.length > 1 && ring.isClosed()) { + coordinates[coordinates.length - 1] = + new Coordinate(coordinates[0]); + } + return ring.getFactory().createLinearRing(coordinates); } diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java index 8ebeda2d6..6a0b1e307 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java @@ -27,6 +27,12 @@ import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.Polygon; +import org.locationtech.proj4j.CoordinateReferenceSystem; +import org.locationtech.proj4j.CoordinateTransform; +import org.locationtech.proj4j.ProjCoordinate; + +import java.util.HashSet; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -168,6 +174,43 @@ public void testContains() throws Exception { false, result2.getContains()); } + /** + * A projection that returns a result one bit different the second time it + * is given the same point, as compiled code can for Math.sin. + */ + private static class DriftingTransform implements CoordinateTransform { + private final Set seen = new HashSet<>(); + + @Override + public CoordinateReferenceSystem getSourceCRS() { + return null; + } + + @Override + public CoordinateReferenceSystem getTargetCRS() { + return null; + } + + @Override + public ProjCoordinate transform(ProjCoordinate src, ProjCoordinate tgt) { + tgt.x = src.x; + tgt.y = src.y; + if (!seen.add(src.x + "," + src.y)) { + tgt.x = Math.nextUp(tgt.x); + } + return tgt; + } + } + + @Test + public void testProjectedRingIsClosedWhenEndsDifferByOneBit() { + LinearRing source = createRectangle(0, 0, 1).getExteriorRing(); + LinearRing projected = Calculations.transformRing( + source, new DriftingTransform()); + assertTrue(projected.isClosed()); + assertEquals(1.0, factory.createPolygon(projected).getArea(), 1e-9); + } + private static Polygon createRectangle(double x, double y, double d) { return factory.createPolygon(new Coordinate[]{ new Coordinate(x, y), From 30b38b9d3b757db754c847eded0a44bb958abf7f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 09:48:58 +0000 Subject: [PATCH 60/95] Wait for the watcher's update in the watcher test, not the polling timer's check (#95) * Wait for the watcher's update in the watcher test, not the polling timer's check With a one second polling interval the timer's check can complete first, finding no update because the watcher is enabled, and release the test before the watcher has refreshed the engine. The failed run took 1.008 seconds. The flag is now released only by a completion whose status is AUTO_UPDATE_SUCCESS, which only the watcher's update gives. * Import AUTO_UPDATE_SUCCESS in the data update service tests The watcher test now compares a completion's status with AUTO_UPDATE_SUCCESS, which rc's test file did not import. --- .../engines/services/DataUpdateServiceTests.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 52176409e..9dc7bc483 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -72,6 +72,7 @@ import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; +import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS; import static org.junit.Assert.*; import static org.junit.Assume.assumeFalse; import static org.mockito.Mockito.*; @@ -415,14 +416,17 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { } }); - // Configure a flag to be set when processing - // is complete. + // Configure a flag to be set when the watcher has applied the + // update. The polling timer's own check also completes, with no + // update found, and must not release the flag. final Semaphore completeFlag = new Semaphore(1); completeFlag.acquire(); dataUpdate.onUpdateComplete(new OnUpdateComplete() { @Override public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { - completeFlag.release(); + if (args.getStatus() == AUTO_UPDATE_SUCCESS) { + completeFlag.release(); + } } }); From 222a380294e2043596b38178c29bd41cc3292efc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 09:50:24 +0000 Subject: [PATCH 61/95] Say in the Element Data documentation that an instance is not written from several threads (#90) The thread safety page of the specification asks that documentation make clear Element Data is not intended to be used from several threads at once. The ElementData interface and ElementDataBase now say that an instance is written only by the Flow Element that creates it, while that element processes, and is not intended to be written from several threads at once, nor read while it is being written. Both link the page. --- .../main/java/fiftyone/pipeline/core/data/ElementData.java | 6 ++++++ .../java/fiftyone/pipeline/core/data/ElementDataBase.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java index 6f164c05d..da33f9074 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java @@ -28,7 +28,13 @@ /** * Represents property values that have be determined by a specific * {@link FlowElement} based on the supplied evidence. + *

    + * An instance is written only by the {@link FlowElement} that creates + * it, while that element processes. It is not intended to be written + * from several threads at once, nor read while it is being written. + *

    * @see Specification + * @see Thread safety */ public interface ElementData extends Data { diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java index fc67b9759..937b3c590 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java @@ -33,7 +33,13 @@ * Base implementation of {@link ElementData}. * Represents property values that have be determined by a specific * {@link FlowElement} based on the supplied evidence. + *

    + * An instance is written only by the {@link FlowElement} that creates + * it, while that element processes. It is not intended to be written + * from several threads at once, nor read while it is being written. + *

    * @see Specification + * @see Thread safety */ public abstract class ElementDataBase extends DataBase implements ElementData { From 9fced9b3d7633865b81cdc9130d1fb5e3aa0a944 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:02:08 +0000 Subject: [PATCH 62/95] Rewrap the findProduct Javadoc paragraph A comment-only change. The paragraph that describes findProduct left "returned, so the engine builds" on a short line of its own, so the paragraph is now wrapped at 80 columns. No code changes. --- .../flowelements/CloudAspectEngineBase.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index a448b3f10..9b57c0368 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -228,11 +228,11 @@ private boolean loadAspectProperties() { /** * Find this engine's product among the products the Cloud Request Engine - * reports for the resource key. A valid resource key can include no - * product that holds this engine's properties. That is logged once for - * this engine, as a warning naming its Data Key, and an empty product is - * returned, so the engine builds - * with empty property metadata and logs nothing on each request. + * reports for the resource key. A valid resource key can include no product + * that holds this engine's properties. That is logged once for this engine, + * as a warning naming its Data Key, and an empty product is returned, so + * the engine builds with empty property metadata and logs nothing on each + * request. * @param products the products from * {@link CloudRequestEngine#getPublicProperties()}, or null if they have * not been loaded From 7bf7192e9e56550c63cfa55c962f9061966de152 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:18:47 +0000 Subject: [PATCH 63/95] List the Iterations match metric before Method, as the specification's table does (#94) * Test that the match metric Properties are listed in the specification's order Engine_Hash_MatchMetricOrder builds a Hash engine on the Lite data file and expects the last seven Properties it lists to be the match metrics in the order of the specification's table, MatchedNodes, Difference, Drift, DeviceId, UserAgents, Iterations and Method. It is written to fail on rc, which lists Method before Iterations. * List the Iterations match metric before Method, as the specification's table does The Hash engine's match metric Properties are now MatchedNodes, Difference, Drift, DeviceId, UserAgents, Iterations and Method, the order of the match metrics table in the device detection on-premise page of the specification and the order .NET's GetMetricProperties returns them in. Only the order of the list changes. * Check that there are enough Properties before reading the match metrics The match metric order test now says how many Properties there were when fewer than the seven match metrics come back, rather than failing with an index out of bounds. --- .../DeviceDetectionHashEngine.java | 32 ++++++------- .../onpremise/flowelements/EngineTests.java | 45 +++++++++++++++++++ 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java index d7b913099..64ede4a73 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java @@ -467,45 +467,45 @@ private List getMetricProperties() { Arrays.asList(), new ValueMetaDataDefault("n/a")), new FiftyOneAspectPropertyMetaDataDefault( - Constants.MatchMetrics.METHOD, + Constants.MatchMetrics.ITERATIONS, this, Constants.MatchMetrics.CATEGORY, - String.class, + Integer.class, dataFileList, true, null, (byte) 0, true, - true, false, false, false, - Constants.MatchMetrics.METHOD_DESCRIPTION, + false, + Constants.MatchMetrics.ITERATIONS_DESCRIPTION, null, - Arrays.asList( - new ValueMetaDataDefault("NONE"), - new ValueMetaDataDefault("PERFORMANCE"), - new ValueMetaDataDefault("COMBINED"), - new ValueMetaDataDefault( "PREDICTIVE")), - new ValueMetaDataDefault("NONE")), + Arrays.asList(), + new ValueMetaDataDefault("0")), new FiftyOneAspectPropertyMetaDataDefault( - Constants.MatchMetrics.ITERATIONS, + Constants.MatchMetrics.METHOD, this, Constants.MatchMetrics.CATEGORY, - Integer.class, + String.class, dataFileList, true, null, (byte) 0, true, + true, false, false, false, - false, - Constants.MatchMetrics.ITERATIONS_DESCRIPTION, + Constants.MatchMetrics.METHOD_DESCRIPTION, null, - Arrays.asList(), - new ValueMetaDataDefault("0")) + Arrays.asList( + new ValueMetaDataDefault("NONE"), + new ValueMetaDataDefault("PERFORMANCE"), + new ValueMetaDataDefault("COMBINED"), + new ValueMetaDataDefault( "PREDICTIVE")), + new ValueMetaDataDefault("NONE")) }; return Arrays.asList(metricProperties); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index 14b30de62..198e43bb2 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -26,6 +26,9 @@ import static org.junit.Assume.assumeTrue; import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.slf4j.ILoggerFactory; @@ -33,6 +36,7 @@ import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -95,4 +99,45 @@ public void Engine_Hash_GetDataSourceTier_Enterprise() throws Exception { testDataSourceTier(TAC_HASH_DATA_FILE_NAME); } + /** + * The match metric Properties follow the data file's Properties, in the + * order of the match metrics table in the specification. + * @throws Exception + */ + @Test + public void Engine_Hash_MatchMetricOrder() throws Exception { + File dataFile = null; + try { + dataFile = getFilePath(LITE_HASH_DATA_FILE_PATH); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + } + List expected = Arrays.asList( + "MatchedNodes", + "Difference", + "Drift", + "DeviceId", + "UserAgents", + "Iterations", + "Method"); + + List names = new ArrayList<>(); + try (DeviceDetectionHashEngine engine = createEngine(dataFile)) { + List properties = + engine.getProperties(); + assertTrue( + "Expected at least " + expected.size() + " Properties, " + + "but there were " + properties.size(), + properties.size() >= expected.size()); + for (FiftyOneAspectPropertyMetaData property : properties.subList( + properties.size() - expected.size(), + properties.size())) { + names.add(property.getName()); + } + } + + assertEquals(expected, names); + } + } From fc55341dc2c15537fb9ace78855cef5f20edc9b5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:19:01 +0000 Subject: [PATCH 64/95] Show a license key only by its last 4 characters in logs and error messages (#85) * Add LicenseKeyMask, which shows a license key by its last 4 characters A license key in a log entry or an error message is shown only by its last 4 characters, with every character before them replaced by '*', so a reader can tell two keys apart but cannot use either. A key of 4 characters or fewer is replaced in full, because its last 4 characters would be all of it. maskIn masks each key in a text, as written and as encoded in a URL, and gives a throwable to log in place of one whose message, causes or suppressed throwables hold a key, keeping the class names and stack traces. * Test that a license key reaches no log entry or error message in full The data update service is tested for a 403, a 429, another failing status, a 304, no response and an exception that names the URL, with two keys in the update URL. Everything it passes to the logger, at every level and with each exception's stack trace, holds each key only by its last 4 characters. The cloud request engine is tested with an error from the cloud that names the license key, and the 51Did client with a 401 body and a 400 errors list that name the licence, and with a key of 4 characters, which is replaced in full. * Mask license keys in data update logs, cloud errors and 51Did redeem errors The data update service logged the formatted update URL, whose query carries the license keys, at debug and error level, and logged the exception an HTTP library raised, whose message often names that URL. Every message now names the URL with each configured key masked by LicenseKeyMask, and the exception is logged as a copy whose messages are masked. The cloud answers a refused license key with an error that names it, and the cloud request engine put that text in the exception it raised, which can reach the log and the JSON builder's errors list. Its messages now mask the license key, and each key in it when it holds several separated by '|'. The 51Did client puts the start of the body in the message of a redeem error, and the errors list of a 400 answer in the message of the IllegalArgumentException it raises. Both now mask the licence key the redeem request sent. The 51Did package depends on nothing in the Pipeline, so it masks with a private method that gives the same result as LicenseKeyMask. The exception's body is kept as the service sent it. * Test the masked form of an encoded key and a key typed into the update URL maskIn_EncodedKeyShowsThePlainLast4 expects a key whose last characters are percent-encoded in a URL to show the encoded form of its plain last 4 characters. maskIn_MasksAKeyEncodedInLowerCase expects a key encoded with lower case hexadecimal digits to be masked too. KeyTypedIntoTheUrl_LogsMaskedKey gives the data update service a URL whose LicenseKeys parameter holds a key that is not among the data update license keys, and expects the key masked in every log entry. All three are written to fail on the previous commit. * Mask an encoded key by its plain form, and keys typed into the update URL LicenseKeyMask.maskIn now replaces a URL-encoded key with the encoded form of its masked plain form, so it shows the same last 4 characters as the plain key, and also masks the key encoded with lower case hexadecimal digits. Its Javadoc now says that every occurrence of a key's text is masked, so a key of only a few characters masks those characters elsewhere too, as the specification asks for keys of 4 characters or fewer. The specification link now points at rc, where the rule is. The data update service now also masks any key in the LicenseKeys parameter of the data update URL, such as one typed straight into the DataUpdateUrl setting rather than given as a data update license key. DidHttpException.getBody now says the body is not masked and can hold the license key sent, so a caller that logs it must mask it first. * Put the Javadoc of downloadFile back above it licenseKeysToMask was added between downloadFile and its Javadoc, so the Javadoc described the wrong method. It is now above downloadFile again. --- .../CloudRequestEngineDefault.java | 17 ++ .../flowelements/CloudRequestEngineTests.java | 62 +++++ .../pipeline/util/LicenseKeyMask.java | 234 ++++++++++++++++++ .../pipeline/util/LicenseKeyMaskTests.java | 191 ++++++++++++++ .../java/fiftyone/pipeline/did/DidClient.java | 58 ++++- .../pipeline/did/DidHttpException.java | 8 +- .../fiftyone/pipeline/did/DidClientTests.java | 68 +++++ .../services/DataUpdateServiceDefault.java | 67 ++++- .../DataUpdateServiceLicenseKeyTests.java | 223 +++++++++++++++++ 9 files changed, 914 insertions(+), 14 deletions(-) create mode 100644 pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 12ccc17f2..985b5429c 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.flowelements.AspectEngineBase; import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.util.LicenseKeyMask; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -425,6 +426,19 @@ private static URL toUrl(String spec) throws MalformedURLException { } } + /** + * The license key sent with each request, and each key in it when it + * holds several separated by '|', to mask in messages. + */ + private List getLicenseKeys() { + List keys = new ArrayList<>(); + if (licenseKey != null && licenseKey.isEmpty() == false) { + keys.add(licenseKey); + keys.addAll(Arrays.asList(licenseKey.split(Pattern.quote("|")))); + } + return keys; + } + private void getCloudProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; @@ -544,6 +558,9 @@ else if (messages.size() == 0 && code != 200) messages.add(message); } + // The cloud's errors can name the license key it was sent. + messages.replaceAll(m -> LicenseKeyMask.maskIn(m, getLicenseKeys())); + Map> headers = null; if (messages.size() > 0){ headers = connection.getHeaderFields(); diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 80ebde301..1259fd677 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -40,6 +40,8 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import java.io.IOException; import java.net.MalformedURLException; @@ -101,6 +103,66 @@ public void Process() throws Exception { } } + /** + * The cloud answers a refused license key with an error that names it. + * The exception the engine raises, and every log entry, show the key + * only by its last 4 characters. + */ + @Test + public void Process_LicenseKeyInCloudError_Masked() throws Exception { + final String licenseKey = "TESTKEYNOTREALABCDEFGHIJ1234"; + final String masked = "************************1234"; + jsonResponse = "{\"errors\":[\"Invalid license key(s) supplied: " + + licenseKey + "\"]}"; + configureMockedClient(); + Logger logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + ILoggerFactory capturingLoggerFactory = mock(ILoggerFactory.class); + when(capturingLoggerFactory.getLogger(anyString())).thenReturn(logger); + + CloudRequestEngine engine = new CloudRequestEngineBuilder( + capturingLoggerFactory, httpClient) + .setResourceKey("resource_key") + .setLicenseKey(licenseKey) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(capturingLoggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + assertFalse(data.getErrors().isEmpty()); + StringBuilder reported = new StringBuilder(); + for (FlowError error : data.getErrors()) { + reported.append(printed(error.getThrowable())); + } + for (org.mockito.invocation.Invocation invocation : + mockingDetails(logger).getInvocations()) { + for (Object argument : invocation.getArguments()) { + reported.append(argument instanceof Throwable ? + printed((Throwable) argument) : + String.valueOf(argument)); + } + } + assertFalse(reported.toString().contains(licenseKey), + reported.toString()); + assertTrue(reported.toString().contains(masked), + reported.toString()); + } + } + + private static String printed(Throwable throwable) { + java.io.StringWriter text = new java.io.StringWriter(); + throwable.printStackTrace(new java.io.PrintWriter(text)); + return text.toString(); + } + /** * Test errors thrown by cloud request engine are added to flow.errors when SuppressProcessExceptions == true */ diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java new file mode 100644 index 000000000..6f323b5ba --- /dev/null +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java @@ -0,0 +1,234 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Masks license keys in text written to a log entry or an error message. A + * masked key shows only its last 4 characters, with every character before + * them replaced by '*', so a reader can tell two keys apart but cannot use + * either. A key of 4 characters or fewer is replaced in full, because its + * last 4 characters would be the whole key. + * @see Specification + */ +public final class LicenseKeyMask { + + /** + * How many characters at the end of a key stay visible. + */ + public static final int VISIBLE_CHARACTERS = 4; + + /** + * The character that replaces each hidden character of a key. + */ + public static final char MASK_CHARACTER = '*'; + + private LicenseKeyMask() { + } + + /** + * The key with every character before its last 4 replaced. + * @param key a license key, or null + * @return the masked key, or null if the key is null + */ + public static String mask(String key) { + if (key == null) { + return null; + } + int hidden = key.length() > VISIBLE_CHARACTERS ? + key.length() - VISIBLE_CHARACTERS : + key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append(MASK_CHARACTER); + } + return masked.append(key, hidden, key.length()).toString(); + } + + /** + * The text with every occurrence of each key masked, both as the key is + * written and as it is encoded in a URL, with upper or lower case + * hexadecimal digits. An encoded key is replaced by the encoded form of + * its masked plain form, so it shows the same last 4 characters. Every + * occurrence of a key's text is masked, so a key of only a few + * characters also masks those characters wherever else they appear. + * @param text the text, such as a URL or an error message, or null + * @param keys the license keys, or null + * @return the masked text, or null if the text is null + */ + public static String maskIn(String text, Iterable keys) { + if (text == null || keys == null) { + return text; + } + String result = text; + for (String key : longestFirst(keys)) { + String masked = mask(key); + result = result.replace(key, masked); + String encoded = urlEncode(key); + if (encoded.equals(key) == false) { + String maskedEncoded = urlEncode(masked); + result = result.replace(encoded, maskedEncoded); + result = result.replace( + lowerCaseEscapes(encoded), + lowerCaseEscapes(maskedEncoded)); + } + } + return result; + } + + /** + * The throwable to log or report in place of the one given, so that no + * message in it, its causes or its suppressed throwables holds a key. + * When none does, the throwable given is returned. Otherwise a copy is + * returned whose messages are masked, each starting with the name of the + * class it stands for, and whose stack traces are the originals. + * @param throwable the throwable, or null + * @param keys the license keys, or null + * @return a throwable that holds no key + */ + public static Throwable maskIn(Throwable throwable, Iterable keys) { + if (throwable == null || + keys == null || + holdsKey(throwable, keys, new IdentityHashMap()) + == false) { + return throwable; + } + return copy(throwable, keys, new IdentityHashMap()); + } + + private static boolean holdsKey( + Throwable throwable, + Iterable keys, + Map seen) { + if (throwable == null || seen.put(throwable, Boolean.TRUE) != null) { + return false; + } + String message = throwable.getMessage(); + if (message != null && message.equals(maskIn(message, keys)) == false) { + return true; + } + if (holdsKey(throwable.getCause(), keys, seen)) { + return true; + } + for (Throwable suppressed : throwable.getSuppressed()) { + if (holdsKey(suppressed, keys, seen)) { + return true; + } + } + return false; + } + + private static Throwable copy( + Throwable throwable, + Iterable keys, + Map copies) { + Throwable existing = copies.get(throwable); + if (existing != null) { + return existing; + } + String message = maskIn(throwable.getMessage(), keys); + MaskedThrowable masked = new MaskedThrowable( + throwable.getClass().getName() + + (message == null ? "" : ": " + message)); + copies.put(throwable, masked); + masked.setStackTrace(throwable.getStackTrace()); + Throwable cause = throwable.getCause(); + if (cause != null) { + Throwable maskedCause = copy(cause, keys, copies); + if (maskedCause != masked) { + masked.initCause(maskedCause); + } + } + for (Throwable suppressed : throwable.getSuppressed()) { + Throwable maskedSuppressed = copy(suppressed, keys, copies); + if (maskedSuppressed != masked) { + masked.addSuppressed(maskedSuppressed); + } + } + return masked; + } + + /** + * The keys that are not null or empty, longest first, so that a key + * that holds another is masked whole. + */ + private static List longestFirst(Iterable keys) { + List result = new ArrayList<>(); + for (String key : keys) { + if (key != null && key.isEmpty() == false) { + result.add(key); + } + } + Collections.sort(result, (a, b) -> b.length() - a.length()); + return result; + } + + /** + * The text with the two hexadecimal digits after each '%' in lower + * case. + */ + private static String lowerCaseEscapes(String text) { + StringBuilder result = new StringBuilder(text); + for (int i = text.indexOf('%'); + i >= 0 && i + 2 < text.length(); + i = text.indexOf('%', i + 1)) { + result.setCharAt(i + 1, Character.toLowerCase(text.charAt(i + 1))); + result.setCharAt(i + 2, Character.toLowerCase(text.charAt(i + 2))); + } + return result.toString(); + } + + private static String urlEncode(String key) { + try { + return URLEncoder.encode(key, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Every Java platform supports UTF-8. + throw new IllegalStateException(e); + } + } + + /** + * Stands for a throwable whose message held a license key. Its message + * starts with the name of the class it stands for, and it prints as + * that message alone. + */ + private static final class MaskedThrowable extends Exception { + private static final long serialVersionUID = 1L; + + MaskedThrowable(String message) { + super(message); + } + + @Override + public String toString() { + return getMessage(); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java new file mode 100644 index 000000000..312e2c08e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java @@ -0,0 +1,191 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import org.junit.Test; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * A license key in a log entry or an error message shows only its last 4 + * characters, with every character before them replaced. + */ +public class LicenseKeyMaskTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + + @Test + public void mask_ShowsTheLast4Characters() { + assertEquals(MASKED, LicenseKeyMask.mask(KEY)); + assertEquals(KEY.length(), LicenseKeyMask.mask(KEY).length()); + } + + /** + * A key of 4 characters or fewer would be shown whole, so all of it is + * replaced. + */ + @Test + public void mask_ShortKeyIsReplacedInFull() { + assertEquals("****", LicenseKeyMask.mask("1234")); + assertEquals("**", LicenseKeyMask.mask("12")); + assertEquals("*2345", LicenseKeyMask.mask("12345")); + assertEquals("", LicenseKeyMask.mask("")); + assertNull(LicenseKeyMask.mask(null)); + } + + @Test + public void maskIn_MasksEveryKeyInAUrl() { + String url = "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + KEY + "|" + OTHER_KEY + "&Download=True"; + + String masked = LicenseKeyMask.maskIn( + url, + Arrays.asList(KEY, OTHER_KEY)); + + assertEquals( + "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + MASKED + "|" + OTHER_MASKED + + "&Download=True", + masked); + } + + /** + * A key that a URL holds encoded is masked in its encoded form. + */ + @Test + public void maskIn_MasksAKeyEncodedInAUrl() { + String key = "KEY+WITH/CHARS=ENCODED9876"; + String url = "https://example.test/?license=" + + "KEY%2BWITH%2FCHARS%3DENCODED9876"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertFalse(masked, masked.contains("KEY%2BWITH")); + assertTrue(masked, masked.endsWith("9876")); + } + + /** + * A key whose last characters are encoded in a URL shows the same last + * 4 characters as its plain form, encoded, rather than the last 4 + * characters of its encoded form. + */ + @Test + public void maskIn_EncodedKeyShowsThePlainLast4() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2B%3D"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2B%3D", + masked); + } + + /** + * A key encoded with lower case hexadecimal digits, as some libraries + * write them, is masked as well. + */ + @Test + public void maskIn_MasksAKeyEncodedInLowerCase() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2b%3d"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2b%3d", + masked); + } + + @Test + public void maskIn_LeavesTextWithoutAKeyAlone() { + String text = "No key here."; + assertEquals(text, LicenseKeyMask.maskIn(text, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((String) null, Arrays.asList(KEY))); + assertEquals(text, LicenseKeyMask.maskIn(text, null)); + assertEquals( + text, + LicenseKeyMask.maskIn(text, Arrays.asList(null, ""))); + } + + /** + * The message of an exception an HTTP library raises often holds the + * URL it requested, and so the key. The throwable to log holds the key + * in none of its messages, its causes' or its suppressed throwables', + * and keeps the name of each class it stands for. + */ + @Test + public void maskIn_MasksAThrowableAndItsCauses() { + IOException cause = new IOException( + "Server returned HTTP response code: 403 for URL: " + + "https://distributor.example/?LicenseKeys=" + KEY); + RuntimeException thrown = new RuntimeException( + "Download failed for " + KEY, cause); + thrown.addSuppressed(new IllegalStateException(OTHER_KEY)); + List keys = Arrays.asList(KEY, OTHER_KEY); + + Throwable masked = LicenseKeyMask.maskIn(thrown, keys); + String printed = printed(masked); + + assertFalse(printed, printed.contains(KEY)); + assertFalse(printed, printed.contains(OTHER_KEY)); + assertTrue(printed, printed.contains(MASKED)); + assertTrue(printed, printed.contains(OTHER_MASKED)); + assertTrue(printed, printed.contains("java.lang.RuntimeException")); + assertTrue(printed, printed.contains("java.io.IOException")); + assertTrue(printed, + printed.contains("java.lang.IllegalStateException")); + assertEquals( + thrown.getStackTrace().length, + masked.getStackTrace().length); + } + + @Test + public void maskIn_ThrowableWithoutAKeyIsReturnedAsItIs() { + IOException thrown = new IOException("Connection refused"); + assertSame(thrown, LicenseKeyMask.maskIn(thrown, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((Throwable) null, Arrays.asList(KEY))); + } + + private static String printed(Throwable throwable) { + StringWriter text = new StringWriter(); + throwable.printStackTrace(new PrintWriter(text)); + return text.toString(); + } +} diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index f0d2c4010..9afb29af1 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -940,7 +940,11 @@ public CompletableFuture redeem( } } - /** The typed result a redeem answer carries. */ + /** + * The typed result a redeem answer carries. The request sent the + * licence key, and the answer can name it, so the messages of the + * errors show the key masked. + */ private RedeemResult readRedeem(HttpTransport.Response response) { int status = response.getStatusCode(); JSONObject json = asObject(response.getBody()); @@ -949,21 +953,22 @@ private RedeemResult readRedeem(HttpTransport.Response response) { case 503: if (json == null) { throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } return RedeemResult.parse(status, json, response.getBody()); case 400: if (json != null && json.has("errors")) { - throw new IllegalArgumentException(errorsText(json)); + throw new IllegalArgumentException( + maskLicence(errorsText(json), licenceKey)); } throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); case 404: throw new CompletionException(new DidNotSupportedException( endpoint, response.getBody())); default: throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } } @@ -1045,13 +1050,54 @@ private static void ensureReadsAs51Did(String fodId) { private static DidHttpException httpError( String what, HttpTransport.Response response) { + return httpError(what, response, null); + } + + /** + * An error whose message summarises the body, with the licence key the + * request sent masked in it. + */ + private static DidHttpException httpError( + String what, HttpTransport.Response response, String licence) { String body = response.getBody(); - String summary = body.length() > 200 ? body.substring(0, 200) : body; + String masked = maskLicence(body, licence); + String summary = masked.length() > 200 ? + masked.substring(0, 200) : + masked; return new DidHttpException( what + " answered HTTP " + response.getStatusCode() + ": " + summary, response.getStatusCode(), body); } + /** + * The text with the licence key shown only by its last 4 characters, + * every character before them replaced by '*', or replaced in full when + * it has 4 characters or fewer. This is how every 51Degrees package + * writes a licence key in a message, and it matches + * fiftyone.pipeline.util.LicenseKeyMask, which this package cannot use + * because it depends on nothing in the Pipeline. + */ + private static String maskLicence(String text, String licence) { + if (text == null || licence == null || licence.isEmpty()) { + return text; + } + String result = text.replace(licence, maskKey(licence)); + String encoded = encode(licence); + if (encoded.equals(licence) == false) { + result = result.replace(encoded, maskKey(encoded)); + } + return result; + } + + private static String maskKey(String key) { + int hidden = key.length() > 4 ? key.length() - 4 : key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append('*'); + } + return masked.append(key, hidden, key.length()).toString(); + } + /** The body as a JSON object, or null when it is not one. */ private static JSONObject asObject(String body) { try { diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java index 49d6d41cb..5a1747925 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java @@ -54,7 +54,13 @@ public int getStatusCode() { return statusCode; } - /** @return the body the service answered with, never null */ + /** + * The body the service answered with, as it was sent. It is not masked, + * and the answer to a request that sent a license key can hold that + * key, so a caller that logs the body or puts it in a message must + * mask any license key in it first. + * @return the body, never null + */ public String getBody() { return body; } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index 71969a6ac..cea3ec788 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -67,6 +67,9 @@ public class DidClientTests { private static final String ENDPOINT = "https://example.test/api/v4/"; + private static final String TEST_LICENCE = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String TEST_LICENCE_MASKED = + "************************1234"; private static final Instant WEEK1 = Instant.parse("2026-08-03T00:00:00Z"); private static final Instant WEEK2 = WEEK1.plus(Duration.ofDays(7)); private static final Instant WEEK3 = WEEK2.plus(Duration.ofDays(7)); @@ -1273,6 +1276,71 @@ public void redeem_TransportFailureFailsWithIoException() { client.redeem(validDid, "sealed", "abc")); } + /** + * The service answers a refused licence key with a body that names it. + * The error message shows the key only by its last 4 characters, with + * every character before them replaced, as every 51Degrees package + * writes a licence key in a message. + */ + @Test + public void redeem_RefusedLicenceIsMaskedInTheMessage() { + transport.queue(401, + "Invalid license key(s) supplied: " + TEST_LICENCE); + + DidHttpException error = failure(DidHttpException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertEquals(401, error.getStatusCode()); + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + @Test + public void redeem_LicenceInErrorsIsMaskedInTheMessage() { + transport.queue(400, "{\"errors\":[\"The license key " + + TEST_LICENCE + " does not include this product.\"]}"); + + IllegalArgumentException error = failure( + IllegalArgumentException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + /** + * A key of 4 characters or fewer is replaced in full, because its last + * 4 characters would be all of it. + */ + @Test + public void redeem_ShortLicenceIsMaskedInFull() { + DidClient shortLicence = DidClient.builder("resource") + .licenceKey("1234") + .endpoint(ENDPOINT) + .transport(transport) + .build(); + transport.queue(401, "Invalid license key(s) supplied: 1234"); + + DidHttpException error = failure(DidHttpException.class, + shortLicence.redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), error.getMessage().contains("1234")); + assertTrue(error.getMessage(), error.getMessage().endsWith("****")); + } + + private DidClient licensedClient() { + return DidClient.builder("resource") + .licenceKey(TEST_LICENCE) + .endpoint(ENDPOINT) + .transport(transport) + .clock(clock) + .build(); + } + @Test public void redeem_WithoutLicenceKeyOmitsTheField() throws Exception { DidClient noLicence = DidClient.builder("resource") diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 007b21211..e3bfd7894 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.services.update.FutureFactory; import fiftyone.pipeline.engines.services.update.FutureFactoryDefault; import fiftyone.pipeline.util.Check; +import fiftyone.pipeline.util.LicenseKeyMask; import jakarta.xml.bind.DatatypeConverter; import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; @@ -44,6 +45,7 @@ import java.io.*; import java.net.HttpURLConnection; import java.net.URI; +import java.net.URLDecoder; import java.nio.file.*; import java.security.DigestInputStream; import java.security.MessageDigest; @@ -64,6 +66,12 @@ */ public class DataUpdateServiceDefault implements DataUpdateService { + /** + * The query parameter of a data update URL that holds the license keys, + * as the 51Degrees URL formatter writes it. + */ + private static final String LICENSE_KEYS_PARAMETER = "LicenseKeys"; + final List configurations = new ArrayList<>(); private final Logger logger; private final boolean closeFutureFactory; @@ -601,6 +609,45 @@ private long applyIntervalRandomisationMillis( return intervalMillis + randomSeconds * 1000; } + /** + * The license keys to mask in messages that name the data update URL, + * being the data update license keys and any key in the URL's + * LicenseKeys parameter, such as one typed straight into the data + * update URL. + * @param dataFile the data file being updated + * @param url the formatted data update URL + * @return the keys to mask + */ + private static List licenseKeysToMask( + AspectEngineDataFile dataFile, + String url) { + List keys = new ArrayList<>(); + List configured = + dataFile.getConfiguration().getDataUpdateLicenseKeys(); + if (configured != null) { + keys.addAll(configured); + } + int query = url == null ? -1 : url.indexOf('?'); + if (query < 0) { + return keys; + } + for (String parameter : url.substring(query + 1).split("&")) { + int equals = parameter.indexOf('='); + if (equals > 0 && parameter.substring(0, equals) + .equalsIgnoreCase(LICENSE_KEYS_PARAMETER)) { + String value = parameter.substring(equals + 1); + try { + value = URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException | + IllegalArgumentException e) { + // Masked as it is written. + } + keys.addAll(Arrays.asList(value.split("\\|"))); + } + } + return keys; + } + /** * Get the most recent data file available from the configured update URL. * If the data currently used by the engine is the newest available then @@ -615,13 +662,17 @@ private AutoUpdateStatus downloadFile( FileWrapper tempFile) { String url = dataFile.getFormattedUrl(); - logger.debug("downloadFile from {}", url); + // The URL carries the license keys, so every message names it with + // each key masked. + List licenseKeys = licenseKeysToMask(dataFile, url); + String loggedUrl = LicenseKeyMask.maskIn(url, licenseKeys); + logger.debug("downloadFile from {}", loggedUrl); try { HttpURLConnection connection = httpClient.connect(new URI(url.trim()).toURL()); if (connection == null) { logger.error("No response from data update service at '{}' for engine '{}'", - dataFile.getFormattedUrl(), Objects.nonNull(dataFile.getEngine()) ? + loggedUrl, Objects.nonNull(dataFile.getEngine()) ? dataFile.getEngine().getClass().getSimpleName() : "No Engine"); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -645,19 +696,19 @@ private AutoUpdateStatus downloadFile( // in some versions of the HttpStatusCode enum. case (429): logger.error("Too many requests to '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS; case HttpURLConnection.HTTP_NOT_MODIFIED: logger.debug("No data update available from '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; case HttpURLConnection.HTTP_FORBIDDEN: logger.error("Access denied to data update service at '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN; default: logger.error("HTTP status code '{}' from data update service at '{}' for {}", - connection.getResponseCode(), dataFile.getFormattedUrl(), + connection.getResponseCode(), loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -665,7 +716,9 @@ private AutoUpdateStatus downloadFile( connection.disconnect(); } } catch (Exception e) { - logger.error("Error while processing data file download", e); + // An HTTP library's exception often names the URL it requested. + logger.error("Error while processing data file download", + LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java new file mode 100644 index 000000000..1ea82ece7 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java @@ -0,0 +1,223 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The data update URL carries the license keys in its query. Every log + * entry the data update service writes about a request, at any level and + * with any exception it logs, shows each key only by its last 4 + * characters. + */ +public class DataUpdateServiceLicenseKeyTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + private static final String UPDATE_URL = + "https://distributor.example/api/v2/download"; + + private Logger logger; + private HttpClient httpClient; + private HttpURLConnection connection; + private DataUpdateServiceDefault service; + private OnPremiseAspectEngine engine; + + @Before + @SuppressWarnings("unchecked") + public void init() throws IOException { + logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + connection = mock(HttpURLConnection.class); + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + service = new DataUpdateServiceDefault( + logger, + httpClient, + mock(FileWrapperFactory.class), + mock(FutureFactory.class)); + + // A data file updated in memory, whose URL carries the keys as the + // 51Degrees URL formatter puts them. + final DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL); + config.setDataUpdateLicenseKeys(Arrays.asList(KEY, OTHER_KEY)); + config.setUrlFormatter(dataFile -> config.getDataUpdateUrl() + + "?LicenseKeys=" + KEY + "|" + OTHER_KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + + engine = mock(OnPremiseAspectEngine.class); + when(engine.getDataFileMetaData()).thenReturn(file); + } + + @Test + public void Forbidden_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + assertLogsMaskedKeys(); + } + + @Test + public void TooManyRequests_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenReturn(429); + assertLogsMaskedKeys(); + } + + @Test + public void OtherStatus_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_INTERNAL_ERROR); + assertLogsMaskedKeys(); + } + + @Test + public void NotModified_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + assertLogsMaskedKeys(); + } + + @Test + public void NoResponse_LogsMaskedKeys() throws IOException { + when(httpClient.connect(any(URL.class))).thenReturn(null); + assertLogsMaskedKeys(); + } + + /** + * The exception an HTTP library raises often names the URL it + * requested, and the exception is logged with its stack trace. + */ + @Test + public void RequestException_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenThrow(new IOException( + "Server returned HTTP response code: 403 for URL: " + + UPDATE_URL + "?LicenseKeys=" + KEY + "|" + OTHER_KEY)); + assertLogsMaskedKeys(); + } + + /** + * A key typed straight into the data update URL, rather than given as + * a data update license key, is masked too. + */ + @Test + @SuppressWarnings("unchecked") + public void KeyTypedIntoTheUrl_LogsMaskedKey() throws IOException { + DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL + "?LicenseKeys=" + KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + OnPremiseAspectEngine typed = + mock(OnPremiseAspectEngine.class); + when(typed.getDataFileMetaData()).thenReturn(file); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + + service.checkForUpdate(typed); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertTrue(logged, logged.contains(MASKED)); + } + + private void assertLogsMaskedKeys() { + service.checkForUpdate(engine); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertFalse(logged, logged.contains(OTHER_KEY)); + assertTrue(logged, logged.contains(MASKED)); + assertTrue(logged, logged.contains(OTHER_MASKED)); + } + + /** + * Everything passed to the logger, at every level, with the stack trace + * of each throwable as it would be printed. + */ + private String logged() { + StringBuilder text = new StringBuilder(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + text.append(invocation.getMethod().getName()).append(':'); + for (Object argument : invocation.getArguments()) { + append(text, argument); + } + text.append('\n'); + } + return text.toString(); + } + + private static void append(StringBuilder text, Object argument) { + if (argument instanceof Object[]) { + for (Object item : (Object[]) argument) { + append(text, item); + } + } else if (argument instanceof Throwable) { + StringWriter trace = new StringWriter(); + ((Throwable) argument).printStackTrace(new PrintWriter(trace)); + text.append(' ').append(trace); + } else { + text.append(' ').append(argument); + } + } +} From 24c6f2587f0562ceb2d6bd1accf7940abc553248 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 13:49:39 +0000 Subject: [PATCH 65/95] Write weighted values with the lower-case keys rawweighting and value (#93) * Test that the JSON builder writes a weighted value with lower-case keys JsonBuilder_WeightedValues gives the JSON builder a property holding two weighted values and expects each to be written as an object with exactly the keys rawweighting and value, as the Serializing section of the weighted values page of the specification shows. It is written to fail on rc, where org.json writes the WeightedValue bean with the keys rawWeighting, value and weighting. * Write weighted values with the lower-case keys rawweighting and value The JSON builder handed WeightedValue instances to org.json, which wrote each as a bean with the keys rawWeighting, value and weighting. A list of weighted values is now written as objects holding only rawweighting and value, as the Serializing section of the weighted values page of the specification shows, and as .NET's WeightedValueConverter writes through its lower-case contract resolver. This is also the form the IP Intelligence cloud engine reads from the cloud service. --- .../flowelements/JsonBuilderElement.java | 27 ++++++++++++++ .../jsonbuilder/JsonBuilderTests.java | 36 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java index 23d4ddf6b..fcf920332 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java @@ -29,6 +29,7 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.data.IWeightedValue; import fiftyone.pipeline.core.data.PropertyMatcher; import fiftyone.pipeline.core.data.TryGetResult; import fiftyone.pipeline.core.data.factories.ElementDataFactory; @@ -62,6 +63,8 @@ public class JsonBuilderElement extends FlowElementBase implements JsonBuilder { private static final String JAVASCRIPT_PROPERTIES_NAME = "javascriptProperties"; + private static final String RAW_WEIGHTING_KEY = "rawweighting"; + private static final String VALUE_KEY = "value"; private final EvidenceKeyFilter evidenceKeyFilter; private final List properties; private final List blacklist; @@ -319,6 +322,11 @@ private Map getValues( } propertyValue = results; } + else if (propertyValue instanceof List && + ((List) propertyValue).size() > 0 && + ((List) propertyValue).get(0) instanceof IWeightedValue) { + propertyValue = getWeightedValues((List) propertyValue); + } // Add this value to the output values.put(value.getKey().toLowerCase(), propertyValue); @@ -339,6 +347,25 @@ private Map getValues( return values; } + /** + * Weighted values as objects with the lower-case keys rawweighting and + * value, as the weighted values page of the specification shows. + * @param weightedValues the weighted values + * @return the objects to write + */ + private static List getWeightedValues(List weightedValues) { + List results = new ArrayList<>(); + for (Object item : weightedValues) { + IWeightedValue weighted = (IWeightedValue) item; + Map result = new LinkedHashMap<>(); + result.put(RAW_WEIGHTING_KEY, weighted.getRawWeighting()); + result.put(VALUE_KEY, weighted.getValue() == null ? + JSONObject.NULL : weighted.getValue()); + results.add(result); + } + return results; + } + private void addJavaScriptProperties( FlowData data, Map allProperties) { diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index 965526329..944e99609 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -482,6 +482,42 @@ public void JsonBuilder_LazyLoading() throws Exception { } } + /** + * A weighted value is written as an object with the lower-case keys + * rawweighting and value, and no others. + * See the Serializing section of the weighted values page of the + * specification. + */ + @Test + public void JsonBuilder_WeightedValues() throws Exception { + List> weighted = new ArrayList<>(); + weighted.add(new WeightedValue<>(32768, "value 1")); + weighted.add(new WeightedValue<>(32767, "value 2")); + Map map = new HashMap<>(); + map.put("weighted", + new AspectPropertyValueDefault>>( + weighted)); + when(elementDataMock.asKeyMap()).thenReturn(map); + + String json = testIteration(1); + + JSONArray values = new JSONObject(json) + .getJSONObject("test") + .getJSONArray("weighted"); + assertEquals(2, values.length(), json); + for (int i = 0; i < values.length(); i++) { + JSONObject value = values.getJSONObject(i); + assertEquals( + new HashSet<>(Arrays.asList("rawweighting", "value")), + value.keySet(), + json); + assertEquals(weighted.get(i).getRawWeighting(), + value.getInt("rawweighting"), json); + assertEquals(weighted.get(i).getValue(), + value.getString("value"), json); + } + } + private String testIteration(int iteration) throws Exception { return testIteration(iteration, null); } From 6870059c9fb0aab8338f2a0352d946e541487379 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 13:57:34 +0000 Subject: [PATCH 66/95] Refresh the engine once when a data update is applied in memory (#92) * Test that an update applied in memory refreshes the engine once DataUpdateService_CheckForUpdate_InMemory_RefreshesOnce runs a check for a data file with an update URL and no file path, so the downloaded data is applied in memory. It counts every refreshData call on the engine and expects exactly one, the call with the downloaded data. * Refresh the engine once when an update is applied in memory When a data file has no file path, checkForUpdateFromUrl applies the downloaded data to the engine in memory, then returned AUTO_UPDATE_IN_PROGRESS. checkForUpdate took that to mean the engine had not been refreshed and called updatedFileAvailable, which refreshed the engine a second time through refreshData(String), the method that reloads from a data file. The in-memory path now returns AUTO_UPDATE_SUCCESS once the engine, or the configuration where there is no engine yet, holds the new data, as .NET's DataUpdateService does. checkForUpdate then clears the timer as updatedFileAvailable did, so the next check is registered as before. * Wait for the watcher's update in the watcher test, not the polling timer's check With a one second polling interval the timer's check can complete first, finding no update because the watcher is enabled, and release the test before the watcher has refreshed the engine. The failed run took 1.008 seconds. The flag is now released only by a completion whose status is AUTO_UPDATE_SUCCESS, which only the watcher's update gives. * Test that a manual update sets the next update timer again The data updates page of the specification on rc says the automatic update master switch "only governs *automatic* activity - it does not disable *Programmatic update*". A programmatic update processes a newer file "as per newFile above", and the steps of newFileAvailable are: - pause file watcher, if any - pause update polling, if any - load file - resume polling, if any - resume file watcher, if any - trigger onUpdateCallbacks Three tests cover a manual check on a data file registered with the service, for each setting of automatic updates. DataUpdateService_CheckForUpdate_FileUpdated_TimerSetAgain applies an update from the data file, and DataUpdateService_CheckForUpdate_InMemory_TimerSetAgain applies one downloaded in memory. Each expects a new timer where automatic updates are on. Both are written to fail where automatic updates are on, because the manual update clears the timer and only a check the timer started registers the data file again. DataUpdateService_CheckForUpdate_NoUpdate_TimerKept checks that a manual check finding no update leaves the existing timer as it was. It passes before and after the change. * Set the next update timer after a manual update, as the specification says The data updates page of the specification on rc says the automatic update master switch "only governs *automatic* activity - it does not disable *Programmatic update*". Under Programmatic update, a newer file is processed "as per newFile above", and the steps of newFileAvailable include "resume polling, if any" and "resume file watcher, if any". checkForUpdate registered the data file again only after a check the timer started. A manual update cleared the timer, through updatedFileAvailable or the in-memory success branch, and set no new one, so polling stopped. checkForUpdate now also registers the data file again after a manual check that applied an update, where the data file is registered with the service, so registerDataFile sets the next timer when automatic updates are on, as it does after an automatic check. A manual check that finds no update still leaves the timer as it was, and a data file that was never registered is not registered by a manual check. * Check that a manual update leaves one update timer, not two Both TimerSetAgain tests now keep the timer pending before the manual update and check, where automatic updates are on, that it was cancelled and that the data file holds a different one. With the existing check that exactly one more timer was scheduled, this shows a manual update with a timer pending leaves one live timer. The pending timer is cancelled and cleared under futureLock, by updatedFileAvailable for the file path or by the in-memory success branch, before registerDataFile schedules a new one, and registerDataFile schedules only while the future is null under the same lock. * Test that a failed manual refresh keeps polling, and that unregistering unregisters DataUpdateService_CheckForUpdate_FileRefreshFails_TimerSetAgain makes a manual check whose refresh from the data file fails, on a data file registered for automatic updates, and expects the next update timer to be set where automatic updates are on. The newFileAvailable steps in the data updates page of the specification say "resume polling, if any", with no exception for a failed load. DataUpdateService_Unregister_NoLongerRegistered expects a data file to be no longer registered after unregisterDataFile, so a manual update of it sets no timer. Both are written to fail on the previous commit. The first fails where automatic updates are on. * Resume polling after a failed manual refresh, and clear the registration on unregister updatedFileAvailable clears the update timer before it refreshes the engine, so after a manual check whose refresh failed polling stopped, where an automatic check registers the data file again. checkForUpdate now also registers a registered data file again after a manual check that returned AUTO_UPDATE_REFRESH_FAILED. A manual failure that did not clear the timer, such as one in memory, keeps the timer it had, because registerDataFile schedules only while there is none. So a manual check leaves one timer whether it applied an update, failed or found none. unregisterDataFile now sets the data file's update service to null, so getIsRegistered is false after it and a later manual update does not register the data file again. * Say that a refresh from the data file clears the update timer A comment-only change. The comment above the re-registration after a manual update said that the refresh clears the timer, which is true only of a refresh from the data file, because a failed refresh in memory leaves the timer as it was. No code changes. --- .../services/DataUpdateServiceDefault.java | 23 +- .../services/DataUpdateServiceTests.java | 293 ++++++++++++++++++ 2 files changed, 314 insertions(+), 2 deletions(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index e3bfd7894..6b8da4adf 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -234,6 +234,15 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { // yet been refreshed. logger.debug("Calling updatedFileAvailable"); result = updatedFileAvailable(dataFile); + } else if (result == AUTO_UPDATE_SUCCESS) { + // The update was applied in memory. Clear the timer, as + // updatedFileAvailable does, so a new one is registered. + synchronized (futureLock) { + if (dataFile.getFuture() != null) { + dataFile.getFuture().cancel(true); + dataFile.setFuture(null); + } + } } else { // No update available. // If this was a manual call to update then do nothing. @@ -257,9 +266,15 @@ public void run() { } } } - if (manualUpdate == false) { + if (manualUpdate == false || + ((result == AUTO_UPDATE_SUCCESS || + result == AUTO_UPDATE_REFRESH_FAILED) && + dataFile.getIsRegistered())) { // Re-register the engine with the data update service // so it knows when next set of data should be available. + // A manual update of a registered data file does this too, + // whether or not the refresh worked, as a refresh from the + // data file clears the timer, so polling resumes after it. registerDataFile(dataFile); } @@ -573,6 +588,9 @@ public void unregisterDataFile(AspectEngineDataFile dataFile) { } } } + if (dataFile != null) { + dataFile.setDataUpdateService(null); + } } /** @@ -858,7 +876,8 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { dataFile.getConfiguration().setData(reader.toByteArray()); } } - return memoryDownloadResult; + // The engine, or the configuration, now holds the new data. + return AUTO_UPDATE_SUCCESS; } catch (Exception ex) { logger.error("An error occurred when applying a data update to engine '{}'" , Objects.nonNull(dataFile.getEngine())? diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 9dc7bc483..4ed689fc2 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -45,6 +45,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; import org.mockito.internal.matchers.LessOrEqual; +import org.mockito.invocation.Invocation; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.LoggerFactory; @@ -61,6 +62,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; @@ -1233,6 +1235,297 @@ public boolean matches(byte[] argument) { verify(futureFactory, times(1)).schedule(any(Runnable.class), anyLong()); } + /** + * A check that finds an update for a data file with no file path + * applies it in memory, so it refreshes the engine once, with the + * downloaded data, and does not refresh it again. + */ + @Test + public void DataUpdateService_CheckForUpdate_InMemory_RefreshesOnce() + throws IOException { + // Arrange + configureNoFileSystem(); + when(httpClientConnection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(httpClientConnection.getResponseMessage()).thenReturn(null); + when(httpClientConnection.getHeaderField("Content-MD5")) + .thenReturn("08527dcbdd437e7fa6c084423d06dba6"); + when(httpClientConnection.getInputStream()).thenReturn( + DataUpdateServiceTests.class.getClassLoader() + .getResourceAsStream("file.gz")); + + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl("http://www.test.com"); + config.setVerifyMd5(true); + config.setDecompressContent(true); + config.setVerifyModifiedSince(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_SUCCESS, status); + List refreshes = new ArrayList<>(); + for (Invocation invocation : mockingDetails(engine).getInvocations()) { + if (invocation.getMethod().getName().equals("refreshData")) { + refreshes.add(invocation.getMethod().toGenericString()); + } + } + assertEquals( + "The engine was not refreshed exactly once: " + refreshes, + 1, + refreshes.size()); + verify(engine, times(1)).refreshData(any(), any(byte[].class)); + } + + /** + * A manual check that applies an update from the data file to a data + * file registered for automatic updates sets the next update timer, as + * a check the timer started does. + */ + @Test + public void DataUpdateService_CheckForUpdate_FileUpdated_TimerSetAgain() { + // Arrange + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setFileSystemWatcherEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + verify(futureFactory, times(timers)).schedule( + any(Runnable.class), + anyLong()); + + ScheduledFuture pending = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_SUCCESS, status); + verify(engine, times(1)).refreshData((String) any()); + verify(futureFactory, times(2 * timers)).schedule( + any(Runnable.class), + anyLong()); + assertEquals( + "The next update timer was not set after the manual update.", + autoUpdateEnabled, + file.getFuture() != null); + + if (autoUpdateEnabled) { + // The pending timer is cancelled, so only the new one is left. + assertTrue( + "The timer pending before the manual update was not cancelled.", + pending.isCancelled()); + assertNotSame(pending, file.getFuture()); + } + } + + /** + * A manual check whose refresh from the data file fails still leaves + * the next update timer set, as a check the timer started does, so + * polling resumes. + */ + @Test + public void DataUpdateService_CheckForUpdate_FileRefreshFails_TimerSetAgain() { + // updatedFileAvailable warns on each of its 10 tries. + ignoreWranings = 10; + + // Arrange + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + doThrow(new IllegalStateException("The refresh failed.")) + .when(engine).refreshData((String) any()); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setFileSystemWatcherEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + ScheduledFuture pending = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED, + status); + verify(futureFactory, times(2 * timers)).schedule( + any(Runnable.class), + anyLong()); + assertEquals( + "The next update timer was not set after the failed refresh.", + autoUpdateEnabled, + file.getFuture() != null); + if (autoUpdateEnabled) { + assertTrue(pending.isCancelled()); + assertNotSame(pending, file.getFuture()); + } + } + + /** + * A data file that has been unregistered is no longer registered, so a + * manual update of it sets no update timer. + */ + @Test + public void DataUpdateService_Unregister_NoLongerRegistered() { + // Arrange + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setFileSystemWatcherEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + + // Act + dataUpdate.unregisterDataFile(file); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertFalse(file.getIsRegistered()); + assertEquals(AUTO_UPDATE_SUCCESS, status); + verify(futureFactory, times(timers)).schedule( + any(Runnable.class), + anyLong()); + assertNull(file.getFuture()); + } + + /** + * A manual check that applies an update in memory to a data file + * registered for automatic updates sets the next update timer, as a + * check the timer started does. + */ + @Test + public void DataUpdateService_CheckForUpdate_InMemory_TimerSetAgain() + throws IOException { + // Arrange + configureNoFileSystem(); + when(httpClientConnection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(httpClientConnection.getResponseMessage()).thenReturn(null); + when(httpClientConnection.getHeaderField("Content-MD5")) + .thenReturn("08527dcbdd437e7fa6c084423d06dba6"); + when(httpClientConnection.getInputStream()).thenReturn( + DataUpdateServiceTests.class.getClassLoader() + .getResourceAsStream("file.gz")); + + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setDataUpdateUrl("http://www.test.com"); + config.setVerifyMd5(true); + config.setDecompressContent(true); + config.setVerifyModifiedSince(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + verify(futureFactory, times(timers)).schedule( + any(Runnable.class), + anyLong()); + + ScheduledFuture pending = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_SUCCESS, status); + verify(futureFactory, times(2 * timers)).schedule( + any(Runnable.class), + anyLong()); + assertEquals( + "The next update timer was not set after the manual update.", + autoUpdateEnabled, + file.getFuture() != null); + + if (autoUpdateEnabled) { + // The pending timer is cancelled, so only the new one is left. + assertTrue( + "The timer pending before the manual update was not cancelled.", + pending.isCancelled()); + assertNotSame(pending, file.getFuture()); + } + } + + /** + * A manual check that finds no update leaves the timer of a data file + * registered for automatic updates as it was. + */ + @Test + public void DataUpdateService_CheckForUpdate_NoUpdate_TimerKept() + throws IOException { + // Arrange + configureNoFileSystem(); + configureHttpNoUpdateAvailable(); + + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setDataUpdateUrl("http://www.test.com"); + config.setVerifyModifiedSince(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + dataUpdate.registerDataFile(file); + Object timer = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_NOT_NEEDED, status); + verify(engine, never()).refreshData(anyString()); + verify(futureFactory, times(autoUpdateEnabled ? 1 : 0)).schedule( + any(Runnable.class), + anyLong()); + assertSame(timer, file.getFuture()); + } + /** From ff6e4c0a851d9f46292611595daf3a5a2712755a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 16:28:15 +0000 Subject: [PATCH 67/95] Send only the Evidence the cloud accepts from the Cloud Request Engine (#89) * Test that a cloud engine whose product the resource key lacks builds and logs once A resource key can be valid and include no product that holds a cloud engine's properties. The Device Detection, IP Intelligence and Location cloud engines are tested to build with empty property metadata, to log the missing product once, to log and throw nothing when processing responses that hold nothing for them, and to report a property read as missing with ProductNotAccessibleWithResourceKey. The base class tests now expect a missing product to build with empty metadata and log once, keep the failure when the accessible properties are not loaded, and check that a property keyed engine gives no profiles. The Location cloud module gets Mockito as a test dependency for its first tests. * Build a cloud engine whose product the resource key lacks, log that once, and process quietly A resource key can be valid and include no product that holds a cloud engine's properties. The Device Detection, IP Intelligence and Location cloud engines then left their property metadata null, logged two errors, and threw on every request when getJSONObject found no part of the response for them. The engines built on CloudAspectEngineBase, such as the hardware profile engines, threw when their metadata was read. CloudAspectEngineBase now has findProduct, which returns the engine's product from the accessible products, or an empty product when the resource key does not include it, logging that once for each engine instance. getElementJson returns the engine's part of the response, or an empty object when the product is missing. The three engines, the base class's own metadata loading and the property keyed base use them, so each builds with empty property metadata, logs one error, and processes a response with nothing for it without logging or throwing. Reading a property then throws PropertyMissingException with ProductNotAccessibleWithResourceKey, as for any property the response does not carry. Accessible products that were never loaded still fail as before. The Location engine's failure message named the Device Detection engine, and now names the Location engine. * Test that the Cloud Request Engine discovers when it is built, and retries only after a failure that can pass CloudRequestEngineDiscoveryTests uses a stand-in for the HTTP client that counts the requests to each endpoint. A build fetches the accessible properties and the evidence keys once each and processing fetches neither again. A 4xx answer to either fails the build with a CloudRequestException carrying the status, once both requests are made, also when the other request failed in a way that can pass. A 5xx answer, a body that cannot be read, a refused connection and a timeout each complete the build with one warning. A list that arrived is kept, processing fetches only what is missing and sends no data request until both are known, the Pipeline's Evidence filter sees the keys a retry fetched, and the configured timeout applies to the discovery requests. The existing tests that expected getPublicProperties and getEvidenceKeyFilter to fetch and throw when the server was unavailable, and a 200 answer holding errors to throw, now expect the build to complete with one warning. The cloud aspect engines are tested to build without logging when the accessible products are not known yet, and to read their metadata once they are. * Discover when the Cloud Request Engine is built, and retry on first use only after a failure that can pass The engine fetched its accessible properties and accepted evidence keys only when something first asked for them, and a failure surfaced wherever that was. The builder now calls discover, which makes both requests. A 4xx answer to either is a definitive configuration error, and its CloudRequestException fails the build once both requests are made. Any other failure, the service unreachable, a timeout, a 5xx answer or a body that cannot be read, is logged as one warning, the build completes and the list that arrived is kept. The configured timeout now applies to both requests. getPublicProperties and getEvidenceKeyFilter make no request. The products are null until they are known. The evidence filter is one CloudEvidenceKeyFilter that includes nothing until the keys are known and reads them on each call, and it derives from EvidenceKeyFilterWhitelist so the Pipeline's filter keeps it rather than copying its keys once. Processing fetches what is missing, evidence keys first, before it sends the data request, and a request that could not be made is reported as its IOException, as for the data request. The Device Detection, IP Intelligence and Location cloud engines, and CloudAspectEngineBase, read their metadata when the products are known rather than logging errors at build and keeping none. A missing product still builds and logs once. The web integration reads a filter that derives from EvidenceKeyFilterWhitelist for the Vary header, as it reads the exact class. * Let a Cloud Request Engine be built from a kept copy of what an earlier one discovered The specification's Resilience section says an engine SHOULD be buildable from a kept copy of the discovery results, the accepted evidence keys and the accessible properties, which depend only on the resource key, so that a Pipeline can be built offline and a host that starts often need not call the cloud service each time. CloudDiscoveryState holds the evidence keys and the accessibleproperties answer, and writes and reads them as one JSON document. CloudRequestEngineBuilder.getDiscoveryState returns the copy after a build that knows both lists, and setDiscoveryState makes a later build use it with no discovery request. The tests are in this commit because they use the new class and methods. * Test that discovery makes both requests at once, times out after 2 seconds, and that the Pipeline reads metadata again once it is known Each discovery request in the new test waits for the other to start, so they only succeed if they are made at once. A build with no timeout set applies 2 seconds to both, as the page's Configuration options table and .NET give. The Pipeline's map of available properties, built before the Cloud Request Engine knows the accessible properties, holds the cloud aspect engine's properties once they are known. The shared test client now gives each request a connection of its own, as the requests can be made at the same time. * Make the discovery requests at once, time out after 2 seconds by default, and read metadata again until it is loaded The two discovery requests are now made at once, as the specification allows, so a service that does not answer delays the build by one timeout rather than two. The evidence keys request runs on another thread while the accessible properties request runs on the building thread, and the build waits for both. The default timeout for each request to the cloud service is now 2 seconds, as the specification's Configuration options table gives and as .NET's CLOUD_REQUEST_TIMEOUT_DEFAULT_SECONDS is. It was 100 seconds. FlowElement gains hasLoadedProperties, true by default, as .NET's aspect engines have HasLoadedProperties. The Cloud Request Engine returns true once both discovery lists are known, the cloud aspect engines once their metadata is loaded, and a parallel group once all its elements have. PipelineDefault keeps its map of available properties only once every element has loaded its properties, and builds it again on each call until then, as .NET's Pipeline.ElementAvailableProperties does after a PropertiesNotYetLoadedException. The web integration works out the Vary headers again on each response until every element has loaded, as .NET's ClientsidePropertyService.HeadersAffectingJavaScript does after an exception from an element's EvidenceKeyFilter. * Test that the SetHeaders element keeps its configuration once every element has loaded its properties With every element loaded, the configuration built from the Pipeline's available properties is built once and kept across requests. While an element has not loaded its properties, it is built again on each request, and kept once every element has loaded. Both fail on the code before the fix, where the configuration was built again on every request, because getOrDefault never stored it. * Keep the SetHeaders element's configuration once every element has loaded its properties The SetHeaders element built its configuration from the Pipeline's available properties on every request, because getOrDefault never stored it. It now keeps it once every element of the Pipeline has loaded its properties, and builds it again on each request until then, as after a cloud element could not reach its service when the Pipeline was built. So the configuration gains the cloud's SetHeader properties once a retry fetches them, and is then built once. This is the same rule the Pipeline's map of available properties and the web integration's Vary header follow. * Give the SetHeaders test mocks their type parameters, as the build treats raw types as errors * Test that the Cloud Request Engine sends only the Evidence the cloud accepts The Processing section of the cloud request engine specification says the engine sends only the Evidence keys the cloud service needs, and never Evidence whose key strips to resource, license or values. Two new tests check the posted request. process_SendsOnlyAcceptedEvidence sends Evidence the service does not list and expects only the listed keys, compared without regard to case. process_LeavesOutEvidenceNamedResourceLicenseOrValues sends Evidence named resource, license and values in several cases, with and without a configured license key, and expects the configured keys once each and nothing else. Existing tests that check the request now answer the evidencekeys request with the keys they send, as the cloud service would. * Send only the Evidence the cloud accepts from the Cloud Request Engine The Cloud Request Engine sends only the Evidence whose keys the cloud service accepts, from its evidencekeys request, and never Evidence whose key strips to resource, license or values, compared without regard to case, as the Processing section of the cloud request engine specification says. The request carries the configured resource key, and license key where one is configured, once each. Before, the engine sent all Evidence except the web request. Conflicts between prefixes are now reported only among the Evidence being sent. * Mark the mocked product map unchecked, as the build treats the varargs warning as an error * Mark the mocked product map unchecked, as the build treats the varargs warning as an error * Mark the mocked product map unchecked, as the build treats the varargs warning as an error * Stub the product map once per call, so no varargs array is created thenReturn(null, products) uses the varargs overload, and a generic Map array is an unchecked warning that the build treats as an error, so each stub now chains two single-value thenReturn calls. * Test that a 408 or 429 at discovery is transient and does not fail the build The specification now treats a 408 or a 429 answer to a discovery request as a transient failure, because the same request can succeed later, so 429 moves from the cases that fail the build to those that do not, and 408 cases join it. * Treat a 408 or 429 at discovery as transient, so it does not fail the build A build fails on a 4xx answer because no retry with the same configuration can succeed, which is not so for a 408 or a 429, as RFC 9110 section 15.5.9 and RFC 6585 section 4 say, so those two are logged as a warning and retried on first use like a 5xx. * Test that the Cloud Request Engine never sends Evidence named licence The Processing section of the cloud request engine specification now also leaves out Evidence whose key strips to licence, because the cloud service reads the license key from licence where license is absent. process_LeavesOutEvidenceNamedLicence lists query.licence and cookie.LICENCE as accepted, adds Evidence under each, and expects only the configured keys and User-Agent, with and without a license key. It is written to fail on the previous commit, which sends both as licence and LICENCE. * Never send Evidence named licence from the Cloud Request Engine Evidence whose key strips to licence, compared without regard to case, is now left out of the cloud request along with resource, license and values, as the Processing section of the cloud request engine specification says, because the cloud service reads the license key from licence where license is absent. * Revert "Treat a 408 or 429 at discovery as transient, so it does not fail the build" This reverts commit 7bc481c8fc509ccbea0eb3dbc9f6138bbe3daede. * Revert "Test that a 408 or 429 at discovery is transient and does not fail the build" This reverts commit 40452b7d5d2a5a4de52ab482ccfa56dbd73f70a3. * Test that a missing product is a warning and a missing response part no error The Start-up activity section of the cloud aspect engine specification on rc says the engine "MUST log the missing product once per Cloud Aspect Engine instance, at warning level, naming its Data Key". The Processing section says "Where the response JSON has no portion for the Engine's Data Key, the Cloud Aspect Engine leaves the Element Data without values and raises no error." The missing product tests now expect one warning and no error. ProductPresent_ResponseWithoutPart_NoValues, for each of the Device Detection, IP Intelligence and Location cloud engines, includes the engine's product and processes a response with no part for it, and expects no exception and a property without a value. propertyKeyed_productPresent_noProfiles does the same for a property keyed engine, with a response that has no part for it and one whose part has no profiles. All are written to fail on the previous commit, which logs an error and throws on a missing part when the product is present. * Log a missing product as a warning, and treat a missing response part as no values The Start-up activity section of the cloud aspect engine specification on rc says the missing product is logged "once per Cloud Aspect Engine instance, at warning level, naming its Data Key". findProduct now logs it with logger.warn. The Processing section says "Where the response JSON has no portion for the Engine's Data Key, the Cloud Aspect Engine leaves the Element Data without values and raises no error." getElementJson now returns an empty object whenever the response has no part for the engine, whether or not the product is included, and PropertyKeyedCloudEngineBase reads profiles with optJSONArray, so a part without profiles gives none. The productMissing flag and isProductMissing, which only chose between those two behaviours, are removed. * Test that a 408 at discovery fails the build, as every 4xx does The specification on rc makes every 4xx answer to a discovery request a definitive configuration error, so the cases that fail the build now include 408 beside 429. * Test that a 4xx at discovery fails the build when its body is not JSON A 4xx answer is a definitive configuration error whatever its body holds, such as an error page from a proxy. The accessibleproperties answer is read as JSON before its status is used, so a 403 with an HTML body there fails this test, as the build completes with a warning. * Keep the status of a cloud answer whose body is not JSON An answer that cannot be read as JSON now raises a CloudRequestException carrying its HTTP status, as .NET does, so a 4xx answer to a discovery request fails the build whatever its body holds, as the specification says. * Rewrap the findProduct Javadoc paragraph A comment-only change. The paragraph that describes findProduct left "returned, so the engine builds" on a short line of its own, so the paragraph is now wrapped at 80 columns. No code changes. --- .../CloudRequestEngineDefault.java | 51 +++++- .../flowelements/CloudRequestEngineTests.java | 149 +++++++++++++++++- 2 files changed, 194 insertions(+), 6 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 17ea52da8..1ca5d11c3 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -77,6 +77,17 @@ public class CloudRequestEngineDefault */ private static final String HTTP_REQUEST_KEY = "server.http-request"; + /** + * Form fields the request reserves for the credentials and the list of + * Properties, with licence, which the cloud service reads as the + * license key where license is absent. Evidence whose key strips to one + * of them is never sent, as the Processing section of the + * specification requires. + */ + private static final Set ENGINE_FIELDS = + Collections.unmodifiableSet(new HashSet<>( + Arrays.asList("resource", "license", "licence", "values"))); + private HttpClient httpClient; private String endPoint; @@ -456,10 +467,21 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { return string.getBytes(StandardCharsets.UTF_8); } + /** + * The Evidence to send, keyed by the name the cloud service expects. + * Only Evidence that {@link #isSent(String)} allows is included, and + * where two prefixes give the same name the precedence order decides. + * @param flowData the FlowData + * @return the form fields to send + */ Map getFormData(FlowData flowData) { - Map evidence = new HashMap<>( - flowData.getEvidence().asKeyMap()); - evidence.keySet().removeIf(HTTP_REQUEST_KEY::equalsIgnoreCase); + Map evidence = new HashMap<>(); + for (Map.Entry entry : + flowData.getEvidence().asKeyMap().entrySet()) { + if (isSent(entry.getKey())) { + evidence.put(entry.getKey(), entry.getValue()); + } + } Map formData = new HashMap<>(); // Add evidence in reverse alphabetical order, excluding special keys. @@ -474,10 +496,29 @@ Map getFormData(FlowData flowData) { return formData; } + /** + * Whether Evidence is sent to the cloud service. Only keys the service + * accepts are sent, never one that strips to a name in + * {@link #ENGINE_FIELDS}, compared without regard to case, and never + * the web request. + * @param key the Evidence key + * @return true if the Evidence is sent + */ + private boolean isSent(String key) { + if (HTTP_REQUEST_KEY.equalsIgnoreCase(key) || + evidenceKeyFilter.include(key) == false) { + return false; + } + String[] parts = key.split( + Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); + return parts.length == 2 && + ENGINE_FIELDS.contains(parts[1].toLowerCase(Locale.ROOT)) == false; + } + /** * Add form data to the evidence. * @param formData the destination map to add the data to - * @param allEvidence all evidence in the FlowData. This is used to + * @param allEvidence all the Evidence being sent. This is used to * report which evidence keys are conflicting * @param evidence evidence to add to the form data */ @@ -539,7 +580,7 @@ private void addFormData( /** * Get evidence with specified prefix. - * @param evidence all evidence in the FlowData + * @param evidence the Evidence being sent * @param type required evidence key prefix * @return evidence with the required key prefix */ diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index f6ad4c5d3..9e9f3a006 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -33,11 +33,14 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.slf4j.ILoggerFactory; @@ -48,7 +51,9 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -394,6 +399,7 @@ public void evidencePrecidence(String name, boolean shouldWarn, String evidence1 loggerFactory.loggers.clear(); String[] evidence1Parts = evidence1.split("="); String[] evidence2Parts = evidence2.split("="); + acceptEvidence(Arrays.asList(evidence1Parts[0], evidence2Parts[0])); configureMockedClient(); CloudRequestEngine engine = new CloudRequestEngineBuilder(loggerFactory, httpClient) @@ -514,6 +520,7 @@ private static Stream getFormDataArgs() { @ParameterizedTest(name = "{0}") @MethodSource("getFormDataArgs") public void getFormData(String name, Map evidence, String expectedValue) throws Exception { + acceptEvidence(evidence.keySet()); configureMockedClient(); CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) .setResourceKey("resourcekey") @@ -535,10 +542,12 @@ public void getFormData(String name, Map evidence, String expect /** * The web integration adds the whole web request as server.http-request * for usage sharing to read. It holds every header and cookie, so it is - * never sent to the cloud service. + * never sent to the cloud service, even where the service lists its key. */ @Test public void getFormData_LeavesOutTheWebRequest() throws Exception { + acceptEvidence(Arrays.asList( + "header.User-Agent", "server.client-ip", "server.http-request")); configureMockedClient(); CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) .setResourceKey("resourcekey") @@ -571,6 +580,7 @@ public void getFormData_LeavesOutTheWebRequest() throws Exception { */ @Test public void getFormData_KeepsDotsInEvidenceName() throws Exception { + acceptEvidence(Arrays.asList("query.id.usage", "header.id.email")); configureMockedClient(); CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) @@ -598,6 +608,7 @@ public void getFormData_KeepsDotsInEvidenceName() throws Exception { */ @Test public void process_PostsWholeEvidenceName() throws Exception { + acceptEvidence(Arrays.asList("query.id.usage")); configureMockedClient(); CloudRequestEngine engine = new CloudRequestEngineBuilder(loggerFactory, httpClient) @@ -619,6 +630,142 @@ public void process_PostsWholeEvidenceName() throws Exception { "The posted body does not carry id.usage: " + items); } + /** + * Only the Evidence whose keys the cloud service accepts is sent, with + * the keys compared without regard to case. + * See the Processing section of the cloud request engine specification. + */ + @Test + public void process_SendsOnlyAcceptedEvidence() throws Exception { + acceptEvidence(Arrays.asList( + "query.User-Agent", "header.Accept-Language")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("header.accept-language", "en-GB"); + evidence.put("header.X-Other", "other"); + evidence.put("cookie.session", "abc"); + evidence.put("server.client-ip", "1.2.3.4"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=iPhone", + "accept-language=en-GB")), + new HashSet<>(items), + "The request carried Evidence the cloud service does not " + + "accept: " + items); + } + + /** + * Evidence whose key strips to resource, license or values is never + * sent, whatever its case and even where the cloud service lists the + * key, so the request carries the configured resource key, and the + * license key where one is configured, once each. + * See the Processing section of the cloud request engine specification. + */ + @ParameterizedTest(name = "license key {0}") + @NullSource + @ValueSource(strings = {"ABCDEFG"}) + public void process_LeavesOutEvidenceNamedResourceLicenseOrValues( + String licenseKey) throws Exception { + acceptEvidence(Arrays.asList( + "query.User-Agent", + "query.resource", + "header.Resource", + "query.license", + "cookie.LICENSE", + "query.values")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("query.resource", "evidence-resource"); + evidence.put("header.Resource", "header-resource"); + evidence.put("query.license", "evidence-license"); + evidence.put("cookie.LICENSE", "cookie-license"); + evidence.put("query.values", "ismobile"); + + List items = postedItems(licenseKey, evidence); + + List expected = new ArrayList<>(); + expected.add("resource=resourcekey"); + if (licenseKey != null) { + expected.add("license=" + licenseKey); + } + expected.add("User-Agent=iPhone"); + assertEquals(expected, items); + } + + /** + * Evidence whose key strips to licence, whatever its case, is never + * sent, because the cloud service reads the license key from licence + * where license is absent. + * See the Processing section of the cloud request engine specification. + */ + @ParameterizedTest(name = "license key {0}") + @NullSource + @ValueSource(strings = {"ABCDEFG"}) + public void process_LeavesOutEvidenceNamedLicence( + String licenseKey) throws Exception { + acceptEvidence(Arrays.asList( + "query.User-Agent", + "query.licence", + "cookie.LICENCE")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("query.licence", "evidence-licence"); + evidence.put("cookie.LICENCE", "cookie-licence"); + + List items = postedItems(licenseKey, evidence); + + List expected = new ArrayList<>(); + expected.add("resource=resourcekey"); + if (licenseKey != null) { + expected.add("license=" + licenseKey); + } + expected.add("User-Agent=iPhone"); + assertEquals(expected, items); + } + + /** + * Answer the evidencekeys request with the given keys. + * @param keys the Evidence keys the cloud service accepts + */ + private void acceptEvidence(Collection keys) { + evidenceKeysResponse = new JSONArray(keys).toString(); + } + + /** + * Process the Evidence with an engine on the resource key + * {@code resourcekey} and return the fields of the one request it + * posted, in order. + * @param licenseKey the license key, or null for none + * @param evidence the Evidence to process + * @return the posted fields + */ + private List postedItems( + String licenseKey, + Map evidence) throws Exception { + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setLicenseKey(licenseKey) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + data.process(); + } + + ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); + verify(httpClient, times(1)).postData(any(), any(), body.capture()); + return Arrays.asList( + new String(body.getValue(), StandardCharsets.UTF_8).split("&")); + } + /** * Verify that the request to the cloud service will contain * the configured origin header value. From d39b6afebc9f8cf1f367c94ba7828fa27d09bcf0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:43:39 +0000 Subject: [PATCH 68/95] Give a null in the Location cloud response no value, with its nullreason (#96) * Test that a null in the Location cloud response has no value A property the cloud response gives as null must have no value, with the text under the property's name followed by nullreason as its message. The tests read the property by name, from the Element Data's map that the JSON Builder reads, and from the typed getter. A property with a value must give that value, and a property the response leaves out must have no value. Mockito is added to the geo-location.cloud tests to stand in for the Cloud Request Engine, the Pipeline and the Flow Data. * Give a null in the Location cloud response no value, with its nullreason The Location cloud engine put each value from the response into the Element Data unwrapped. org.json reads a JSON null as JSONObject.NULL, which is not null, so a property the cloud gave as null held JSONObject.NULL when read by name or from the map, and the JSON Builder wrote it as null with no nullreason. Only the typed getters gave no value, through a separate map of reasons that the engine handed to the Element Data. The engine now stores every property as an AspectPropertyValue, as the Device Detection and IP Intelligence cloud engines do. A JSON null, or no entry, is no value with the text under the property's name followed by nullreason as its message, or Unknown when the response gives none. CloudGeoDataDefault no longer wraps values itself, so its map of reasons, setNoValueReasons on it and on CloudGeoData, and its getters, which only repeated those of GeoDataDefault, are removed. The live value test expected the unwrapped value. It now expects an AspectPropertyValue, and a reason where there is no value, as the Device Detection cloud value test does. * Say what the live Location value test checks for a property with no value The comment described what the cloud does. The test only checks that a property with no value carries a message, because its type cannot be read, and the engine always sets a message. The comment and the assertion message now say that. * Test that a Location property the response leaves out is missing A property the cloud response leaves out altogether must throw PropertyMissingException when read, as the Missing Properties section of the properties specification says and as .NET does. A response with no Location part leaves out every property, so the no part test now expects the exception rather than no value. A property given as null with no nullreason must have no value with the message Unknown, which no test covered once the left out case changed. CloudGeoData.setNoValueReasons stays, deprecated, so callers still compile. A test calls it and checks that the messages the engine set are unchanged. * Leave out a Location property the response leaves out, and keep setNoValueReasons The Location cloud engine gave a property the response leaves out no value with the message Unknown. It now stores no entry for that property, so reading it throws PropertyMissingException, as the Missing Properties section of the properties specification says and as .NET does. A property the response gives as null still has no value, with its nullreason as the message. CloudGeoData.setNoValueReasons and its implementation in CloudGeoDataDefault are back, deprecated, so callers still compile. The method has no effect, because the engine sets each message on the value. --- .../geolocation/data/CloudGeoData.java | 13 +- .../geolocation/data/CloudGeoDataDefault.java | 139 +---------- .../flowelements/GeoLocationCloudEngine.java | 78 ++++-- .../GeoLocationCloudEngineNullValueTests.java | 230 ++++++++++++++++++ .../GeoLocationCloudEngineTests.java | 13 +- .../geolocation/data/tests/ValueTests.java | 19 +- 6 files changed, 335 insertions(+), 157 deletions(-) create mode 100644 location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java index 181e740f4..5ca54ae3b 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java @@ -26,9 +26,16 @@ import java.util.Map; /** - * - * @author joseph + * The Location Element Data from the cloud. */ public interface CloudGeoData extends GeoData { - public void setNoValueReasons(Map value); + + /** + * Has no effect. + * @param value the messages for properties with no value, by name + * @deprecated The cloud engine sets the message of each property with no + * value on the value itself. + */ + @Deprecated + void setNoValueReasons(Map value); } diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java index 4a217422d..f6080b94c 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java @@ -24,145 +24,32 @@ import fiftyone.geolocation.core.data.GeoDataDefault; import fiftyone.pipeline.core.data.FlowData; -import fiftyone.pipeline.core.data.TryGetResult; -import fiftyone.pipeline.core.data.types.JavaScript; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; -import fiftyone.pipeline.engines.data.AspectPropertyValue; -import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.services.MissingPropertyService; import java.util.Map; -import java.util.Objects; -import org.json.JSONObject; import org.slf4j.Logger; +/** + * The Location Element Data from the cloud. The cloud engine stores each + * property the response holds as an AspectPropertyValue, so the getters of + * GeoDataDefault return them as they are. + */ public class CloudGeoDataDefault extends GeoDataDefault implements CloudGeoData { - private Map noValueReasons; - public CloudGeoDataDefault(Logger logger, FlowData flowData, AspectEngine engine, MissingPropertyService missingPropertyService) { super(logger, flowData, engine, missingPropertyService); } - public void setNoValueReasons(Map value){ - noValueReasons = value; - } - - @SuppressWarnings("unchecked") - @Override - public AspectPropertyValue getJavaScript() { - return getAs("javascript", AspectPropertyValue.class, JavaScript.class); - } - - @Override - public AspectPropertyValue getRoad() { - return getValueAsString("road"); - } - - @Override - public AspectPropertyValue getTown() { - return getValueAsString("town"); - } - - @Override - public AspectPropertyValue getSuburb() { - return getValueAsString("suburb"); - } - - @Override - public AspectPropertyValue getCounty() { - return getValueAsString("county"); - } - - @Override - public AspectPropertyValue getRegion() { - return getValueAsString("region"); - } - - @Override - public AspectPropertyValue getState() { - return getValueAsString("state"); - } - - @Override - public AspectPropertyValue getZipCode() { - return getValueAsString("zipcode"); - } - - @Override - public AspectPropertyValue getCountry() { - return getValueAsString("country"); - } - - @Override - public AspectPropertyValue getCountryCode() { - return getValueAsString("countrycode"); - } - - @Override - public AspectPropertyValue getCountryCode3() { - return getValueAsString("countrycode3"); - } - - @Override - public AspectPropertyValue getTimeZoneOffset() { - return getValueAs("timezoneoffset"); - } - + /** + * Has no effect. + * @param value the messages for properties with no value, by name + * @deprecated The cloud engine sets the message of each property with no + * value on the value itself. + */ + @Deprecated @Override - public AspectPropertyValue getAddress() { - return getValueAsString("address"); + public void setNoValueReasons(Map value) { } - - private AspectPropertyValue getValueAsString(String key) { - return getValueAs(key); - } - - @SuppressWarnings("unchecked") - private AspectPropertyValue getValueAs(String key) { - AspectPropertyValue value = getAs(key, AspectPropertyValue.class); - - if(noValueReasons.containsKey(key)) - value.setNoValueMessage(noValueReasons.get(key)); - - return value; - } - - @Override - protected TryGetResult tryGetValue(String key, Class type, Class... parameterisedTypes) { - if (AspectPropertyValue.class.isAssignableFrom(type)) { - TryGetResult result = new TryGetResult<>(); - Map map = asKeyMap(); - if (map.containsKey(key)) { - Object obj = map.get(key); - - try { - Object temp = setValue(key, obj); - T value = type.cast(temp); - result.setValue(value); - } catch (ClassCastException e) { - throw new ClassCastException("Expected property '" + key + - "' to be of type '" + type.getSimpleName() + - "' but it is '" + obj.getClass().getSimpleName() + "'"); - } - } - return result; - } - else { - return super.tryGetValue(key, type, parameterisedTypes); - } - } - - private AspectPropertyValue setValue(String key, Object obj) { - AspectPropertyValue temp = new AspectPropertyValueDefault(); - if (Objects.isNull(obj) || obj == JSONObject.NULL) { - temp.setNoValueMessage(noValueReasons.get(key)); - } - else { - temp.setValue(obj); - } - return temp; - } - } diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index cde6a4932..6c8eae436 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -37,6 +37,8 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import org.json.JSONObject; import org.slf4j.Logger; @@ -141,39 +143,34 @@ protected void processEngine(FlowData data, CloudGeoData aspectData) { Map methodData = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (AspectPropertyMetaData property : getProperties()) { + String key = property.getName().toLowerCase(); + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (geoObj.has(key) == false) { + continue; + } + Object value; switch(property.getType().getSimpleName()) { case ("List"): - JSONArray jsonArray = geoObj.optJSONArray(property.getName().toLowerCase()); - methodData.put( - property.getName(), - jsonArray == null ? null : jsonArray.toList()); + JSONArray jsonArray = geoObj.optJSONArray(key); + value = jsonArray == null ? null : jsonArray.toList(); break; case ("JavaScript"): - String javascript = geoObj.optString(property.getName().toLowerCase(), null); - methodData.put( - property.getName(), - javascript == null ? null : new JavaScript(javascript)); + String javascript = geoObj.optString(key, null); + value = javascript == null ? + null : + new JavaScript(javascript); break; default: - methodData.put( - property.getName(), - geoObj.opt(property.getName().toLowerCase())); - break; + value = geoObj.opt(key); + break; } + methodData.put( + property.getName(), + toAspectPropertyValue(geoObj, key, value)); } aspectData.populateFromMap(methodData); - - Map noValueReasonsMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - for(Map.Entry entry : methodData.entrySet()){ - if("null".equals(String.valueOf(entry.getValue()))){ - String key = entry.getKey().toLowerCase(); - Object reason = tryToGet(geoObj, key); - noValueReasonsMap.put(entry.getKey().toLowerCase(), (reason == null ? "" : reason.toString())); - } - } - - aspectData.setNoValueReasons(noValueReasonsMap); } else { throw new PipelineConfigurationException( @@ -187,6 +184,41 @@ public static Object tryToGet(JSONObject jsonObj, String key) { return jsonObj.opt(key + "nullreason"); } + /** + * The message for a property with no value when the response gives no + * reason. + */ + private static final String UNKNOWN_NULL_REASON = "Unknown"; + + /** + * Wrap a value read from the response. A JSON null is no value, with the + * reason the response gives under the property's name followed by + * nullreason. + * @param geoObj the response's part for this engine + * @param key the lower case property name + * @param value the value read, null or JSONObject.NULL where the + * response has none + * @return the property value + */ + private static AspectPropertyValue toAspectPropertyValue( + JSONObject geoObj, + String key, + Object value) { + AspectPropertyValue result = + new AspectPropertyValueDefault<>(); + if (value == null || value == JSONObject.NULL) { + Object reason = tryToGet(geoObj, key); + result.setNoValueMessage( + reason == null || reason == JSONObject.NULL ? + UNKNOWN_NULL_REASON : + reason.toString()); + } + else { + result.setValue(value); + } + return result; + } + @Override protected void unmanagedResourcesCleanup() { } diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java new file mode 100644 index 000000000..8bfe09c1b --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java @@ -0,0 +1,230 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.geolocation.data.CloudGeoDataDefault; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A property the cloud response gives as null has no value, and its message + * is the text under the property's name followed by nullreason, as the + * Processing section of the cloud aspect engine specification says. A + * property the response leaves out is missing, so reading it throws, as the + * Missing Properties section of the properties specification says. The tests + * read each property by name, from the map the JSON Builder reads, and from + * the typed getter. + */ +public class GeoLocationCloudEngineNullValueTests { + + private static final String SUBURB_NULL_REASON = + "No suburb was found at this location."; + + private static final String NULL_SUBURB_RESPONSE = + "{ 'location': { " + + "'suburb': null, " + + "'suburbnullreason': '" + SUBURB_NULL_REASON + "', " + + "'town': 'Reading' }, " + + "'javascriptProperties': [] }"; + + private static final String NULL_SUBURB_WITHOUT_REASON_RESPONSE = + "{ 'location': { 'suburb': null, 'town': 'Reading' }, " + + "'javascriptProperties': [] }"; + + private static final String NO_SUBURB_RESPONSE = + "{ 'location': { 'town': 'Reading' }, " + + "'javascriptProperties': [] }"; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void NullValue_NoValueWithNullReason() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NULL_SUBURB_RESPONSE); + + assertNoValue(SUBURB_NULL_REASON, data.get("Suburb")); + assertNoValue(SUBURB_NULL_REASON, data.asKeyMap().get("Suburb")); + assertNoValue(SUBURB_NULL_REASON, data.getSuburb()); + } + } + + @Test + public void Value_HasValue() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NULL_SUBURB_RESPONSE); + + assertValue("Reading", data.get("Town")); + assertValue("Reading", data.asKeyMap().get("Town")); + assertValue("Reading", data.getTown()); + } + } + + @Test + public void NullValueWithoutReason_NoValueUnknown() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process( + engine, + NULL_SUBURB_WITHOUT_REASON_RESPONSE); + + assertNoValue("Unknown", data.get("Suburb")); + assertNoValue("Unknown", data.asKeyMap().get("Suburb")); + assertNoValue("Unknown", data.getSuburb()); + } + } + + @Test + public void NoEntry_PropertyIsMissing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NO_SUBURB_RESPONSE); + + assertFalse(data.asKeyMap().containsKey("Suburb")); + assertMissing(() -> data.get("Suburb")); + assertMissing(() -> data.getSuburb()); + } + } + + @Test + @SuppressWarnings("deprecation") + public void SetNoValueReasons_ChangesNothing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NULL_SUBURB_RESPONSE); + Map reasons = new HashMap<>(); + reasons.put("suburb", "Another reason"); + data.setNoValueReasons(reasons); + + assertNoValue(SUBURB_NULL_REASON, data.get("Suburb")); + assertNoValue(SUBURB_NULL_REASON, data.getSuburb()); + } + } + + private static void assertMissing(Runnable read) { + try { + read.run(); + fail("Expected a PropertyMissingException"); + } + catch (PropertyMissingException e) { + // The property is not in the result, as expected. + } + } + + private static AspectPropertyValue asAspectPropertyValue(Object value) { + assertTrue( + "Expected an AspectPropertyValue but found " + + (value == null ? "null" : value.getClass().getName()), + value instanceof AspectPropertyValue); + return (AspectPropertyValue) value; + } + + private static void assertNoValue(String reason, Object value) { + AspectPropertyValue propertyValue = asAspectPropertyValue(value); + assertFalse( + "Expected no value but found " + propertyValue, + propertyValue.hasValue()); + assertEquals(reason, propertyValue.getNoValueMessage()); + } + + private static void assertValue(Object expected, Object value) + throws Exception { + AspectPropertyValue propertyValue = asAspectPropertyValue(value); + assertTrue( + "Expected a value but found none, because " + + propertyValue.getNoValueMessage(), + propertyValue.hasValue()); + assertEquals(expected, propertyValue.getValue()); + } + + /** + * An engine whose Cloud Request Engine reports the Location product with + * the Suburb and Town properties. + */ + private GeoLocationCloudEngine buildEngine() throws Exception { + GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("Suburb", "String", "Location", null), + new PropertyMetaData("Town", "String", "Location", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("location", product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the given cloud response. + */ + private CloudGeoData process( + GeoLocationCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + CloudGeoData data = new CloudGeoDataDefault( + loggerFactory.getLogger(CloudGeoData.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, data); + return data; + } +} diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java index 3336d8ddc..34a070f5f 100644 --- a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -49,7 +49,6 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -103,17 +102,23 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { * With the Location product included, a response that holds no * part for it leaves the element data without values and throws * nothing, as the Processing section of the cloud aspect engine - * specification says. + * specification says. Reading a property then throws, as for any + * property the response leaves out. */ @Test - public void ProductPresent_ResponseWithoutPart_NoValues() + public void ProductPresent_ResponseWithoutPart_PropertyIsMissing() throws Exception { try (GeoLocationCloudEngine engine = buildEngine("location", "town")) { CloudGeoData data = process( engine, "{ 'ip': { 'registeredname': 'Example' }, " + "'javascriptProperties': [] }"); - assertFalse(data.getTown().hasValue()); + try { + data.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response held nothing for the Location product. + } } } diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java b/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java index 060e6625c..ce3edd363 100644 --- a/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java +++ b/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java @@ -67,9 +67,26 @@ public static void valueTypes(Wrapper wrapper) throws Exception { assertNotNull("Value of " + property.getName() + " is null. ", value); assertTrue("Value of '" + property.getName() + "' was of type " + value.getClass().getSimpleName() + + " but should have been AspectPropertyValue.", + value instanceof AspectPropertyValue); + AspectPropertyValue propertyValue = + (AspectPropertyValue) value; + // A property can have no value at these coordinates, such + // as Suburb. Its type cannot be read then, so only that it + // carries a message is checked. + if (propertyValue.hasValue() == false) { + assertNotNull("Property '" + property.getName() + + "' has no value and no message.", + propertyValue.getNoValueMessage()); + continue; + } + assertTrue("Value of '" + property.getName() + + "' was of type " + + propertyValue.getValue().getClass().getSimpleName() + " but should have been " + expectedType.getSimpleName() + ".", - expectedType.isAssignableFrom(value.getClass())); + expectedType.isAssignableFrom( + propertyValue.getValue().getClass())); } } } From c77fb051acb84caf730f3b0f0c854d5349b27ff2 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:02:57 +0100 Subject: [PATCH 69/95] Merged Pull Request '#103 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cxx b/cxx index e866f6d3c..c6d4e4d90 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit e866f6d3cea936587790f87967eff7863778b30e +Subproject commit c6d4e4d907e8da83845c095e80374280f8db29c5 From 1c2c5789fa3281122d2fe86b02ebc28c8446b427 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 10:34:29 +0000 Subject: [PATCH 70/95] Register the data file again after a refresh from a watched file, so polling carries on (#98) * Test that polling carries on after a refresh from a watched file A new data file found by the file watcher refreshes the engine, and then polling for updates must carry on, as the newFileAvailable steps of the data update specification say. The test runs the watcher's task itself and records each polling timer the service schedules. After the refresh there must be a second timer, held by the data file. * Register the data file again after a refresh from a watched file updatedFileAvailable cancels the polling timer before it refreshes the engine. checkForUpdate registers the data file again afterwards, which sets a new timer, but dataFileUpdated, which the file watcher calls, did not, so HTTP polling stopped after the first refresh from a watched file. The newFileAvailable steps of the data update specification say to resume polling after loading the file and before the callbacks. dataFileUpdated now registers a registered data file again after the refresh and before the callbacks. Registering a file that is already registered only sets the timer, because the watcher and the start-up check are each set up once. The comment on updatedFileAvailable said it registered the file itself, which it does not, and now says the caller does. * Leave a data file Windows still holds to be deleted at exit On Windows the full build failed in the watched file test's clean up, with "The process cannot access the file because it is being used by another process". The data update service reads the file through BinaryReader, which maps it, and Windows keeps a mapped file until the buffer is collected. The test's assertions had passed. The clean up now deletes what it can and leaves the rest to be deleted when the test run ends, as the existing watcher test does by ignoring a failed delete. * Test that a watched file that cannot be opened is given up on The data update specification's Errors table says a watched file that cannot be opened for reading within 10 minutes ends the update with AUTO_UPDATE_TIMEOUT and logs nothing, as .NET does. Java tries for ever with no pause, so the new test, which shortens the wait to one second, never sees the update end. This commit adds the AUTO_UPDATE_TIMEOUT status, the ten minute wait and the half second pause as constants, and the field the test shortens, so the test compiles and fails on behaviour. The loop does not use them yet. The setup the two watched file tests share moves into helpers. * Give up on a watched file that cannot be opened after ten minutes The file watcher tried to open a new data file for ever, with no pause and a warning on every try, while holding the data file's update lock. It now tries every half second for up to ten minutes, logs nothing, and then ends the update with AUTO_UPDATE_TIMEOUT, leaving the engine and the polling timer as they were. This is what .NET does and what the Errors table of the data update specification says. The file is registered again only when the refresh ran, because only the refresh cancels the polling timer. * Test that an interrupted wait for a watched file leaves the update unfinished An interrupt, as unregistering the data file gives the watcher's task, ends the wait for a file that cannot be opened, and the update reports AUTO_UPDATE_TIMEOUT although the file was not tried for the whole wait. The new test requires AUTO_UPDATE_IN_PROGRESS, the status for an update that did not finish, so it fails on the timeout. The two tests of a file that cannot be opened now share one fixture. The stand-in file pauses without sleeping, so an interrupt reaches the service rather than the stand-in. * Leave an interrupted wait for a watched file unfinished, not timed out An interrupt ended the wait and reported AUTO_UPDATE_TIMEOUT, although the file had not been tried for the whole wait. The update now keeps the status it started with, AUTO_UPDATE_IN_PROGRESS, which is what it reports for any watcher event that does not finish an update. The engine and the polling timer are left as they were, as for a timeout. The IOException the loop catches is no longer named e, as nothing reads it. --- .../engines/services/DataUpdateService.java | 4 +- .../services/DataUpdateServiceDefault.java | 61 ++- .../DataUpdateServiceWatchedFileTests.java | 406 ++++++++++++++++++ 3 files changed, 461 insertions(+), 10 deletions(-) create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java index bb262ae78..2ee0231c0 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java @@ -127,7 +127,9 @@ enum AutoUpdateStatus { // The new data file can't be renamed to replace the previous one. AUTO_UPDATE_NEW_FILE_CANT_RENAME, // Refreshing the engine with the new data caused an error to occur. - AUTO_UPDATE_REFRESH_FAILED + AUTO_UPDATE_REFRESH_FAILED, + // An operation could not be completed within the expected time. + AUTO_UPDATE_TIMEOUT } class DataUpdateCompleteArgs { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 6b8da4adf..a2c4af853 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -72,6 +72,24 @@ public class DataUpdateServiceDefault implements DataUpdateService { */ private static final String LICENSE_KEYS_PARAMETER = "LicenseKeys"; + /** + * How long a watched data file is tried before the update gives up with + * {@link AutoUpdateStatus#AUTO_UPDATE_TIMEOUT}, as a copy over it may + * still be in progress. + */ + static final long WATCHED_FILE_OPEN_TIMEOUT_MILLIS = + TimeUnit.MINUTES.toMillis(10); + + /** + * The pause between tries to open a watched data file. + */ + static final long WATCHED_FILE_OPEN_PAUSE_MILLIS = 500; + + /** + * The wait for a watched data file, which tests shorten. + */ + long watchedFileOpenTimeoutMillis = WATCHED_FILE_OPEN_TIMEOUT_MILLIS; + final List configurations = new ArrayList<>(); private final Logger logger; private final boolean closeFutureFactory; @@ -290,9 +308,10 @@ public void run() { /** * Called when a data update is available and the file at * engine.DataFilePath contains this new data. - * 1. Refresh the engine. - * 2. Dispose of the existing update timer if there is one. - * 3. Re-register the engine with the update service. + * 1. Dispose of the existing update timer if there is one. + * 2. Refresh the engine. + * Where the data file is registered, the caller then registers it again, + * which sets a new update timer. * * @param dataFile the data file to update * @return {@link AutoUpdateStatus#AUTO_UPDATE_SUCCESS} if the data file was @@ -372,17 +391,41 @@ private void dataFileUpdated(Object sender) { // before notifying the engine, otherwise the copy // may still be in progress. boolean fileLockable = false; - while (fileLockable == false) { + boolean interrupted = false; + long deadline = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos( + watchedFileOpenTimeoutMillis); + while (fileLockable == false && + System.nanoTime() - deadline < 0) { try (BinaryReader reader = fileWrapperFactory.build( aspectDataFile.getDataFilePath()).getReader()) { fileLockable = true; - // Complete the update - status = updatedFileAvailable(dataFile); - } catch (IOException e) { - logger.warn("Exception while reading the data file at '{}'", - aspectDataFile.getDataFilePath(), e); + } catch (IOException ignored) { + try { + Thread.sleep(WATCHED_FILE_OPEN_PAUSE_MILLIS); + } catch (InterruptedException ex) { + // Interrupted, as unregistering the data + // file does, so stop trying. + Thread.currentThread().interrupt(); + interrupted = true; + break; + } + } + } + if (fileLockable) { + // Complete the update + status = updatedFileAvailable(dataFile); + // The refresh cancelled the polling timer, so + // register the file again to set a new one. + if (dataFile.getIsRegistered()) { + registerDataFile(dataFile); } + } else if (interrupted == false) { + // The polling timer is left as it was. + status = AutoUpdateStatus.AUTO_UPDATE_TIMEOUT; } + // An interrupted wait leaves the update unfinished, + // so the status stays AUTO_UPDATE_IN_PROGRESS. } } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java new file mode 100644 index 000000000..b57a16ca9 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java @@ -0,0 +1,406 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.data.BinaryReader; +import fiftyone.common.wrappers.io.FileWrapper; +import fiftyone.common.wrappers.io.FileWrapperDefault; +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Test; +import org.mockito.Answers; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +/** + * A new data file found by the file watcher is loaded, and then polling for + * updates carries on, as the newFileAvailable steps of the data update + * specification say ("resume polling, if any"). A new file that cannot be + * opened is given up on after a time, as its Errors table says, or at + * once when the wait is interrupted. + */ +public class DataUpdateServiceWatchedFileTests { + + private static final long WAIT_FOR_EVENT_MILLIS = 10000; + + private static final long RETRY_MILLIS = 100; + + /** + * The wait for a file that cannot be opened, shortened from ten minutes. + */ + private static final long OPEN_TIMEOUT_MILLIS = 1000; + + /** + * The most tries that wait allows, being one at the start, one after + * each pause and one more for a slow runner. + */ + private static final long MAX_TRIES = OPEN_TIMEOUT_MILLIS / + DataUpdateServiceDefault.WATCHED_FILE_OPEN_PAUSE_MILLIS + 2; + + /** + * A wait long enough that only an interrupt can end it within + * {@link #INTERRUPTED_WAIT_MILLIS}. + */ + private static final long LONG_OPEN_TIMEOUT_MILLIS = 60000; + + private static final long INTERRUPTED_WAIT_MILLIS = 5000; + + private static final String MAC_MESSAGE = + "File watchers are not well implemented in OS X, " + + "so don't run this unit test as it is unlikely to pass."; + + @Test + public void WatchedFileRefresh_PollingCarriesOn() throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + + final List timers = new ArrayList<>(); + final AtomicReference watcher = new AtomicReference<>(); + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new FileWrapperDefault((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory(timers, watcher)); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = watchedFile(engine, dataFile); + + try { + dataUpdate.registerDataFile(file); + assertEquals("Registering sets one polling timer", + 1, timers.size()); + assertNotNull("Registering sets up the file watcher", + watcher.get()); + + Files.write(dataFile, "New".getBytes()); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (refreshCount(engine) == 0 && + System.currentTimeMillis() < end) { + watcher.get().run(); + Thread.sleep(RETRY_MILLIS); + } + + assertEquals("The watched file refreshes the engine once", + 1, refreshCount(engine)); + assertEquals("A new polling timer is set after the refresh", + 2, timers.size()); + assertNotNull("The data file holds the new polling timer", + file.getFuture()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A watched file that cannot be opened is tried, with a pause between + * tries, until the wait ends. The update then reports + * AUTO_UPDATE_TIMEOUT, logs nothing, and leaves the engine and the + * polling timer as they were. + */ + @Test + public void WatchedFileCannotBeOpened_GivesUpAfterTheWait() + throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + try (LockedFile locked = new LockedFile(OPEN_TIMEOUT_MILLIS)) { + locked.start(); + boolean gaveUp = locked.done.await( + WAIT_FOR_EVENT_MILLIS, TimeUnit.MILLISECONDS); + + assertTrue("The watcher tries to open the new file", + locked.tries.get() > 0); + assertTrue("The update gives up on a file it cannot open", + gaveUp); + assertEquals(AutoUpdateStatus.AUTO_UPDATE_TIMEOUT, + locked.status.get()); + assertTrue("The tries are paused, but " + locked.tries.get() + + " were made", locked.tries.get() <= MAX_TRIES); + locked.assertLeftAsItWas(); + } + } + + /** + * An interrupt, as unregistering the data file gives the watcher's task, + * ends the wait at once. The update is left unfinished, so it reports + * AUTO_UPDATE_IN_PROGRESS rather than a timeout, logs nothing, and leaves + * the engine and the polling timer as they were. + */ + @Test + public void WatchedFileWaitInterrupted_LeavesTheUpdateUnfinished() + throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + try (LockedFile locked = new LockedFile(LONG_OPEN_TIMEOUT_MILLIS)) { + locked.start(); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (locked.tries.get() == 0 && + System.currentTimeMillis() < end) { + Thread.sleep(RETRY_MILLIS); + } + assertTrue("The watcher tries to open the new file", + locked.tries.get() > 0); + + locked.runner.interrupt(); + boolean ended = locked.done.await( + INTERRUPTED_WAIT_MILLIS, TimeUnit.MILLISECONDS); + + assertTrue("The wait ends when it is interrupted", ended); + assertEquals(AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS, + locked.status.get()); + locked.assertLeftAsItWas(); + } + } + + /** + * A watched data file that throws on every open, the service watching + * it, and the thread that runs the watcher's task. + */ + private static class LockedFile implements AutoCloseable { + + final List timers = new ArrayList<>(); + final AtomicReference watcher = new AtomicReference<>(); + final AtomicBoolean locked = new AtomicBoolean(true); + final AtomicInteger tries = new AtomicInteger(); + final AtomicInteger logged = new AtomicInteger(); + final AtomicReference status = + new AtomicReference<>(); + final CountDownLatch done = new CountDownLatch(1); + final AtomicBoolean stop = new AtomicBoolean(false); + final OnPremiseAspectEngine engine = engine(); + final DataUpdateServiceDefault service; + final Path directory; + final Path dataFile; + final AspectEngineDataFileDefault file; + final Thread runner; + int loggedAtStart; + + LockedFile(long timeoutMillis) throws Exception { + // The service may call these many times while it waits, so they + // are stub only mocks, which keep no record of their calls. + final BinaryReader reader = + mock(BinaryReader.class, withSettings().stubOnly()); + FileWrapper wrapper = + mock(FileWrapper.class, withSettings().stubOnly()); + when(wrapper.getReader()).then(invocation -> { + tries.incrementAndGet(); + if (locked.get()) { + // Stops a loop with no pause using a whole processor, + // and leaves an interrupt for the service to see. + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + throw new IOException("The file is locked"); + } + return reader; + }); + FileWrapperFactory fileWrapperFactory = + mock(FileWrapperFactory.class, withSettings().stubOnly()); + when(fileWrapperFactory.build(anyString())).thenReturn(wrapper); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + Logger logger = mock(Logger.class, withSettings().stubOnly() + .defaultAnswer(invocation -> { + String name = invocation.getMethod().getName(); + if (name.equals("warn") || name.equals("error")) { + logged.incrementAndGet(); + } + return Answers.RETURNS_DEFAULTS.answer(invocation); + })); + service = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory(timers, watcher)); + service.watchedFileOpenTimeoutMillis = timeoutMillis; + // The first update after the change is the one that waits, + // because only the watcher sets the time it compares with. + service.onUpdateComplete((sender, args) -> { + if (status.compareAndSet(null, args.getStatus())) { + done.countDown(); + } + }); + + directory = Files.createTempDirectory("watched"); + dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + file = watchedFile(engine, dataFile); + + // The watcher runs on its own thread, so the test can stop + // waiting for an update that does not end. + runner = new Thread(() -> { + while (stop.get() == false && done.getCount() > 0) { + watcher.get().run(); + try { + Thread.sleep(RETRY_MILLIS); + } catch (InterruptedException e) { + return; + } + } + }); + runner.setDaemon(true); + } + + /** + * Registers the file, changes it, and starts the watcher's thread. + */ + void start() throws Exception { + service.registerDataFile(file); + assertNotNull("Registering sets up the file watcher", + watcher.get()); + loggedAtStart = logged.get(); + Files.write(dataFile, "New".getBytes()); + runner.start(); + } + + void assertLeftAsItWas() { + assertEquals("The engine is not refreshed", + 0, refreshCount(engine)); + assertEquals("Nothing is logged", loggedAtStart, logged.get()); + assertEquals("The polling timer is kept", 1, timers.size()); + assertNotNull("The data file holds the polling timer", + file.getFuture()); + } + + @Override + public void close() throws Exception { + // Lets an update that would not give up end, so the thread stops. + stop.set(true); + locked.set(false); + if (runner.isAlive()) { + runner.join(WAIT_FOR_EVENT_MILLIS); + } + cleanUp(file, directory, dataFile); + } + } + + /** + * Records the polling timers scheduled, and keeps the watcher's task for + * the test to run itself. + */ + private static FutureFactory futureFactory( + final List timers, + final AtomicReference watcher) { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + timers.add((Long) invocation.getArgument(1)); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> { + watcher.set((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + return futureFactory; + } + + @SuppressWarnings("unchecked") + private static OnPremiseAspectEngine engine() { + return mock(OnPremiseAspectEngine.class); + } + + private static AspectEngineDataFileDefault watchedFile( + OnPremiseAspectEngine engine, + Path dataFile) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(true); + config.setPollingIntervalSeconds(1); + config.setDataFilePath(dataFile.toString()); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + return file; + } + + private static void cleanUp( + AspectEngineDataFileDefault file, + Path directory, + Path dataFile) { + if (file.getConfiguration().getWatchKey() != null) { + file.getConfiguration().getWatchKey().cancel(); + } + // Windows keeps the file until the reader's mapped buffer is + // collected, so whatever cannot be deleted now is deleted when the + // test run ends. Those are deleted in the reverse order they are + // registered, so the folder is registered first. + directory.toFile().deleteOnExit(); + if (dataFile.toFile().delete() == false) { + dataFile.toFile().deleteOnExit(); + } + directory.toFile().delete(); + } + + private static boolean isMac() { + return System.getProperty("os.name").contains("Mac OS X"); + } + + private static long lastModified(Object path) { + return new File((String) path).lastModified(); + } + + private static long refreshCount(Object engine) { + return mockingDetails(engine).getInvocations().stream() + .filter(i -> i.getMethod().getName().equals("refreshData")) + .count(); + } +} From f2e3981fbd4c847f234382495278be5c0223b4f3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 12:17:17 +0000 Subject: [PATCH 71/95] Add switches and paths to the web integration, and Element Data to the Sequence Element (#80) * Test the web integration switches, the Sequence Element data and usage sharing characters The tests cover the init parameters that set the JavaScript and JSON paths and switch the Sequence Element and the Set Headers Element, the Element Data the Sequence Element writes, how the JSON Builder Element shows that data, and the characters XML 1.0 allows in a shared Evidence value. * Add switches and paths to the web integration, and Element Data to the Sequence Element The filter takes init parameters for the JavaScript path, the JSON path, and whether the Sequence Element and the Set Headers Element are added. The defaults keep today's paths and today's behaviour. The Sequence Element writes session-id and sequence as Element Data as well as Evidence. Usage sharing keeps every character XML 1.0 allows instead of replacing everything above U+00FF. * Test the MVC integration's paths and element switches * Give the MVC integration the filter's paths and element switches, and say what the Lite file lacks The MVC configuration takes the JavaScript path, the JSON path and the two element switches the filter takes, with defaults that keep today's behaviour. The example comments about the Lite data file say it has a small number of Properties, as the specification does. * Test the Sequence Element data under the key sequence * Key the Sequence Element data as sequence, as the other languages do * Do not call an overridable method from the constructor Java 21 warns about it, and the build treats warnings as errors. * Test that the JSON is served from /51dpipeline/json and from the old path * Serve the JSON from /51dpipeline/json by default, and from the old path as well The specification's JSON path becomes the default in the filter and the MVC integration, and the JavaScript they serve calls it. The path 51Degrees.core.json is still answered, and json-path still sets the path. * Test the Sequence Element with session ids and sequences of every kind * Treat the session id and the sequence independently, as the specification says A session id that is not a non-empty string, and a sequence that is not a whole number or text of decimal digits that fits in an int with room to add one, count as absent. The Evidence and the Element Data are written from the same values. * Test that the usage sharing filter never includes the web request * Keep the web request out of the usage sharing filter, as .NET does * Expect a new session id when the Sequence Element is given an empty one * Test that a build with the JSON Builder Element and no Sequence Element is refused The json-path test uses a path that is not the default. * Refuse a build with the JSON Builder Element and no Sequence Element The JSON Builder Element fails on every request without the sequence number, so the build says which option to change. * Use a configuration without the JSON Builder Element when client-side properties are off * Test the exact path match and the rule that a sequence is not negative - FiftyOneJSServiceCoreTests checks that a configured path, with or without its leading slash, answers only that path inside the application, so /anything/json, /json/more and /notjson are not answered, and that the context path is not part of the path. - SequenceElementTests checks that a negative sequence of any whole number type, a sign and white space each count as no sequence, and that 0 gives 1. * Match the JSON and JavaScript paths exactly, and count a negative sequence as absent FiftyOneJSServiceCore.Default answered any request whose whole URL ended with the configured path, so a configured path of json also answered /anything/json. The path is now compared with the whole path of the request inside the application, after the context path, ignoring case and with or without a leading slash, and a request with no URI is not answered. The old 51Degrees.core.json path is matched the same way. The Sequence Element took a negative whole number as a sequence. The Sequence Element specification counts a sequence only from 0 up to 2,147,483,646, so a negative one now counts as absent. * Give a caller with no JSON path the default path --- .../console/GettingStartedOnPrem.java | 4 +- .../examples/console/MetadataOnPrem.java | 4 +- .../examples/console/OfflineProcessing.java | 2 +- .../examples/console/UpdateDataFile.java | 2 +- .../main/webapp/WEB-INF/51Degrees-Cloud.xml | 2 +- .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 2 +- .../main/webapp/WEB-INF/51Degrees-Mixed.xml | 2 +- .../data/EvidenceKeyFilterShareUsage.java | 5 + .../engines/fiftyone/data/SequenceData.java | 86 ++++ .../flowelements/SequenceElement.java | 108 +++-- .../EvidenceKeyFilterShareUsageTests.java | 12 + .../flowelements/SequenceElementTests.java | 256 +++++++++++ .../SessionIdAndSequenceTests.java | 13 +- .../jsonbuilder/JsonBuilderTests.java | 33 ++ .../mvc/components/FiftyOneInterceptor.java | 42 +- .../FiftyOneInterceptorConfig.java | 34 ++ .../FiftyOneInterceptorConfigDefault.java | 44 ++ .../web/mvc/services/FiftyOneJSService.java | 10 + .../web/mvc/FiftyOneInterceptorTests.java | 424 ++++++++++++------ .../src/test/resources/NoJson.xml | 7 + .../src/test/resources/NoSequence.xml | 10 + .../java/fiftyone/pipeline/web/Constants.java | 41 +- .../fiftyone/pipeline/web/PipelineFilter.java | 44 +- .../fiftyone/pipeline/web/StartupHelpers.java | 71 ++- .../web/services/FiftyOneJSServiceCore.java | 74 ++- .../web/FiftyOneJSServiceCoreTests.java | 188 ++++++++ .../pipeline/web/PipelineFilterTests.java | 199 ++++++++ 27 files changed, 1523 insertions(+), 196 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java create mode 100644 pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml create mode 100644 pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 57c898e34..6cb6e8433 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -64,8 +64,8 @@ public class GettingStartedOnPrem { cxx/data, and is downloaded if it is missing, or you may specify another file as a command line parameter. - Note that the Lite data file is only used for illustration, and has limited accuracy and - capabilities. The example shows the properties only the Enterprise data file has as needing + Note that the Lite data file is only used for illustration, and has a small number of + Properties. The example shows the properties only the Enterprise data file has as needing it. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static String ENTERPRISE_HASH = "TAC-HashV41.hash"; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 8d8a06bcf..c85d852b8 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -51,8 +51,8 @@ public class MetadataOnPrem { cxx/data, and is downloaded if it is missing, or you may specify another file as a command line parameter. - Note that the Lite data file is only used for illustration, and has limited accuracy and - capabilities. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ + Note that the Lite data file is only used for illustration, and has a small number of + Properties. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; /** diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index cba0c7e89..fbcdee51b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -72,7 +72,7 @@ public class OfflineProcessing { // cxx/data, and is downloaded if it is missing // // Note that the Lite data file is only used for illustration, and has - // limited accuracy and capabilities. Contact us for a copy of the + // a small number of Properties. Contact us for a copy of the // Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash public static final String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index 9748b15f5..ebffdef86 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -75,7 +75,7 @@ * https://distributor.51degrees.com/api/v2/download?LicenseKeys=&Type=27&Download=True&Product=22 * ``` * ## Lite Data File - * Lite data files (free-to-use, limited capabilities, no license key required) are created roughly + * Lite data files (free-to-use, a small number of Properties, no license key required) are created roughly * once a month and cannot be updated using auto-update, they may be downloaded with the * scripts in the [data repository](https://github.com/51Degrees/data) on GitHub, which is * the cxx/data folder of the source. diff --git a/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml b/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml index dacfeb006..ced491606 100644 --- a/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml +++ b/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml @@ -42,7 +42,7 @@ JsonBuilderElement diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index bba6dd60e..b74ec24f8 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -39,7 +39,7 @@ JsonBuilderElement diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml index 127b4665d..83f577d3e 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml @@ -42,7 +42,7 @@ IPIntelligenceOnPremiseEngine - + JsonBuilderElement diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 07a21856e..db1091fbe 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -136,6 +136,11 @@ public EvidenceKeyFilterShareUsage( @Override public boolean include(String key) { + // The Usage Sharing Element reads the whole web request itself, and + // the request is never a value to share. + if (ShareUsageEvidence.HTTP_REQUEST_KEY.equalsIgnoreCase(key)) { + return false; + } String lowerKey = key.toLowerCase(); for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) { if (lowerKey.equals(neverShared) || diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java new file mode 100644 index 000000000..e6f18024d --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import org.slf4j.Logger; + +/** + * Data containing the result of {@link SequenceElement}. + * @see Specification + */ +public class SequenceData extends ElementDataBase { + + /** + * Name of the property holding the session id. + */ + public static final String SESSION_ID_PROPERTY_NAME = "session-id"; + + /** + * Name of the property holding the sequence. + */ + public static final String SEQUENCE_PROPERTY_NAME = "sequence"; + + /** + * Construct a new instance. + * @param logger the logger to use + * @param flowData the flow data the data belongs to + */ + public SequenceData(Logger logger, FlowData flowData) { + super(logger, flowData); + } + + /** + * Get the identifier of the session. + * @return the session id + */ + public String getSessionId() { + return getAs(SESSION_ID_PROPERTY_NAME, String.class); + } + + /** + * Set the identifier of the session. + * @param sessionId the session id + */ + public void setSessionId(String sessionId) { + put(SESSION_ID_PROPERTY_NAME, sessionId); + } + + /** + * Get the number of consecutive callbacks made in the session. + * @return the sequence + */ + public Integer getSequence() { + return getAs(SEQUENCE_PROPERTY_NAME, Integer.class); + } + + /** + * Set the number of consecutive callbacks made in the session. + * @param sequence the sequence + */ + public void setSequence(int sequence) { + put(SEQUENCE_PROPERTY_NAME, sequence); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java index 5542e99f9..79a70b91e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java @@ -23,13 +23,17 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; import fiftyone.pipeline.core.data.*; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.regex.Pattern; import org.slf4j.Logger; /** @@ -39,6 +43,10 @@ public class SequenceElement extends FlowElementBase { + private static final Pattern DIGITS = Pattern.compile("[0-9]+"); + + private final List properties; + /** * The session id and sequence of a callback, which the specification * lists as this element's accepted Evidence. Without them the web @@ -55,48 +63,92 @@ public class SequenceElement * Construct a new instance of the {@link FlowElement}. * @param logger logger instance to use for logging */ - public SequenceElement(Logger logger) { - super(logger, null); + @SuppressWarnings("this-escape") + public SequenceElement(final Logger logger) { + super(logger, new ElementDataFactory() { + @Override + public ElementData create( + FlowData flowData, + FlowElement flowElement) { + return new SequenceData(logger, flowData); + } + }); + properties = new ArrayList<>(); + properties.add(new ElementPropertyMetaDataDefault( + SequenceData.SESSION_ID_PROPERTY_NAME, + this, + "", + String.class, + true)); + properties.add(new ElementPropertyMetaDataDefault( + SequenceData.SEQUENCE_PROPERTY_NAME, + this, + "", + Integer.class, + true)); } @Override protected void processInternal(FlowData data) throws Exception { Map evidence = data.getEvidence().asKeyMap(); - - // If the evidence does not contain a session id then create a new one. - if (evidence.containsKey(Constants.EVIDENCE_SESSIONID) == false) { - data.addEvidence(Constants.EVIDENCE_SESSIONID, getNewSessionId()); - } - // If the evidence does not have a sequence then add one. - if (evidence.containsKey(Constants.EVIDENCE_SEQUENCE) == false) { - data.addEvidence(Constants.EVIDENCE_SEQUENCE, 1); - } else { - data.addEvidence( - Constants.EVIDENCE_SEQUENCE, - getSequence(evidence.get(Constants.EVIDENCE_SEQUENCE)) + 1); - } + // The session id and the sequence are each used only when they are + // valid, and otherwise count as absent. + Object givenSession = evidence.get(Constants.EVIDENCE_SESSIONID); + String sessionId = + givenSession instanceof String && + ((String) givenSession).isEmpty() == false ? + (String) givenSession : getNewSessionId(); + Integer previous = toSequence(evidence.get(Constants.EVIDENCE_SEQUENCE)); + int sequence = previous == null ? 1 : previous + 1; + + data.addEvidence(Constants.EVIDENCE_SESSIONID, sessionId); + data.addEvidence(Constants.EVIDENCE_SEQUENCE, sequence); + + SequenceData sequenceData = (SequenceData) data.getOrAdd( + getElementDataKey(), + getDataFactory()); + sequenceData.setSessionId(sessionId); + sequenceData.setSequence(sequence); } /** - * The sequence in Evidence, which is a number when an element set it and - * text when it came from the query string of a callback. Anything that is - * not a number counts as no sequence. + * The sequence in Evidence, which is a whole number from 0 up, or text + * made only of decimal digits. A value that is anything else, is + * negative or has no larger int after it counts as no sequence, so that + * adding one never overflows. + * @return the sequence, or null if there is none */ - private static int getSequence(Object sequence) { - if (sequence instanceof Number) { - return ((Number) sequence).intValue(); - } - try { - return Integer.parseInt(String.valueOf(sequence).trim()); - } catch (NumberFormatException e) { - return 0; + private static Integer toSequence(Object sequence) { + long value; + if (sequence instanceof Integer || sequence instanceof Long || + sequence instanceof Short || sequence instanceof Byte) { + value = ((Number) sequence).longValue(); + } else if (sequence instanceof BigInteger) { + BigInteger big = (BigInteger) sequence; + if (big.bitLength() > 31) { + return null; + } + value = big.longValue(); + } else if (sequence instanceof String && + DIGITS.matcher((String) sequence).matches()) { + // More digits than any int has cannot fit, however many zeros + // come first, so only the value of a short string is read. + String digits = ((String) sequence).replaceFirst("^0+(?=.)", ""); + if (digits.length() > 10) { + return null; + } + value = Long.parseLong(digits); + } else { + return null; } + return value < Integer.MAX_VALUE && value >= 0 ? + Integer.valueOf((int) value) : null; } @Override public String getElementDataKey() { - return "sequence-element"; + return "sequence"; } @Override @@ -106,7 +158,7 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { @Override public List getProperties() { - return new ArrayList<>(); + return properties; } @Override diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java index c0c97c288..33eacefc6 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -61,6 +61,18 @@ private static EvidenceKeyFilterShareUsage defaults() { * values and query string parameters only when their names start with * 51D_, and any other evidence. */ + /** + * The whole web request is read by the Usage Sharing Element and never + * shared as a value, so the filter does not include it. + */ + @Test + public void webRequestIsNeverIncluded() { + EvidenceKeyFilterShareUsage filter = defaults(); + + assertFalse(filter.include("server.http-request")); + assertFalse(filter.include("Server.Http-Request")); + } + @Test public void defaultRules() { EvidenceKeyFilterShareUsage filter = defaults(); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java new file mode 100644 index 000000000..9243c966a --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java @@ -0,0 +1,256 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The session id and sequence are written to Element Data as well as + * Evidence. + * @see Specification + */ +public class SequenceElementTests { + + private static final String SESSION_ID_EVIDENCE = "query.session-id"; + private static final String SEQUENCE_EVIDENCE = "query.sequence"; + + private Pipeline newPipeline() throws Exception { + ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); + return new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .build(); + } + + private ElementData sequenceData(FlowData flowData) throws Exception { + ElementData data = flowData.get("sequence"); + assertNotNull(data, "Expected Element Data under 'sequence'"); + return data; + } + + @Test + public void NoEvidence_NewSessionAndSequenceOne() throws Exception { + try (Pipeline pipeline = newPipeline(); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + + ElementData data = sequenceData(flowData); + assertNotNull(data.get("session-id")); + assertEquals(1, data.get("sequence")); + assertEquals( + data.get("session-id"), + flowData.getEvidence().asKeyMap().get(SESSION_ID_EVIDENCE)); + assertEquals( + 1, + flowData.getEvidence().asKeyMap().get(SEQUENCE_EVIDENCE)); + } + } + + @Test + public void EvidenceGiven_SessionKeptAndSequenceIncreased() + throws Exception { + try (Pipeline pipeline = newPipeline(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(SESSION_ID_EVIDENCE, "abc"); + flowData.addEvidence(SEQUENCE_EVIDENCE, 2); + flowData.process(); + + ElementData data = sequenceData(flowData); + assertEquals("abc", data.get("session-id")); + assertEquals(3, data.get("sequence")); + assertEquals( + 3, + flowData.getEvidence().asKeyMap().get(SEQUENCE_EVIDENCE)); + } + } + + @Test + public void Properties_AreSessionIdAndSequence() throws Exception { + try (Pipeline pipeline = newPipeline()) { + SequenceElement element = + (SequenceElement) pipeline.getFlowElements().get(0); + List names = new ArrayList<>(); + for (ElementPropertyMetaData property : element.getProperties()) { + names.add(property.getName()); + } + assertTrue(names.contains("session-id")); + assertTrue(names.contains("sequence")); + assertEquals(2, names.size()); + } + } + + /** + * Process with the given Evidence values, where null means none, and + * check that the Evidence and the Element Data hold the same values. + * @return the session id and the sequence + */ + private Object[] run(Object sessionId, Object sequence) throws Exception { + try (Pipeline pipeline = newPipeline(); + FlowData flowData = pipeline.createFlowData()) { + if (sessionId != null) { + flowData.addEvidence(SESSION_ID_EVIDENCE, sessionId); + } + if (sequence != null) { + flowData.addEvidence(SEQUENCE_EVIDENCE, sequence); + } + flowData.process(); + + ElementData data = sequenceData(flowData); + Object evidenceSession = + flowData.getEvidence().asKeyMap().get(SESSION_ID_EVIDENCE); + Object evidenceSequence = + flowData.getEvidence().asKeyMap().get(SEQUENCE_EVIDENCE); + assertEquals(evidenceSession, data.get("session-id")); + assertEquals(evidenceSequence, data.get("sequence")); + return new Object[] {evidenceSession, evidenceSequence}; + } + } + + private void assertNewSession(Object sessionId, Object sequence, + int expectedSequence) throws Exception { + Object[] result = run(sessionId, sequence); + assertTrue(result[0] instanceof String); + assertFalse(((String) result[0]).isEmpty()); + if (sessionId != null) { + assertNotEquals(sessionId, result[0]); + } + assertEquals(expectedSequence, result[1]); + } + + private void assertSequence(Object sequence, int expected) + throws Exception { + Object[] result = run("abc", sequence); + assertEquals("abc", result[0]); + assertEquals(expected, result[1]); + } + + @Test + public void SequenceAsText_Digits_CountsUp() throws Exception { + assertSequence("2", 3); + assertSequence("0", 1); + assertSequence("00012", 13); + } + + @Test + public void SequenceAsOtherWholeNumbers_CountsUp() throws Exception { + assertSequence(2L, 3); + assertSequence((short) 2, 3); + assertSequence(new java.math.BigInteger("2"), 3); + } + + @Test + public void SequenceNotAWholeNumber_CountsAsAbsent() throws Exception { + assertSequence("abc", 1); + assertSequence("", 1); + assertSequence(" 2", 1); + assertSequence("2 ", 1); + assertSequence("-2", 1); + assertSequence("+2", 1); + assertSequence("2.5", 1); + assertSequence(2.5, 1); + assertSequence(new Object(), 1); + } + + /** + * A sequence runs from 0 up, and as text is the digits 0 to 9 alone, + * so a negative number, a sign or white space counts as absent. + */ + @Test + public void SequenceNegativeSignedOrSpaced_CountsAsAbsent() + throws Exception { + assertSequence(-1, 1); + assertSequence(-1L, 1); + assertSequence((short) -1, 1); + assertSequence((byte) -1, 1); + assertSequence(new java.math.BigInteger("-1"), 1); + assertSequence(Integer.MIN_VALUE, 1); + assertSequence("-1", 1); + assertSequence("+5", 1); + assertSequence(" 5", 1); + assertSequence("5 ", 1); + assertSequence("\t5", 1); + assertSequence("5\n", 1); + assertSequence("5 6", 1); + } + + @Test + public void SequenceZero_GivesOne() throws Exception { + assertSequence(0, 1); + assertSequence(0L, 1); + assertSequence("0", 1); + assertSequence("000", 1); + } + + @Test + public void SequenceOutOfRange_CountsAsAbsent() throws Exception { + assertSequence("99999999999999999999", 1); + assertSequence("2147483648", 1); + assertSequence(2147483648L, 1); + // One more would not fit in an int either. + assertSequence("2147483647", 1); + assertSequence(2147483647, 1); + assertSequence("2147483646", 2147483647); + } + + @Test + public void SessionIdNotANonEmptyString_CountsAsAbsent() throws Exception { + assertNewSession("", 2, 3); + assertNewSession(5, 2, 3); + assertNewSession(new Object(), null, 1); + } + + @Test + public void SessionIdZero_Kept() throws Exception { + Object[] result = run("0", 4); + assertEquals("0", result[0]); + assertEquals(5, result[1]); + } + + @Test + public void SequenceWithoutSessionId_CountsUpWithNewSession() + throws Exception { + assertNewSession(null, 4, 5); + } + + @Test + public void SessionIdWithoutSequence_GivesOne() throws Exception { + Object[] result = run("abc", null); + assertEquals("abc", result[0]); + assertEquals(1, result[1]); + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java index b7bbe2e9b..8c74887c4 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java @@ -154,15 +154,22 @@ public void SessionId_Invalid_RenderedEmpty(String id) throws Exception { } /** - * The Sequence Element keeps a session id that is already in the - * evidence, so an invalid one reaches the builder that way too. + * The Sequence Element keeps a session id that is a non-empty string in + * the evidence, so an invalid one reaches the builder that way too. */ @ParameterizedTest @MethodSource("invalidSessionIds") public void SessionId_InvalidWithSequenceElement_RenderedEmpty(String id) throws Exception { String script = render(true, id, 1); - assertEquals("", sessionId(script)); + if (id.isEmpty()) { + // The Sequence Element counts an empty session id as absent + // and creates a new one. + assertTrue(sessionId(script).matches("[A-Za-z0-9-]{1,64}"), + sessionId(script)); + } else { + assertEquals("", sessionId(script)); + } assertEquals("2", sequence(script)); } diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index 944e99609..fe9923ff7 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -482,6 +482,39 @@ public void JsonBuilder_LazyLoading() throws Exception { } } + /** + * The Sequence Element's Element Data is in the JSON like any other + * element's, because the JSON Builder Element does not exclude it. + * @throws Exception + */ + @Test + public void JsonBuilder_SequenceElementData_InJson() throws Exception { + EmptyEngine engine = new EmptyEngineBuilder(loggerFactory) + .setLazyLoading(new LazyLoadingConfiguration(1000)) + .setProcessCost(500) + .build(); + JsonBuilderElement jsonBuilder = new JsonBuilderElementBuilder(loggerFactory) + .build(); + SequenceElement sequenceElement = new SequenceElementBuilder(loggerFactory) + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(sequenceElement) + .addFlowElement(engine) + .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) + .build(); + FlowData flowData = pipeline.createFlowData()) { + + flowData.process(); + + JsonBuilderData jsonResult = flowData.get(JsonBuilderData.class); + JSONObject jsonData = new JSONObject(jsonResult.getJson()); + JSONObject sequence = jsonData.getJSONObject("sequence"); + assertEquals(1, sequence.getLong("sequence")); + assertFalse(sequence.getString("session-id").isEmpty()); + } + } + /** * A weighted value is written as an object with the lower-case keys * rawweighting and value, and no others. diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java index 309b7669b..61022376d 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java @@ -51,6 +51,7 @@ import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; import static fiftyone.pipeline.web.Constants.CORE_JSON_NAME; +import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; /** * The 51Degrees middleware component. @@ -93,6 +94,10 @@ public FiftyOneInterceptor( configFileName = Constants.DEFAULT_CONFIG_FILE; } + String jsonPath = config.getJsonPath() == null || + config.getJsonPath().isEmpty() ? + DEFAULT_JSON_PATH : config.getJsonPath(); + File configFile = new File(configFileName); PipelineBuilder builder = new PipelineBuilder() .addService(new DataUpdateServiceDefault( @@ -104,9 +109,13 @@ public FiftyOneInterceptor( // Bind the configuration to a pipeline options instance PipelineOptions options = (PipelineOptions) unmarshaller.unmarshal(configFile); pipeline = StartupHelpers.buildFromConfiguration( - builder, - options, - config.getClientsidePropertiesEnabled()); + builder, + options, + config.getClientsidePropertiesEnabled(), + null, + jsonPath, + config.getSequenceElementEnabled(), + config.getSetHeadersElementEnabled()); } catch (Exception e) { throw new RuntimeException(e); } @@ -114,13 +123,40 @@ public FiftyOneInterceptor( resultService.setPipeline(pipeline); clientsidePropertyService.setPipeline(pipeline); fiftyOneJsService.enable(config.getClientsidePropertiesEnabled()); + fiftyOneJsService.setPaths(config.getJavaScriptPath(), jsonPath); } public static void enableClientsideProperties(ViewControllerRegistry viewControllerRegistry) { viewControllerRegistry.addViewController("/" + CORE_JS_NAME); + viewControllerRegistry.addViewController(DEFAULT_JSON_PATH); + viewControllerRegistry.addViewController("/" + CORE_JSON_NAME); + } + + /** + * Map the JavaScript and JSON paths that are not the defaults, which + * the application sets in its {@link FiftyOneInterceptorConfig}. + * @param viewControllerRegistry the registry to add the paths to + * @param javaScriptPath the JavaScript path, or null or empty for the + * default + * @param jsonPath the JSON path, or null or empty for the default + */ + public static void enableClientsideProperties( + ViewControllerRegistry viewControllerRegistry, + String javaScriptPath, + String jsonPath) { + viewControllerRegistry.addViewController("/" + withoutSlash( + javaScriptPath == null || javaScriptPath.isEmpty() ? + CORE_JS_NAME : javaScriptPath)); + viewControllerRegistry.addViewController("/" + withoutSlash( + jsonPath == null || jsonPath.isEmpty() ? + DEFAULT_JSON_PATH : jsonPath)); viewControllerRegistry.addViewController("/" + CORE_JSON_NAME); } + private static String withoutSlash(String path) { + return path.startsWith("/") ? path.substring(1) : path; + } + @Override public boolean preHandle( HttpServletRequest request, diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java index c2d6ddfbc..5dcdf3cd2 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java @@ -40,4 +40,38 @@ public interface FiftyOneInterceptorConfig { * @return true if client-side properties are enabled */ boolean getClientsidePropertiesEnabled(); + + /** + * Get the end of the request path the JavaScript is served for. + * @return the path, or null or empty for the default + */ + default String getJavaScriptPath() { + return null; + } + + /** + * Get the end of the request path the JSON is served for. + * @return the path, or null or empty for the default + */ + default String getJsonPath() { + return null; + } + + /** + * Get whether the Sequence Element is added to the Pipeline when the + * configuration does not include it. + * @return true to add the Sequence Element + */ + default boolean getSequenceElementEnabled() { + return true; + } + + /** + * Get whether the Set Headers Element is added to the Pipeline when the + * configuration does not include it. + * @return true to add the Set Headers Element + */ + default boolean getSetHeadersElementEnabled() { + return true; + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java index 4b3172ff2..bd22601cf 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java @@ -34,6 +34,14 @@ public class FiftyOneInterceptorConfigDefault implements FiftyOneInterceptorConf private boolean clientsidePropertiesEnabled; + private String javaScriptPath; + + private String jsonPath; + + private boolean sequenceElementEnabled = true; + + private boolean setHeadersElementEnabled = true; + @Autowired public ServletContext servletContext; @@ -45,6 +53,42 @@ public void setClientsidePropertiesEnabled(boolean clientsidePropertiesEnabled) this.clientsidePropertiesEnabled = clientsidePropertiesEnabled; } + public void setJavaScriptPath(String javaScriptPath) { + this.javaScriptPath = javaScriptPath; + } + + public void setJsonPath(String jsonPath) { + this.jsonPath = jsonPath; + } + + public void setSequenceElementEnabled(boolean sequenceElementEnabled) { + this.sequenceElementEnabled = sequenceElementEnabled; + } + + public void setSetHeadersElementEnabled(boolean setHeadersElementEnabled) { + this.setHeadersElementEnabled = setHeadersElementEnabled; + } + + @Override + public String getJavaScriptPath() { + return javaScriptPath; + } + + @Override + public String getJsonPath() { + return jsonPath; + } + + @Override + public boolean getSequenceElementEnabled() { + return sequenceElementEnabled; + } + + @Override + public boolean getSetHeadersElementEnabled() { + return setHeadersElementEnabled; + } + @Override public String getDataFilePath() { return dataFilePath; diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java index 57c8ff3d8..974b1be61 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java @@ -34,6 +34,16 @@ public interface FiftyOneJSService extends FiftyOneJSServiceCore { void enable(boolean enabled); + /** + * Set the paths the JavaScript and the JSON are served from. + * @param javaScriptPath the end of the request path that the JavaScript + * is served for, or null or empty for the default + * @param jsonPath the end of the request path that the JSON is served + * for, or null or empty for the default + */ + default void setPaths(String javaScriptPath, String jsonPath) { + } + @Service class Default extends FiftyOneJSServiceCore.Default implements FiftyOneJSService { diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java index 077a88b46..3f000fa3b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java @@ -1,141 +1,283 @@ -/* ********************************************************************* - * This Original Work is copyright of 51 Degrees Mobile Experts Limited. - * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, - * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. - * - * This Original Work is licensed under the European Union Public Licence - * (EUPL) v.1.2 and is subject to its terms as set out below. - * - * If a copy of the EUPL was not distributed with this file, You can obtain - * one at https://opensource.org/licenses/EUPL-1.2. - * - * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be - * amended by the European Commission) shall be deemed incompatible for - * the purposes of the Work and the provisions of the compatibility - * clause in Article 5 of the EUPL shall not apply. - * - * If using the Work as, or as part of, a network application, by - * including the attribution notice(s) required under Article 5 of the EUPL - * in the end user terms of the application under an appropriate heading, - * such notice(s) shall fulfill the requirements of that article. - * ********************************************************************* */ - -package fiftyone.pipeline.web.mvc; - -import static org.junit.Assert.assertFalse; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -import java.io.IOException; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -import fiftyone.pipeline.core.data.FlowData; -import fiftyone.pipeline.web.mvc.components.FiftyOneInterceptor; -import fiftyone.pipeline.web.mvc.components.FlowDataProvider; -import fiftyone.pipeline.web.mvc.configuration.FiftyOneInterceptorConfig; -import fiftyone.pipeline.web.mvc.services.ClientsidePropertyService; -import fiftyone.pipeline.web.mvc.services.FiftyOneJSService; -import fiftyone.pipeline.web.mvc.services.PipelineResultService; -import fiftyone.pipeline.web.mvc.services.UACHService; - -public class FiftyOneInterceptorTests { - private HttpServletRequest request; - private HttpServletResponse response; - private Object handler; - - private FiftyOneInterceptor interceptor; - private PipelineResultService resultService; - private FlowData flowData; - - @Before - public void init() throws IOException { - FiftyOneInterceptorConfig config = - mock(FiftyOneInterceptorConfig.class); - when(config.getDataFilePath()) - .thenReturn("src/test/resources/Test.xml"); - - FlowDataProvider flowDataProvider = mock(FlowDataProvider.class); - ClientsidePropertyService clientsidePropertyService = - mock(ClientsidePropertyService.class); - - FiftyOneJSService fiftyOneJsService = mock(FiftyOneJSService.class); - when(fiftyOneJsService.serveJS( - any(HttpServletRequest.class), - any(HttpServletResponse.class))) - .thenReturn(true); - when(fiftyOneJsService.serveJson( - any(HttpServletRequest.class), - any(HttpServletResponse.class))) - .thenReturn(true); - - UACHService uachService = mock(UACHService.class); - - resultService = mock(PipelineResultService.class); - flowData = mock(FlowData.class); - - // Mock request, response and handler object - request = mock(HttpServletRequest.class); - response = mock(HttpServletResponse.class); - handler = mock(Object.class); - - when(flowDataProvider.getFlowData(any(HttpServletRequest.class))) - .thenReturn(flowData); - - // Create the interceptor object for testing - interceptor = new FiftyOneInterceptor( - config, - resultService, - flowDataProvider, - clientsidePropertyService, - fiftyOneJsService, - uachService); - } - - /** - * Check that 'resultService.process()' is called during - * interceptor's preHandle process - * @throws Exception - */ - @Test - public void FiftyOneInterceptor_PreHandle() throws Exception { - interceptor.preHandle(request, response, handler); - verify(resultService, times(1)) - .process(any(HttpServletRequest.class)); - } - - /** - * Check that 'close()' is called on the FlowData instance in the - * interceptor's afterCompletion processed - * @throws Exception - */ - @Test - public void FiftyOneInterceptor_AfterCompletion() throws Exception { - interceptor.afterCompletion( - request, - response, - handler, - new Exception()); - verify(flowData, times(1)) - .close(); - } - - /** - * Check that the FlowData is closed when preHandle serves the JavaScript - * itself, as Spring does not call afterCompletion when preHandle returns - * false. - * @throws Exception - */ - @Test - public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() - throws Exception { - assertFalse(interceptor.preHandle(request, response, handler)); - verify(flowData, times(1)) - .close(); - } -} +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web.mvc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import fiftyone.pipeline.core.data.FlowData; +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.web.mvc.components.FiftyOneInterceptor; +import fiftyone.pipeline.web.mvc.components.FlowDataProvider; +import fiftyone.pipeline.web.mvc.configuration.FiftyOneInterceptorConfig; +import fiftyone.pipeline.web.mvc.services.ClientsidePropertyService; +import fiftyone.pipeline.web.mvc.services.FiftyOneJSService; +import fiftyone.pipeline.web.mvc.services.PipelineResultService; +import fiftyone.pipeline.web.mvc.services.UACHService; + +public class FiftyOneInterceptorTests { + private HttpServletRequest request; + private HttpServletResponse response; + private Object handler; + + private FiftyOneInterceptor interceptor; + private PipelineResultService resultService; + private FlowData flowData; + + @Before + public void init() throws IOException { + FiftyOneInterceptorConfig config = + mock(FiftyOneInterceptorConfig.class); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/Test.xml"); + + FlowDataProvider flowDataProvider = mock(FlowDataProvider.class); + ClientsidePropertyService clientsidePropertyService = + mock(ClientsidePropertyService.class); + + FiftyOneJSService fiftyOneJsService = mock(FiftyOneJSService.class); + when(fiftyOneJsService.serveJS( + any(HttpServletRequest.class), + any(HttpServletResponse.class))) + .thenReturn(true); + when(fiftyOneJsService.serveJson( + any(HttpServletRequest.class), + any(HttpServletResponse.class))) + .thenReturn(true); + + UACHService uachService = mock(UACHService.class); + + resultService = mock(PipelineResultService.class); + flowData = mock(FlowData.class); + + // Mock request, response and handler object + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + handler = mock(Object.class); + + when(flowDataProvider.getFlowData(any(HttpServletRequest.class))) + .thenReturn(flowData); + + // Create the interceptor object for testing + interceptor = new FiftyOneInterceptor( + config, + resultService, + flowDataProvider, + clientsidePropertyService, + fiftyOneJsService, + uachService); + } + + /** + * Check that 'resultService.process()' is called during + * interceptor's preHandle process + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_PreHandle() throws Exception { + interceptor.preHandle(request, response, handler); + verify(resultService, times(1)) + .process(any(HttpServletRequest.class)); + } + + /** + * Check that 'close()' is called on the FlowData instance in the + * interceptor's afterCompletion processed + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_AfterCompletion() throws Exception { + interceptor.afterCompletion( + request, + response, + handler, + new Exception()); + verify(flowData, times(1)) + .close(); + } + + /** + * Check that the FlowData is closed when preHandle serves the JavaScript + * itself, as Spring does not call afterCompletion when preHandle returns + * false. + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() + throws Exception { + assertFalse(interceptor.preHandle(request, response, handler)); + verify(flowData, times(1)) + .close(); + } + + private FiftyOneInterceptor newInterceptor( + FiftyOneInterceptorConfig config, + FiftyOneJSService jsService) throws IOException { + return new FiftyOneInterceptor( + config, + mock(PipelineResultService.class), + mock(FlowDataProvider.class), + mock(ClientsidePropertyService.class), + jsService, + mock(UACHService.class)); + } + + private FiftyOneInterceptorConfig newConfig( + boolean sequence, + boolean setHeaders, + String jsonPath, + String javaScriptPath) { + FiftyOneInterceptorConfig config = mock(FiftyOneInterceptorConfig.class); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/NoSequence.xml"); + when(config.getClientsidePropertiesEnabled()).thenReturn(true); + when(config.getSequenceElementEnabled()).thenReturn(sequence); + when(config.getSetHeadersElementEnabled()).thenReturn(setHeaders); + when(config.getJsonPath()).thenReturn(jsonPath); + when(config.getJavaScriptPath()).thenReturn(javaScriptPath); + return config; + } + + private static boolean hasElement( + FiftyOneInterceptor interceptor, + Class type) { + for (FlowElement element : interceptor.pipeline.getFlowElements()) { + if (type.isInstance(element)) { + return true; + } + } + return false; + } + + /** + * Check that both elements are added when the configuration allows it. + */ + @Test + public void FiftyOneInterceptor_ElementsEnabled_Added() throws Exception { + FiftyOneInterceptor created = newInterceptor( + newConfig(true, true, null, null), mock(FiftyOneJSService.class)); + + assertTrue(hasElement(created, SequenceElement.class)); + assertTrue(hasElement(created, SetHeadersElement.class)); + } + + /** + * Check that the switches stop each element being added. + */ + @Test + public void FiftyOneInterceptor_ElementsDisabled_NotAdded() + throws Exception { + FiftyOneInterceptorConfig config = newConfig(false, false, null, null); + when(config.getClientsidePropertiesEnabled()).thenReturn(false); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/NoJson.xml"); + FiftyOneInterceptor created = newInterceptor( + config, mock(FiftyOneJSService.class)); + + assertFalse(hasElement(created, SequenceElement.class)); + assertFalse(hasElement(created, SetHeadersElement.class)); + } + + /** + * Check that the build is refused when the JSON Builder Element would + * have no Sequence Element before it. + */ + @Test + public void FiftyOneInterceptor_NoSequenceElementWithJson_Refused() { + RuntimeException e = assertThrows(RuntimeException.class, + () -> newInterceptor( + newConfig(false, true, null, null), + mock(FiftyOneJSService.class))); + + assertTrue(e.getCause().getMessage().contains("Sequence Element")); + } + + /** + * Check that the JSON path is the JavaScript Builder Element's callback + * endpoint and, with the JavaScript path, goes to the service that serves + * them. + */ + @Test + public void FiftyOneInterceptor_Paths_UsedByBuilderAndService() + throws Exception { + FiftyOneJSService jsService = mock(FiftyOneJSService.class); + FiftyOneInterceptor created = newInterceptor( + newConfig(true, true, "/51dpipeline/json", "/scripts/51d.js"), + jsService); + + verify(jsService).setPaths("/scripts/51d.js", "/51dpipeline/json"); + for (FlowElement element : created.pipeline.getFlowElements()) { + if (element instanceof JavaScriptBuilderElement) { + Field field = + JavaScriptBuilderElement.class.getDeclaredField("endpoint"); + field.setAccessible(true); + assertEquals("/51dpipeline/json", field.get(element)); + } + } + } + + /** + * Check that no paths in the configuration mean the default paths. + */ + @Test + public void FiftyOneInterceptor_NoPaths_DefaultPaths() throws Exception { + FiftyOneJSService jsService = mock(FiftyOneJSService.class); + newInterceptor(newConfig(true, true, null, null), jsService); + + verify(jsService).setPaths(null, "/51dpipeline/json"); + } + + /** + * Check that the paths the application can call are mapped, the old JSON + * path included. + */ + @Test + public void FiftyOneInterceptor_EnableClientsideProperties_MapsBothJsonPaths() { + ViewControllerRegistry registry = mock(ViewControllerRegistry.class); + + FiftyOneInterceptor.enableClientsideProperties(registry); + + verify(registry).addViewController("/51Degrees.core.js"); + verify(registry).addViewController("/51dpipeline/json"); + verify(registry).addViewController("/51Degrees.core.json"); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml new file mode 100644 index 000000000..eece81fbe --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml @@ -0,0 +1,7 @@ + + + + JavaScriptBuilderElement + + + diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml new file mode 100644 index 000000000..3fd181f26 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml @@ -0,0 +1,10 @@ + + + + JsonBuilderElement + + + JavaScriptBuilderElement + + + diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java index acd9791db..7298dd10e 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java @@ -30,10 +30,17 @@ public class Constants { public static final String CORE_JS_NAME = "51Degrees.core.js"; /** - * The name of the JSON served to the client. + * The name the JSON was served under before {@link #DEFAULT_JSON_PATH} + * became the default. It is still answered. */ public static final String CORE_JSON_NAME = "51Degrees.core.json"; + /** + * The default path the JSON is served from, which the specification + * gives. + */ + public static final String DEFAULT_JSON_PATH = "/51dpipeline/json"; + /** * Key prefix used for 51Degrees data stored in the HTTP context. */ @@ -74,4 +81,36 @@ public class Constants { */ public static final boolean DEFAULT_CLIENTSIDE_ENABLED = true; + /** + * Default value for the options that add the Sequence Element and the + * Set Headers Element. + */ + public static final boolean DEFAULT_ELEMENT_ENABLED = true; + + /** + * Name of the filter init parameter that sets the path the JavaScript is + * served from. + */ + public static final String INIT_PARAM_JAVASCRIPT_PATH = "javascript-path"; + + /** + * Name of the filter init parameter that sets the path the JSON is + * served from. + */ + public static final String INIT_PARAM_JSON_PATH = "json-path"; + + /** + * Name of the filter init parameter that sets whether the Sequence + * Element is added to the Pipeline when it is missing. + */ + public static final String INIT_PARAM_SEQUENCE_ELEMENT_ENABLED = + "sequence-element-enabled"; + + /** + * Name of the filter init parameter that sets whether the Set Headers + * Element is added to the Pipeline when it is missing. + */ + public static final String INIT_PARAM_SET_HEADERS_ELEMENT_ENABLED = + "set-headers-element-enabled"; + } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java index 098da0926..b1090a6ef 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java @@ -40,8 +40,15 @@ import java.io.File; import java.io.IOException; +import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; +import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_ENABLED; import static fiftyone.pipeline.web.Constants.DEFAULT_CONFIG_FILE; +import static fiftyone.pipeline.web.Constants.DEFAULT_ELEMENT_ENABLED; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_JAVASCRIPT_PATH; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_JSON_PATH; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_SEQUENCE_ELEMENT_ENABLED; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_SET_HEADERS_ELEMENT_ENABLED; /** * Servlet filter used to intercept HTTP requests and process them using the @@ -81,6 +88,19 @@ public void init(FilterConfig filterConfig) throws ServletException { clientsideEnabled = Boolean.parseBoolean(clientsideEnabledString); } + boolean sequenceElementEnabled = getBooleanInitParameter( + filterConfig, + INIT_PARAM_SEQUENCE_ELEMENT_ENABLED, + DEFAULT_ELEMENT_ENABLED); + boolean setHeadersElementEnabled = getBooleanInitParameter( + filterConfig, + INIT_PARAM_SET_HEADERS_ELEMENT_ENABLED, + DEFAULT_ELEMENT_ENABLED); + String javaScriptPath = getPathInitParameter( + filterConfig, INIT_PARAM_JAVASCRIPT_PATH, CORE_JS_NAME); + String jsonPath = getPathInitParameter( + filterConfig, INIT_PARAM_JSON_PATH, DEFAULT_JSON_PATH); + ServletContext context = config.getServletContext(); File configFile = new File(context.getRealPath(configFileName)); @@ -96,7 +116,8 @@ public void init(FilterConfig filterConfig) throws ServletException { PipelineOptions options = (PipelineOptions) unmarshaller.unmarshal(configFile); pipeline = StartupHelpers.buildFromConfiguration(builder, options, - clientsideEnabled, context.getContextPath()); + clientsideEnabled, context.getContextPath(), jsonPath, + sequenceElementEnabled, setHeadersElementEnabled); } catch (Exception e) { throw new ServletException(e); } @@ -110,7 +131,26 @@ public void init(FilterConfig filterConfig) throws ServletException { pipeline); jsService = new FiftyOneJSServiceCore.Default( clientsidePropertyServiceCore, - clientsideEnabled); + clientsideEnabled, + javaScriptPath, + jsonPath); + } + + private static boolean getBooleanInitParameter( + FilterConfig filterConfig, + String name, + boolean defaultValue) { + String value = filterConfig.getInitParameter(name); + return value == null ? defaultValue : Boolean.parseBoolean(value); + } + + private static String getPathInitParameter( + FilterConfig filterConfig, + String name, + String defaultValue) { + String value = filterConfig.getInitParameter(name); + return value == null || value.trim().isEmpty() ? + defaultValue : value.trim(); } @Override diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java index 77a373429..7e13e8a2a 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java @@ -100,6 +100,43 @@ public static Pipeline buildFromConfiguration( boolean clientSideEvidenceEnabled, String contextRoot) throws Exception { + return StartupHelpers.buildFromConfiguration(builder, options, + clientSideEvidenceEnabled, contextRoot, Constants.DEFAULT_JSON_PATH, + Constants.DEFAULT_ELEMENT_ENABLED, + Constants.DEFAULT_ELEMENT_ENABLED); + } + + /** + * Configure the extra web elements required and build the Pipeline using + * the {@link PipelineBuilder#buildFromConfiguration(PipelineOptions)} + * method. + * @param builder to build the Pipeline + * @param options to build the Pipeline with + * @param clientSideEvidenceEnabled true if client-side evidence is enabled + * in the configuration. This will add JSON + * and JavaScript elements to the Pipeline + * @param contextRoot The context-root setting from the web.xml. This is + * needed in order to create the correct callback URL + * for the JavascriptBuilder. + * @param jsonPath the path the JSON is served from, which the + * JavascriptBuilder calls back to + * @param sequenceElementEnabled true to add the Sequence Element when the + * configuration does not include one + * @param setHeadersElementEnabled true to add the Set Headers Element + * when the configuration does not + * include one + * @return new {@link Pipeline} instance + * @throws Exception if there was an error building the Pipeline + */ + public static Pipeline buildFromConfiguration( + PipelineBuilder builder, + PipelineOptions options, + boolean clientSideEvidenceEnabled, + String contextRoot, + String jsonPath, + boolean sequenceElementEnabled, + boolean setHeadersElementEnabled) throws Exception { + if (options == null || options.elements == null || options.elements.size() == 0) { @@ -107,8 +144,16 @@ public static Pipeline buildFromConfiguration( "Could not find pipeline configuration information"); } + // A caller that has no path set gives the default, so that the + // JavaScript still calls back to somewhere the JSON is served from. + String path = jsonPath == null || jsonPath.isEmpty() ? + Constants.DEFAULT_JSON_PATH : jsonPath; + String endpoint = "/" + (path.startsWith("/") ? + path.substring(1) : path); + // Add the sequence element. - if (getElementIndex( + if (sequenceElementEnabled && + getElementIndex( options, SequenceElement.class.getSimpleName()) == -1) { // The sequence element is not included so add it. @@ -147,7 +192,7 @@ public static Pipeline buildFromConfiguration( // The js builder element is not included so add it. ElementOptions javaScriptBuilderElement = new ElementOptions(); javaScriptBuilderElement.builderName = JavaScriptBuilderElement.class.getSimpleName(); - javaScriptBuilderElement.buildParameters.put("Endpoint", "/" + Constants.CORE_JSON_NAME); + javaScriptBuilderElement.buildParameters.put("Endpoint", endpoint); if(contextRoot != null) { javaScriptBuilderElement.buildParameters.put("ContextRoot", contextRoot); } @@ -157,7 +202,7 @@ public static Pipeline buildFromConfiguration( // There is already a JavaScript builder config so check if // the endpoint is specified. If not, add it. if (options.elements.get(jsIndex).buildParameters.containsKey("Endpoint") == false) { - options.elements.get(jsIndex).buildParameters.put("Endpoint", "/" + Constants.CORE_JSON_NAME); + options.elements.get(jsIndex).buildParameters.put("Endpoint", endpoint); } // Same for context root. if(options.elements.get(jsIndex).buildParameters.containsKey("ContextRoot") == false && @@ -168,8 +213,26 @@ public static Pipeline buildFromConfiguration( } - // Add the set-headers element. + // The JSON Builder Element fails on every request without a + // sequence number, which only the Sequence Element supplies. if (getElementIndex( + options, + JsonBuilderElement.class.getSimpleName()) != -1 && + getElementIndex( + options, + SequenceElement.class.getSimpleName()) == -1) { + throw new PipelineConfigurationException( + "The Pipeline has a JSON Builder Element but no Sequence " + + "Element, and the JSON Builder Element needs the " + + "sequence number the Sequence Element supplies. Turn " + + "sequence-element-enabled back on, add a Sequence " + + "Element to the configuration, or turn off " + + "client-side properties."); + } + + // Add the set-headers element. + if (setHeadersElementEnabled && + getElementIndex( options, SetHeadersElement.class.getSimpleName()) == -1) { // The set-headers element is not included so add it. diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java index f1737d9b0..70d27fa0b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java @@ -30,6 +30,7 @@ import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; import static fiftyone.pipeline.web.Constants.CORE_JSON_NAME; +import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; /** * Service that provides the 51Degrees javascript when requested. @@ -71,9 +72,12 @@ boolean serveJson(HttpServletRequest request, HttpServletResponse response) class Default implements FiftyOneJSServiceCore { protected final ClientsidePropertyServiceCore clientsidePropertyServiceCore; protected boolean enabled; + protected String javaScriptPath; + protected String jsonPath; /** - * Construct a new instance. + * Construct a new instance that serves the JavaScript and the JSON + * from the default paths. * @param clientsidePropertyServiceCore used to get the JavaScript to * add to the returned file * @param enabled true if the service should be enabled @@ -81,8 +85,46 @@ class Default implements FiftyOneJSServiceCore { public Default( ClientsidePropertyServiceCore clientsidePropertyServiceCore, boolean enabled) { + this(clientsidePropertyServiceCore, enabled, null, null); + } + + /** + * Construct a new instance. + * @param clientsidePropertyServiceCore used to get the JavaScript to + * add to the returned file + * @param enabled true if the service should be enabled + * @param javaScriptPath the path inside the application that the + * JavaScript is served for, or null or empty + * for the default + * @param jsonPath the path inside the application that the JSON + * is served for, or null or empty for the default + */ + public Default( + ClientsidePropertyServiceCore clientsidePropertyServiceCore, + boolean enabled, + String javaScriptPath, + String jsonPath) { this.clientsidePropertyServiceCore = clientsidePropertyServiceCore; this.enabled = enabled; + this.javaScriptPath = orDefault(javaScriptPath, CORE_JS_NAME); + this.jsonPath = orDefault(jsonPath, DEFAULT_JSON_PATH); + } + + private static String orDefault(String path, String defaultPath) { + return path == null || path.isEmpty() ? defaultPath : path; + } + + /** + * Set the paths the JavaScript and the JSON are served from. + * @param javaScriptPath the path inside the application that the + * JavaScript is served for, or null or empty + * for the default + * @param jsonPath the path inside the application that the JSON + * is served for, or null or empty for the default + */ + public void setPaths(String javaScriptPath, String jsonPath) { + this.javaScriptPath = orDefault(javaScriptPath, CORE_JS_NAME); + this.jsonPath = orDefault(jsonPath, DEFAULT_JSON_PATH); } @Override @@ -90,14 +132,36 @@ public boolean serveJS( HttpServletRequest request, HttpServletResponse response) throws IOException { boolean result = false; - if (request.getRequestURL().toString().toLowerCase() - .endsWith(CORE_JS_NAME.toLowerCase())) { + if (matches(request, javaScriptPath)) { serveCoreJS(request, response); result = true; } return result; } + /** + * Whether the request is for the path. The path is compared with + * the whole path of the request inside the application, after the + * context path, ignoring case and with or without a leading slash. + */ + private static boolean matches( + HttpServletRequest request, + String path) { + String requested = request.getRequestURI(); + if (requested == null) { + return false; + } + String contextPath = request.getContextPath(); + if (contextPath != null && requested.startsWith(contextPath)) { + requested = requested.substring(contextPath.length()); + } + return withSlash(requested).equalsIgnoreCase(withSlash(path)); + } + + private static String withSlash(String path) { + return path.startsWith("/") ? path : "/" + path; + } + private void serveCoreJS( HttpServletRequest request, HttpServletResponse response) throws IOException { @@ -119,8 +183,8 @@ public boolean serveJson( boolean result = false; - if (request.getRequestURI().toString().toLowerCase() - .endsWith(CORE_JSON_NAME.toLowerCase())) { + if (matches(request, jsonPath) || + matches(request, CORE_JSON_NAME)) { serveCoreJson(request, response); result = true; } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java new file mode 100644 index 000000000..c0f8ae388 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java @@ -0,0 +1,188 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FiftyOneJSServiceCore; +import org.junit.Before; +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The paths the JavaScript and the JSON are served from. + * @see Specification + */ +public class FiftyOneJSServiceCoreTests { + + private ClientsidePropertyServiceCore clientside; + private HttpServletRequest request; + private HttpServletResponse response; + + @Before + public void init() { + clientside = mock(ClientsidePropertyServiceCore.class); + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + } + + private void requestPath(String path) { + requestPath("", path); + } + + /** + * The request URI holds the context path and then the path inside the + * application. + */ + private void requestPath(String contextPath, String path) { + when(request.getRequestURL()).thenReturn( + new StringBuffer("http://example.com" + contextPath + path)); + when(request.getRequestURI()).thenReturn(contextPath + path); + when(request.getContextPath()).thenReturn(contextPath); + } + + @Test + public void DefaultPaths_JavaScriptAndJsonServed() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + + requestPath("/51Degrees.core.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + + verify(clientside, times(1)).serveJavascript(request, response); + verify(clientside, times(1)).serveJson(request, response); + } + + /** + * The path the JSON was served from before the specification's path + * became the default is still answered. + */ + @Test + public void DefaultPaths_OldJsonPathStillServed() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + + requestPath("/51Degrees.core.json"); + assertTrue(service.serveJson(request, response)); + requestPath("/app", "/51Degrees.core.json"); + assertTrue(service.serveJson(request, response)); + + verify(clientside, times(2)).serveJson(request, response); + } + + @Test + public void ConfiguredPaths_ServedInsteadOfTheDefaults() + throws IOException { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, true, "/scripts/51d.js", "/51dpipeline/json"); + + requestPath("/scripts/51d.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + + requestPath("/51Degrees.core.js"); + assertFalse(service.serveJS(request, response)); + requestPath("/other/json"); + assertFalse(service.serveJson(request, response)); + + verify(clientside, times(1)).serveJavascript(request, response); + verify(clientside, times(1)).serveJson(request, response); + } + + /** + * The configured path is the whole path inside the application, so a + * longer path that ends the same way is not answered. + */ + @Test + public void ConfiguredPath_OnlyAnExactMatchIsAnswered() + throws IOException { + for (String configured : new String[] {"json", "/json"}) { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, true, configured, configured); + + requestPath("/json"); + assertTrue(configured, service.serveJson(request, response)); + assertTrue(configured, service.serveJS(request, response)); + requestPath("/anything/json"); + assertFalse(configured, service.serveJson(request, response)); + assertFalse(configured, service.serveJS(request, response)); + requestPath("/json/more"); + assertFalse(configured, service.serveJson(request, response)); + requestPath("/notjson"); + assertFalse(configured, service.serveJson(request, response)); + } + } + + @Test + public void ContextPath_IsNotPartOfThePath() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + + requestPath("/app", "/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + requestPath("/app", "/other/51dpipeline/json"); + assertFalse(service.serveJson(request, response)); + requestPath("/app", "/51Degrees.core.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/app", "/other/51Degrees.core.js"); + assertFalse(service.serveJS(request, response)); + } + + @Test + public void EmptyPaths_UseTheDefaults() throws IOException { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, true, "", null); + + requestPath("/51Degrees.core.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + } + + @Test + public void ClientSideDisabled_NothingWritten() throws IOException { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, false, "/51d.js", "/51dpipeline/json"); + + requestPath("/51d.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + + verify(clientside, never()).serveJavascript(request, response); + verify(clientside, never()).serveJson(request, response); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java index e0f7b7508..d0e60d9e4 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java @@ -22,32 +22,53 @@ package fiftyone.pipeline.web; +import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.data.ElementData; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClientDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement; +import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.Unmarshaller; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.util.HashMap; +import java.util.List; import java.util.Map; import static fiftyone.pipeline.engines.fiftyone.data.SetHeadersData.*; import static fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement.*; import static fiftyone.pipeline.web.Constants.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; +@SuppressWarnings("rawtypes") public class PipelineFilterTests { private FilterConfig config; @@ -183,4 +204,182 @@ public void PipelineFilter_CloseFlowData_ChainFails() throws Exception { } verify(flowData, times(1)).close(); } + + private List elementsOf(PipelineFilter initialised) + throws Exception { + Field field = PipelineFilter.class.getDeclaredField("pipeline"); + field.setAccessible(true); + return ((Pipeline) field.get(initialised)).getFlowElements(); + } + + private static boolean hasElement( + List elements, + Class type) { + for (FlowElement element : elements) { + if (type.isInstance(element)) { + return true; + } + } + return false; + } + + private static String endpointOf(List elements) + throws Exception { + for (FlowElement element : elements) { + if (element instanceof JavaScriptBuilderElement) { + Field field = + JavaScriptBuilderElement.class.getDeclaredField("endpoint"); + field.setAccessible(true); + return (String) field.get(element); + } + } + return null; + } + + private PipelineFilter initFilterWith(String name, String value) + throws Exception { + doReturn(value).when(config).getInitParameter(name); + PipelineFilter initialised = new PipelineFilter(); + initialised.init(config); + return initialised; + } + + /** + * Check that the Sequence Element and the Set Headers Element are added + * when the init parameters do not say otherwise. + */ + @Test + public void PipelineFilter_Defaults_AddsSequenceAndSetHeaders() + throws Exception { + filter = new PipelineFilter(); + filter.init(config); + + List elements = elementsOf(filter); + assertTrue(hasElement(elements, SequenceElement.class)); + assertTrue(hasElement(elements, SetHeadersElement.class)); + assertTrue(hasElement(elements, JsonBuilderElement.class)); + assertTrue(hasElement(elements, JavaScriptBuilderElement.class)); + assertEquals("/51dpipeline/json", endpointOf(elements)); + } + + @Test + public void PipelineFilter_SequenceElementDisabled_NotAdded() + throws Exception { + doReturn("false").when(config) + .getInitParameter("clientside-properties-enabled"); + filter = initFilterWith("sequence-element-enabled", "false"); + + List elements = elementsOf(filter); + assertFalse(hasElement(elements, SequenceElement.class)); + assertTrue(hasElement(elements, SetHeadersElement.class)); + } + + /** + * Check that the build is refused when the JSON Builder Element would + * have no Sequence Element before it, as the JSON Builder Element then + * fails on every request. + */ + @Test + public void PipelineFilter_SequenceElementDisabledWithClientSide_Refused() { + doReturn("false").when(config) + .getInitParameter("sequence-element-enabled"); + filter = new PipelineFilter(); + + ServletException e = assertThrows( + ServletException.class, () -> filter.init(config)); + + assertTrue(e.getCause().getMessage().contains("Sequence Element")); + assertTrue(e.getCause().getMessage() + .contains("sequence-element-enabled")); + } + + @Test + public void PipelineFilter_SetHeadersElementDisabled_NotAdded() + throws Exception { + filter = initFilterWith("set-headers-element-enabled", "false"); + + List elements = elementsOf(filter); + assertTrue(hasElement(elements, SequenceElement.class)); + assertFalse(hasElement(elements, SetHeadersElement.class)); + } + + /** + * Check that the client-side switch still removes only the JSON Builder + * Element and the JavaScript Builder Element. + */ + @Test + public void PipelineFilter_ClientSideDisabled_KeepsSequenceAndSetHeaders() + throws Exception { + filter = initFilterWith("clientside-properties-enabled", "false"); + + List elements = elementsOf(filter); + assertTrue(hasElement(elements, SequenceElement.class)); + assertTrue(hasElement(elements, SetHeadersElement.class)); + assertFalse(hasElement(elements, JsonBuilderElement.class)); + assertFalse(hasElement(elements, JavaScriptBuilderElement.class)); + } + + /** + * Check that the script the JavaScript Builder Element renders calls back + * to the specification's JSON path when the init parameter is not set. + */ + @Test + public void PipelineFilter_Defaults_ScriptCallsTheNewJsonPath() + throws Exception { + filter = new PipelineFilter(); + filter.init(config); + Field field = PipelineFilter.class.getDeclaredField("pipeline"); + field.setAccessible(true); + Pipeline built = (Pipeline) field.get(filter); + + try (FlowData processed = built.createFlowData()) { + processed.addEvidence("header.Host", "example.com"); + processed.addEvidence("header.protocol", "https"); + processed.process(); + + String script = processed + .get(JavaScriptBuilderData.class) + .getJavaScript(); + assertTrue(script.contains("https://example.com/51dpipeline/json")); + } + } + + /** + * Check that the JavaScript Builder Element calls back to the JSON path + * set in the init parameter. + */ + @Test + public void PipelineFilter_JsonPath_IsTheCallbackEndpoint() + throws Exception { + filter = initFilterWith("json-path", "/custom/json"); + + assertEquals("/custom/json", endpointOf(elementsOf(filter))); + } + + /** + * Check that a caller of the public method that passes no JSON path gets + * the default path, rather than a null pointer exception. + */ + @Test + public void StartupHelpers_NoJsonPath_IsTheDefault() throws Exception { + for (String jsonPath : new String[] { null, "" }) { + File configFile = new File(getClass().getClassLoader() + .getResource("config.xml").getFile()); + Unmarshaller unmarshaller = JAXBContext + .newInstance(PipelineOptions.class).createUnmarshaller(); + PipelineOptions options = + (PipelineOptions) unmarshaller.unmarshal(configFile); + PipelineBuilder builder = new PipelineBuilder() + .addService(new DataUpdateServiceDefault( + LoggerFactory.getLogger( + DataUpdateService.class.getSimpleName()), + new HttpClientDefault())); + + try (Pipeline built = StartupHelpers.buildFromConfiguration( + builder, options, true, null, jsonPath, true, true)) { + assertEquals(DEFAULT_JSON_PATH, + endpointOf(built.getFlowElements())); + } + } + } } From ddb7d04ab8c0d55f5104cc811381392dca38dc2e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 12:18:33 +0000 Subject: [PATCH 72/95] Send every Evidence value but the reserved names in the cloud request (#99) * Test that the cloud request carries every Evidence value but the reserved names The Cloud Request Engine must send every Evidence value apart from the reserved names resource, license, licence and values, under any prefix and in any letter case, whether or not the cloud service lists the key in its evidencekeys answer. The test that only accepted Evidence is sent is replaced by one that the request carries Evidence the service does not list, and still leaves out reserved names under prefixes it does not list. * Send every Evidence value but the reserved names in the cloud request The Cloud Request Engine sent only the Evidence whose keys the cloud service lists in its evidencekeys answer. It now sends every Evidence value apart from the reserved names resource, license, licence and values, under any prefix and in any letter case, and apart from the web request, as .NET does. What the service uses changes over time, and knowing about the other Evidence helps it improve. The accepted Evidence still serves as the engine's Evidence key filter, for the Pipeline and the web Vary header. * Test that Evidence with no prefix is sent, and that prefixes ignore case Keys only usually carry a prefix, as the Overview of the Evidence specification says, and .NET sends a key with no prefix under its whole key unless it is a reserved name. Java drops it. .NET also compares prefixes without regard to case, where Java treats Query.User-Agent as Evidence with some other prefix, so header Evidence of the same name takes precedence over it with a warning. * Send Evidence with no prefix under its whole key, and compare prefixes ignoring case A key with no separator now has the whole key as both its prefix and the name it is sent under, as .NET's AddQueryData takes it, so it is sent unless it is a reserved name. Prefixes are compared without regard to case when Evidence is grouped and when a conflict is reported, as .NET's KeyHasPrefix and AddQueryData compare them. * Test where Evidence with no prefix ranks in the cloud request Evidence with no prefix ranks with Evidence of any prefix other than query, header and cookie, so header Evidence of the same name takes precedence over it, and a bare key that is itself a prefix, such as query, ranks with that prefix's Evidence, as in .NET. The Evidence specification's precedence order covers prefixes only, so the test records the choice. * Say where Evidence with no prefix ranks, and what hasKeyPrefix compares The getFormData comment now gives the precedence order and where Evidence with no prefix ranks in it, and the hasKeyPrefix comment says a key with no separator is its own prefix. * Test that a repeated cloud request still hits the cache * Make the cache test's second request differ in Evidence the service does not list Two identical requests share a cache entry whatever the key is built from, so the test passed without showing that the key is built from the advertised Evidence. The second request now differs only in header.sec-ch-ua-platform, which the cloud service does not list, and one request must still be posted, carrying the first request's value. The test fails if the cache key takes every Evidence value. --- .../CloudRequestEngineDefault.java | 68 +++++--- .../flowelements/CloudRequestEngineTests.java | 146 +++++++++++++++++- 2 files changed, 183 insertions(+), 31 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 1ca5d11c3..2e2571081 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -470,7 +470,11 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { /** * The Evidence to send, keyed by the name the cloud service expects. * Only Evidence that {@link #isSent(String)} allows is included, and - * where two prefixes give the same name the precedence order decides. + * where two prefixes give the same name the precedence order decides, + * query first, then header, then cookie, then any other prefix. + * Evidence with no prefix ranks with the other prefixes, unless the + * whole key is query, header or cookie, as {@link #keyPrefix(String)} + * gives the whole key as its prefix. * @param flowData the FlowData * @return the form fields to send */ @@ -497,22 +501,43 @@ Map getFormData(FlowData flowData) { } /** - * Whether Evidence is sent to the cloud service. Only keys the service - * accepts are sent, never one that strips to a name in - * {@link #ENGINE_FIELDS}, compared without regard to case, and never - * the web request. + * Whether Evidence is sent to the cloud service. Every key is sent, + * including keys the service does not list as accepted and keys with no + * prefix, apart from the web request and a key whose + * {@link #keySuffix(String)} is in {@link #ENGINE_FIELDS}, compared + * without regard to case. * @param key the Evidence key * @return true if the Evidence is sent */ private boolean isSent(String key) { - if (HTTP_REQUEST_KEY.equalsIgnoreCase(key) || - evidenceKeyFilter.include(key) == false) { - return false; - } - String[] parts = key.split( - Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); - return parts.length == 2 && - ENGINE_FIELDS.contains(parts[1].toLowerCase(Locale.ROOT)) == false; + return HTTP_REQUEST_KEY.equalsIgnoreCase(key) == false && + ENGINE_FIELDS.contains( + keySuffix(key).toLowerCase(Locale.ROOT)) == false; + } + + /** + * The prefix of an Evidence key, being the part before its first + * separator, or the whole key where it has none. + * @param key the Evidence key + * @return the prefix + */ + private static String keyPrefix(String key) { + int separator = key.indexOf(Constants.EVIDENCE_SEPERATOR); + return separator >= 0 ? key.substring(0, separator) : key; + } + + /** + * The name an Evidence key is sent under, being the part after its + * first separator, or the whole key where it has none. The name can + * hold a dot itself, as id.usage does in query.id.usage. + * @param key the Evidence key + * @return the name to send + */ + private static String keySuffix(String key) { + int separator = key.indexOf(Constants.EVIDENCE_SEPERATOR); + return separator >= 0 ? + key.substring(separator + Constants.EVIDENCE_SEPERATOR.length()) : + key; } /** @@ -531,13 +556,8 @@ private void addFormData( Collections.sort(evidenceKeys, Collections.reverseOrder()); for (String evidenceKey : evidenceKeys) { - // Split at the first separator only. Everything after the prefix - // is the name the cloud service expects, and that name can hold - // a dot itself, as id.usage does in query.id.usage. - String[] evidenceKeyParts = evidenceKey.split( - Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); - String prefix = evidenceKeyParts[0]; - String suffix = evidenceKeyParts[1]; + String prefix = keyPrefix(evidenceKey); + String suffix = keySuffix(evidenceKey); // Check and add the evidence to the query parameters. if (formData.containsKey(suffix) == false) { @@ -549,7 +569,7 @@ private void addFormData( // warning, if the evidence prefix is not query. Otherwise a // warning is not needed as query evidence is expected // to overwrite any existing evidence with the same suffix. - if (prefix.equals("query") == false) { + if (prefix.equalsIgnoreCase("query") == false) { Map conflicts = new HashMap<>(); for (String key : allEvidence.keySet()) { if (key.equals(evidenceKey) == false && key.contains(suffix)) { @@ -607,13 +627,15 @@ Map getSelectedEvidence(Map evidence, String typ } /** - * Check that the key of a KeyValuePair has the given prefix. + * Check that an Evidence key has the given prefix, being the part + * before its first separator, or the whole key where it has none, + * compared without regard to case. * @param itemKey key to check * @param prefix the prefix to check for * @return true if the key has the prefix */ private boolean hasKeyPrefix(String itemKey, String prefix) { - return itemKey.startsWith(prefix + "."); + return keyPrefix(itemKey).equalsIgnoreCase(prefix); } @Override diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 9e9f3a006..994408d1e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -631,20 +631,55 @@ public void process_PostsWholeEvidenceName() throws Exception { } /** - * Only the Evidence whose keys the cloud service accepts is sent, with - * the keys compared without regard to case. + * Every Evidence value is sent, including Evidence the cloud service + * does not list as accepted, apart from the reserved names under any + * prefix. * See the Processing section of the cloud request engine specification. */ @Test - public void process_SendsOnlyAcceptedEvidence() throws Exception { - acceptEvidence(Arrays.asList( - "query.User-Agent", "header.Accept-Language")); + public void process_SendsEveryEvidenceValueButTheReservedNames() + throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); Map evidence = new HashMap<>(); evidence.put("query.User-Agent", "iPhone"); evidence.put("header.accept-language", "en-GB"); evidence.put("header.X-Other", "other"); evidence.put("cookie.session", "abc"); evidence.put("server.client-ip", "1.2.3.4"); + evidence.put("other.VALUES", "ismobile"); + evidence.put("server.licence", "evidence-licence"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=iPhone", + "accept-language=en-GB", + "X-Other=other", + "session=abc", + "client-ip=1.2.3.4")), + new HashSet<>(items), + "The request did not carry every Evidence value apart from " + + "the reserved names: " + items); + } + + /** + * Evidence whose key has no prefix is sent under its whole key, unless + * the key is a reserved name in any letter case. Keys only usually + * carry a prefix, as the Overview of the Evidence specification says. + */ + @Test + public void process_SendsEvidenceWithNoPrefixUnderItsWholeKey() + throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("bare", "value"); + evidence.put("Values", "ismobile"); + evidence.put("license", "evidence-license"); + evidence.put("RESOURCE", "evidence-resource"); + evidence.put("licence", "evidence-licence"); List items = postedItems(null, evidence); @@ -652,10 +687,64 @@ public void process_SendsOnlyAcceptedEvidence() throws Exception { new HashSet<>(Arrays.asList( "resource=resourcekey", "User-Agent=iPhone", - "accept-language=en-GB")), + "bare=value")), new HashSet<>(items), - "The request carried Evidence the cloud service does not " + - "accept: " + items); + "The request did not carry the Evidence with no prefix, or " + + "carried a reserved name: " + items); + } + + /** + * Prefixes are compared without regard to case, so query Evidence + * takes precedence over header Evidence of the same name, with no + * warning, whatever the case of its prefix. Keys are case-insensitive, + * as the Overview of the Evidence specification says. + */ + @Test + public void process_ComparesPrefixesWithoutRegardToCase() + throws Exception { + loggerFactory.loggers.clear(); + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("Query.User-Agent", "query-value"); + evidence.put("header.User-Agent", "header-value"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=query-value")), + new HashSet<>(items), + "The query Evidence did not take precedence: " + items); + loggerFactory.assertMaxWarnings(0); + } + + /** + * Evidence with no prefix ranks with Evidence of any prefix other than + * query, header and cookie, so it has the lowest precedence, unless the + * whole key is one of those prefixes, which puts it with that prefix's + * Evidence. The precedence order of the Evidence specification orders + * prefixes only, so this follows .NET. + */ + @Test + public void process_RanksEvidenceWithNoPrefix() throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("X-Thing", "bare"); + evidence.put("header.X-Thing", "header"); + evidence.put("query", "bare-query"); + evidence.put("header.query", "header-query"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "X-Thing=header", + "query=bare-query")), + new HashSet<>(items), + "Evidence with no prefix was not ranked as .NET ranks it: " + + items); } /** @@ -727,6 +816,47 @@ public void process_LeavesOutEvidenceNamedLicence( assertEquals(expected, items); } + /** + * A cache key is built from the Evidence the engine advertises, which is + * still the keys the cloud service lists, so a second request that + * differs only in Evidence the service does not list finds the entry + * the first one wrote, and one request is posted. That request carries + * the Evidence the service does not list all the same. + * See the Generation of keys section of the caching specification. + */ + @Test + public void process_RepeatedRequestHitsTheCache() throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (String platform : Arrays.asList("Windows", "macOS")) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.addEvidence("header.sec-ch-ua-platform", platform); + data.process(); + } + } + } + + assertFalse( + engine.getEvidenceKeyFilter().include("header.sec-ch-ua-platform"), + "The engine advertises a key the cloud service does not list."); + ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); + verify(httpClient, times(1)).postData(any(), any(), body.capture()); + List items = Arrays.asList( + new String(body.getValue(), StandardCharsets.UTF_8).split("&")); + assertTrue(items.contains("sec-ch-ua-platform=Windows"), + "The request did not carry the Evidence the cloud service does " + + "not list: " + items); + } + /** * Answer the evidencekeys request with the given keys. * @param keys the Evidence keys the cloud service accepts From ce406b2b4ac772120c71f8d9b98ff19ad2eb5932 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 13:50:39 +0000 Subject: [PATCH 73/95] Refuse the Window property names the specification lists as the object name (#88) * Test that the object name refuses the Window properties the specification lists * Refuse the Window property names the specification lists as the object name --- .../JavaScriptBuilderElement.java | 17 ++- .../JavaScriptObjectNameTests.java | 104 ++++++++++++++++++ 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index f8d828dc5..782c9fa7a 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -89,9 +89,11 @@ public class JavaScriptBuilderElement * Words the pattern accepts that cannot be the name of the object. These * are the reserved words of the language, including those reserved only * in strict mode, plus the three global values a top level var cannot - * replace, where the object would silently never be created. The last - * entry is the constructor the script itself defines and calls to create - * the object, so that name would clash with it. + * replace, eval, and the properties of the browser's Window object that a + * top level var cannot replace or that navigate the page away, where the + * object would never be created. The last entry is the constructor the + * script itself defines and calls to create the object, so that name + * would clash with it. */ private static final Set RESERVED_OBJECT_NAMES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( @@ -103,6 +105,15 @@ public class JavaScriptBuilderElement "return", "static", "super", "switch", "this", "throw", "true", "try", "typeof", "var", "void", "while", "with", "yield", "Infinity", "NaN", "undefined", + "eval", "arguments", + "window", "document", "top", "name", "status", "history", + "navigator", "closed", "location", "frameElement", + "isSecureContext", "crossOriginIsolated", "trustedTypes", + "crypto", "indexedDB", "localStorage", "sessionStorage", + "caches", "cookieStore", "speechSynthesis", "styleMedia", + "crashReport", "credentialless", "customElements", + "documentPictureInPicture", "fence", "launchQueue", + "originAgentCluster", "fiftyoneDegreesManager"))); /** diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java index 4c49c3fa6..712bc6873 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java @@ -223,6 +223,110 @@ public void ObjectName_InvalidFromConstructor_Refused(String name) { "")); } + /** + * The names the specification refuses that are not reserved words of the + * language, being eval, arguments, and the properties of the browser's + * Window object that a top level var cannot replace or that navigate the + * page away. + */ + static final String[] REFUSED_NAMES = { + "eval", "arguments", "window", "document", "top", "name", "status", + "history", "navigator", "closed", "location", "frameElement", + "isSecureContext", "crossOriginIsolated", "trustedTypes", "crypto", + "indexedDB", "localStorage", "sessionStorage", "caches", + "cookieStore", "speechSynthesis", "styleMedia", "crashReport", + "credentialless", "customElements", "documentPictureInPicture", + "fence", "launchQueue", "originAgentCluster" + }; + + /** + * Window properties that a top level var can replace, so the names stay + * allowed. + */ + static final String[] ALLOWED_WINDOW_NAMES = { + "self", "frames", "parent", "opener", "length" + }; + + static Stream refusedNames() { + return Stream.of(REFUSED_NAMES); + } + + static Stream allowedWindowNames() { + return Stream.of(ALLOWED_WINDOW_NAMES); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_RefusedNameFromEvidence_DefaultUsed(String name) { + JavaScriptBuilderElement element = + new JavaScriptBuilderElementBuilder(loggerFactory).build(); + evidence.put(EVIDENCE_OBJECT_NAME, name); + + String script = render(element); + + assertWorkingObject(script, Constants.DEFAULT_OBJECT_NAME); + assertFalse(Pattern.compile("var " + Pattern.quote(name) + + " = new fiftyoneDegreesManager\\(\\);") + .matcher(withoutParameters(script)).find(), + "the script should not declare '" + name + "'"); + assertEquals(1, count(script, "new fiftyoneDegreesManager()")); + List warnings = warnings(); + assertEquals(1, warnings.size(), String.join("\n", warnings)); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_RefusedNameFromBuilder_Refused(String name) { + assertThrows( + PipelineConfigurationException.class, + () -> new JavaScriptBuilderElementBuilder(loggerFactory) + .setObjectName(name) + .build()); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_RefusedNameFromConstructor_Refused(String name) { + assertThrows( + PipelineConfigurationException.class, + () -> new JavaScriptBuilderElement( + loggerFactory.getLogger("test"), + null, + "", + name, + false, + "", + "")); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_IsValidObjectName_RefusedName(String name) { + assertFalse(JavaScriptBuilderElement.isValidObjectName(name)); + } + + @ParameterizedTest + @MethodSource("allowedWindowNames") + public void ObjectName_WindowPropertyVarCanReplace_Allowed(String name) { + assertTrue(JavaScriptBuilderElement.isValidObjectName(name)); + JavaScriptBuilderElement element = + new JavaScriptBuilderElementBuilder(loggerFactory) + .setObjectName(name) + .build(); + + String script = render(element); + + assertTrue(script.contains("var " + name + " = new"), + "the script should declare '" + name + "'"); + assertEquals(0, warnings().size()); + } + + @ParameterizedTest + @ValueSource(strings = {"windows", "Window", "cachesX"}) + public void ObjectName_NearMissOfRefusedName_Allowed(String name) { + assertTrue(JavaScriptBuilderElement.isValidObjectName(name)); + } + /** * No name configured means the default name, whether the name was never * set or was set to null. From c99c6614b5692b85977dfe747ebe239a6592679c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 14:53:05 +0000 Subject: [PATCH 74/95] Read property values by index for an engine built with property names (#101) * Read property values by index for an engine built with names * Test the property index rule on both on-premise engines * Test that a property name given twice has the index of its first place C gives a property the place of its name among the names given, the first where the name is given more than once, and compares names without regard to case. The builder drops only an exact repeat, so IsMobile and ismobile both reach C. The engines' map keeps the last index for such a name, so the new tests fail on it. The test comments no longer say which test goes red or describe the change. * Take the property indexes from the names the builder gives, keeping the first Each engine now makes its map from the list of names its builder passes to the native engine, in the constructor, before any data is loaded. That is the list C takes the required property indexes from, one place for each entry in the order given, so the map holds the same indexes without running a detection on empty Evidence to get a results object, which for an engine built with names walked every component's graph at build. A name given more than once keeps its first place, as C keeps it. The map is final and never rebuilt, and the native engine's required properties configuration is no longer read back. * Time a read of IsMobile by name against a read by index PropertyIndex_ReadByIndexAgainstByName builds the native engine with the same names, reads IsMobile 20,000 times by name and 20,000 times by index from the same native results, and 20,000 times through the typed accessor, in five rounds after a warm-up. It requires every read to give the same value and prints the lowest, median and highest time of one read, so each build reports the figures for its platform. * Time the read by name through the overload the reads use A read by name with no index goes through getValueAsBool(Swig.asBytes( name)), which is also how every read by name went on rc, so the timing test now reads by name that way rather than through the String overload. * Test a reload from a data file that lacks a property the engine was named with The engine is built on the TAC file with a name the Lite file lacks, then reloaded from the Lite file. The name keeps its index, a read of it by index gives no value rather than an error, as a read by name does, and IsMobile still reads its own value. C maps a named index to the data set before it reads, and a name the data set lacks maps to -1, so the read reports no value. The test pins that, so it has no red. --- .../flowelements/DeviceDataHashDefault.java | 55 ++- .../DeviceDetectionHashEngine.java | 46 ++ .../DeviceDetectionHashEngineBuilder.java | 1 + .../flowelements/PropertyIndexTests.java | 467 ++++++++++++++++++ .../IPIntelligenceDataHashDefault.java | 74 ++- .../IPIntelligenceOnPremiseEngine.java | 46 ++ .../IPIntelligenceOnPremiseEngineBuilder.java | 1 + .../flowelements/PropertyIndexTests.java | 163 ++++++ 8 files changed, 823 insertions(+), 30 deletions(-) create mode 100644 device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java create mode 100644 ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index d78e6b443..3dad6384f 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -68,6 +68,12 @@ public class DeviceDataHashDefault */ private final List resultsList = new ArrayList<>(); + /** + * The engine which created this instance, used to turn a property name + * into the required property index the engine was built with. + */ + private final DeviceDetectionHashEngine hashEngine; + /** * Constructs a new instance. * @param logger used for logging @@ -83,6 +89,18 @@ public class DeviceDataHashDefault DeviceDetectionHashEngine engine, MissingPropertyService missingPropertyService) { super(logger, flowData, engine, missingPropertyService); + this.hashEngine = engine; + } + + /** + * Get the required property index for a property name, or null when the + * engine was not built with that name, so the read falls back to the name. + * @param propertyName name of the property + * @return required property index, or null + */ + private Integer indexOf(String propertyName) { + return hashEngine == null ? + null : hashEngine.getPropertyIndex(propertyName); } /** @@ -256,8 +274,10 @@ public AspectPropertyValue> getValues(String propertyName) { new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (VectorStringValuesSwig value = results.getValues( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (VectorStringValuesSwig value = index == null ? + results.getValues(Swig.asBytes(propertyName)) : + results.getValues(index)) { if (value.hasValue()) { try (VectorStringSwig vector = value.getValue()) { result.setValue(Collections.unmodifiableList( @@ -277,9 +297,10 @@ protected AspectPropertyValue getValueAsString(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - - try (StringValueSwig value = results.getValueAsString( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(Swig.asBytes(propertyName)) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -298,8 +319,10 @@ protected AspectPropertyValue getValueAsJavaScript( new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (StringValueSwig value = results.getValueAsString( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(Swig.asBytes(propertyName)) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(new JavaScript(value.getValue())); } @@ -317,8 +340,10 @@ protected AspectPropertyValue getValueAsInteger( AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (IntegerValueSwig value = results.getValueAsInteger( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (IntegerValueSwig value = index == null ? + results.getValueAsInteger(Swig.asBytes(propertyName)) : + results.getValueAsInteger(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -335,8 +360,10 @@ protected AspectPropertyValue getValueAsBool(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (BoolValueSwig value = results.getValueAsBool( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (BoolValueSwig value = index == null ? + results.getValueAsBool(Swig.asBytes(propertyName)) : + results.getValueAsBool(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -353,8 +380,10 @@ protected AspectPropertyValue getValueAsDouble(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (DoubleValueSwig value = results.getValueAsDouble( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (DoubleValueSwig value = index == null ? + results.getValueAsDouble(Swig.asBytes(propertyName)) : + results.getValueAsDouble(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java index 64ede4a73..2da75263e 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java @@ -62,6 +62,17 @@ public class DeviceDetectionHashEngine private volatile boolean propertiesPopulated = false; private final Random rand = new Random(); + /** + * Case insensitive map from each property name the engine was built with + * to its required property index, which is the place of the name in that + * list, the first where a name is given more than once, as the native + * engine gives it. Made from the names given at build and never from the + * data set, so an index keeps naming the same property after a reload + * from a data file whose property order differs. Empty when the engine + * was built with no names, in which case reads go by name. + */ + private final Map propertyIndexes; + /** * Construct a new instance of the {@link DeviceDetectionHashEngine}. * @param logger logger instance to use for logging @@ -70,6 +81,8 @@ public class DeviceDetectionHashEngine * @param properties native required properties configuration which define * the properties which the engine should be initialised * with + * @param propertyNames the property names {@code properties} was made + * from, in the same order * @param deviceDataFactory the factory to use when creating a * {@link DeviceDataHash} instance * @param tempDataFileDir the file where a temporary data file copy @@ -80,11 +93,13 @@ public class DeviceDetectionHashEngine AspectEngineDataFile dataFile, ConfigHashSwig config, RequiredPropertiesConfigSwig properties, + List propertyNames, ElementDataFactory deviceDataFactory, String tempDataFileDir) { super(logger, deviceDataFactory, tempDataFileDir); this.config = config; this.propertiesConfigSwig = properties; + this.propertyIndexes = indexesOf(propertyNames); addDataFile(dataFile); } @@ -113,6 +128,37 @@ public MetaDataSwig getMetaData() { return engine.getMetaData(); } + /** + * Get the required property index for a property name, or null when the + * engine was not built with that name and the read has to go by name. + * @param propertyName name of the property + * @return required property index, or null + */ + Integer getPropertyIndex(String propertyName) { + return propertyIndexes.get(propertyName); + } + + /** + * Map each name to its place in the list, keeping the first place of a + * name given more than once in any letter case. + * @param names the property names in the order the engine is built with, + * or null + * @return unmodifiable case insensitive map from name to index + */ + private static Map indexesOf(List names) { + Map indexes = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + if (names != null) { + for (int i = 0; i < names.size(); i++) { + String name = names.get(i); + if (name != null && name.isEmpty() == false) { + indexes.putIfAbsent(name, i); + } + } + } + return Collections.unmodifiableMap(indexes); + } + @Override public List getProperties() { if (isFalse(propertiesPopulated)) { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java index 774e1b574..1de52bba8 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java @@ -341,6 +341,7 @@ protected DeviceDetectionHashEngine newEngine(List properties) { dataFile, config, requiredProperties, + properties, new HashDataFactory(loggerFactory), tempDir); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java new file mode 100644 index 000000000..4c222ce41 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java @@ -0,0 +1,467 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.data.DeviceDataHash; +import fiftyone.devicedetection.hash.engine.onpremise.interop.Swig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ConfigHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.EngineHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for the property index rule, which says that a property's index is + * its place in the list of names the engine was built with, that the index is + * fixed at build, and that every read by name goes through that index. + */ +public class PropertyIndexTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * Names in an order that is deliberately not the order the data file + * holds them in, so an index taken from the data file would be wrong. + */ + private static final List NAMES = Arrays.asList( + "ScreenPixelsHeight", + "IsMobile", + "PlatformName", + "BrowserName"); + + private static final String MOBILE = + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.0 Mobile/15E148 Safari/604.1"; + + private static final int READS = 20000; + + private static final int ROUNDS = 5; + + private static File dataFile(String name) { + try { + return getFilePath(name); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private static DeviceDetectionHashEngine build( + File file, List names) throws Exception { + DeviceDetectionHashEngineBuilder builder = + new DeviceDetectionHashEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.HighPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + return builder.build(file.toString(), false); + } + + /** + * Every name the engine was built with has an index, and the index is the + * place of the name in the build time list, not its place in the data + * file. + */ + @Test + public void PropertyIndex_FromBuildTimeNames() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, NAMES)) { + for (int i = 0; i < NAMES.size(); i++) { + Integer index = engine.getPropertyIndex(NAMES.get(i)); + assertNotNull( + "Expected an index for '" + NAMES.get(i) + "'.", + index); + assertEquals( + "Expected '" + NAMES.get(i) + "' at index " + i + ".", + Integer.valueOf(i), + index); + } + } + } + + /** + * The lookup is case insensitive, because a caller may ask for a property + * in any casing. + */ + @Test + public void PropertyIndex_CaseInsensitive() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, NAMES)) { + assertEquals( + engine.getPropertyIndex("IsMobile"), + engine.getPropertyIndex("ismobile")); + } + } + + /** + * A name given twice, in any letter case, has the index of its first + * place, as the native engine gives it. + */ + @Test + public void PropertyIndex_NameGivenTwiceHasFirstIndex() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, + Arrays.asList("IsMobile", "BrowserName", "ismobile"))) { + assertEquals( + Integer.valueOf(0), + engine.getPropertyIndex("IsMobile")); + assertEquals( + Integer.valueOf(1), + engine.getPropertyIndex("BrowserName")); + } + } + + /** + * A name the engine was not built with has no index, so the read goes by + * name. + */ + @Test + public void PropertyIndex_UnknownNameHasNoIndex() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, NAMES)) { + assertNull(engine.getPropertyIndex("HardwareVendor")); + assertNull(engine.getPropertyIndex("NotAPropertyAtAll")); + } + } + + /** + * An engine built with no names has no indexes, so every read goes by + * name. + */ + @Test + public void PropertyIndex_NoNamesReadsByName() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, null)) { + assertNull(engine.getPropertyIndex("IsMobile")); + } + } + + /** + * A read by name and the typed read give the same value, because the + * typed accessor goes through the same path. + */ + @Test + public void PropertyIndex_ByNameMatchesTyped() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + DeviceDetectionHashEngine engine = build(file, NAMES); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence( + "header.user-agent", + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.0 Mobile/15E148 Safari/604.1"); + data.process(); + DeviceData device = data.get(DeviceData.class); + AspectPropertyValue typed = device.getIsMobile(); + AspectPropertyValue byName = + (AspectPropertyValue) device.get("IsMobile"); + assertEquals(typed.hasValue(), byName.hasValue()); + if (typed.hasValue()) { + assertEquals(typed.getValue(), byName.getValue()); + } + } + } + + /** + * After a reload from a data file whose properties are a superset and in + * a different order, each index still names the property it named at + * build, so every value read by name is still the right one. + */ + @Test + public void PropertyIndex_SurvivesReload() throws Exception { + File lite = dataFile(LITE_HASH_DATA_FILE_PATH); + File bigger = dataFile(TAC_HASH_DATA_FILE_NAME); + + String userAgent = + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.0 Mobile/15E148 Safari/604.1"; + + DeviceDetectionHashEngine engine = build(lite, NAMES); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + + // The indexes the engine was built with. + for (int i = 0; i < NAMES.size(); i++) { + assertEquals( + Integer.valueOf(i), + engine.getPropertyIndex(NAMES.get(i))); + } + + // Reload from the larger data file, whose property order differs. + engine.refreshData( + engine.getDataFiles().get(0).getIdentifier(), + Files.readAllBytes(bigger.toPath())); + + // The indexes have not moved. + for (int i = 0; i < NAMES.size(); i++) { + assertEquals( + "Index of '" + NAMES.get(i) + "' moved on reload.", + Integer.valueOf(i), + engine.getPropertyIndex(NAMES.get(i))); + } + + // Every name still reads its own property, and the typed read + // agrees with the read by name. + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + DeviceData device = data.get(DeviceData.class); + AspectPropertyValue typed = device.getIsMobile(); + AspectPropertyValue byName = + (AspectPropertyValue) device.get("IsMobile"); + assertEquals(typed.hasValue(), byName.hasValue()); + if (typed.hasValue()) { + assertEquals(typed.getValue(), byName.getValue()); + } + for (String name : NAMES) { + assertNotNull( + "Expected a result for '" + name + "'.", + device.get(name)); + } + } + } + } + + /** + * A name the engine was built with keeps its index after a reload from a + * data file that lacks the property. A read by index then gives no value, + * as a read by name does, rather than an error, and the other names still + * read their own properties. + */ + @Test + public void PropertyIndex_NameTheReloadedFileLacks() throws Exception { + File lite = dataFile(LITE_HASH_DATA_FILE_PATH); + File bigger = dataFile(TAC_HASH_DATA_FILE_NAME); + String missing = nameOnlyIn(bigger, lite); + assumeTrue("The TAC file has no property the Lite file lacks.", + missing != null); + List names = Arrays.asList(missing, "IsMobile"); + + DeviceDetectionHashEngine engine = build(bigger, names); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + engine.refreshData( + engine.getDataFiles().get(0).getIdentifier(), + Files.readAllBytes(lite.toPath())); + + assertEquals( + "Index of '" + missing + "' moved on reload.", + Integer.valueOf(0), + engine.getPropertyIndex(missing)); + assertEquals( + Integer.valueOf(1), + engine.getPropertyIndex("IsMobile")); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + DeviceDataHash device = + (DeviceDataHash) data.get(DeviceData.class); + AspectPropertyValue> values = + device.getValues(missing); + assertFalse( + "'" + missing + "' has a value from a file without it.", + values.hasValue()); + assertNotNull(values.getNoValueMessage()); + assertTrue("IsMobile has no value after the reload.", + device.getIsMobile().hasValue()); + assertEquals(Boolean.TRUE, device.getIsMobile().getValue()); + } + } + } + + /** + * The first property name, in name order, that one data file has and the + * other lacks, or null if there is none. + */ + private static String nameOnlyIn(File has, File lacks) throws Exception { + List lacking = propertyNames(lacks); + for (String name : propertyNames(has)) { + if (lacking.contains(name) == false) { + return name; + } + } + return null; + } + + private static List propertyNames(File file) throws Exception { + List names = new ArrayList<>(); + try (DeviceDetectionHashEngine engine = build(file, null)) { + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + names.add(property.getName()); + } + } + Collections.sort(names); + return names; + } + + /** + * Reads IsMobile {@link #READS} times by name and {@link #READS} times by + * index from the same native results, and {@link #READS} times through + * the typed accessor, which is the whole path a caller takes, and prints + * the median nanoseconds of one read over {@link #ROUNDS} rounds after a + * warm-up. Every read must give the same value. The times are printed + * rather than asserted, because runners vary. + */ + @Test + public void PropertyIndex_ReadByIndexAgainstByName() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + DeviceDetectionHashEngine engine = build(file, NAMES); + int index = engine.getPropertyIndex("IsMobile"); + ConfigHashSwig config = new ConfigHashSwig(); + config.setHighPerformance(); + RequiredPropertiesConfigSwig required; + try (VectorStringSwig names = new VectorStringSwig()) { + names.addAll(NAMES); + required = new RequiredPropertiesConfigSwig(names); + } + EngineHashSwig nativeEngine = + new EngineHashSwig(file.getAbsolutePath(), config, required); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData(); + ResultsHashSwig results = nativeEngine.process(MOBILE)) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + DeviceData device = data.get(DeviceData.class); + + boolean expected = readByName(results); + assertEquals(expected, readByIndex(results, index)); + assertEquals(expected, (boolean) device.getIsMobile().getValue()); + + List byName = new ArrayList<>(); + List byIndex = new ArrayList<>(); + List typed = new ArrayList<>(); + int different = 0; + for (int round = 0; round <= ROUNDS; round++) { + long start = System.nanoTime(); + for (int i = 0; i < READS; i++) { + if (readByName(results) != expected) { + different++; + } + } + long nameNanos = System.nanoTime() - start; + start = System.nanoTime(); + for (int i = 0; i < READS; i++) { + if (readByIndex(results, index) != expected) { + different++; + } + } + long indexNanos = System.nanoTime() - start; + start = System.nanoTime(); + for (int i = 0; i < READS; i++) { + if (device.getIsMobile().getValue() != expected) { + different++; + } + } + long typedNanos = System.nanoTime() - start; + if (round > 0) { + byName.add(nameNanos / READS); + byIndex.add(indexNanos / READS); + typed.add(typedNanos / READS); + } + } + assertEquals("Every read gives the same value", 0, different); + System.out.println(String.format( + "READ COST IsMobile reads=%d rounds=%d byNameNs=%s " + + "byIndexNs=%s typedNs=%s java=%s os=%s", + READS, ROUNDS, spread(byName), spread(byIndex), + spread(typed), System.getProperty("java.version"), + System.getProperty("os.name"))); + } + finally { + nativeEngine.delete(); + required.delete(); + config.delete(); + } + } + + /** + * Reads IsMobile by name through the overload that takes the name's + * bytes, which is how a read goes when the engine has no index for it. + */ + private static boolean readByName(ResultsHashSwig results) { + try (BoolValueSwig value = + results.getValueAsBool(Swig.asBytes("IsMobile"))) { + return value.getValue(); + } + } + + private static boolean readByIndex(ResultsHashSwig results, int index) { + try (BoolValueSwig value = results.getValueAsBool(index)) { + return value.getValue(); + } + } + + /** + * The lowest, median and highest, in that order. + */ + private static String spread(List values) { + List sorted = new ArrayList<>(values); + Collections.sort(sorted); + return sorted.get(0) + "/" + sorted.get(sorted.size() / 2) + "/" + + sorted.get(sorted.size() - 1); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index 6baf8af0e..a93045407 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -68,6 +68,12 @@ public class IPIntelligenceDataHashDefault */ private final List resultsList = new ArrayList<>(); + /** + * The engine which created this instance, used to turn a property name + * into the required property index the engine was built with. + */ + private final IPIntelligenceOnPremiseEngine ipiEngine; + /** * Constructs a new instance. * @param logger used for logging @@ -83,6 +89,18 @@ public class IPIntelligenceDataHashDefault IPIntelligenceOnPremiseEngine engine, MissingPropertyService missingPropertyService) { super(logger, flowData, engine, missingPropertyService); + this.ipiEngine = engine; + } + + /** + * Get the required property index for a property name, or null when the + * engine was not built with that name, so the read falls back to the name. + * @param propertyName name of the property + * @return required property index, or null + */ + private Integer indexOf(String propertyName) { + return ipiEngine == null ? + null : ipiEngine.getPropertyIndex(propertyName); } /** @@ -184,10 +202,13 @@ public AspectPropertyValue> getValues(String propertyName, return null; } Class valueType = parametrizedTypes[2]; + Integer index = indexOf(propertyName); if (valueType.equals(Boolean.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedBoolList(propertyName), + () -> index == null ? + results.getValuesAsWeightedBoolList(propertyName) : + results.getValuesAsWeightedBoolList(index), WeightedBoolListValueSwig::hasValue, WeightedBoolListValueSwig::getNoValueMessage, WeightedBoolListValueSwig::getValue, @@ -196,7 +217,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(Double.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedDoubleList(propertyName), + () -> index == null ? + results.getValuesAsWeightedDoubleList(propertyName) : + results.getValuesAsWeightedDoubleList(index), WeightedDoubleListValueSwig::hasValue, WeightedDoubleListValueSwig::getNoValueMessage, WeightedDoubleListValueSwig::getValue, @@ -205,7 +228,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(Float.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedDoubleList(propertyName), + () -> index == null ? + results.getValuesAsWeightedDoubleList(propertyName) : + results.getValuesAsWeightedDoubleList(index), WeightedDoubleListValueSwig::hasValue, WeightedDoubleListValueSwig::getNoValueMessage, WeightedDoubleListValueSwig::getValue, @@ -214,7 +239,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(Integer.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedIntegerList(propertyName), + () -> index == null ? + results.getValuesAsWeightedIntegerList(propertyName) : + results.getValuesAsWeightedIntegerList(index), WeightedIntListValueSwig::hasValue, WeightedIntListValueSwig::getNoValueMessage, WeightedIntListValueSwig::getValue, @@ -225,7 +252,9 @@ public AspectPropertyValue> getValues(String propertyName, || valueType.equals(Inet6Address.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedStringList(propertyName), + () -> index == null ? + results.getValuesAsWeightedStringList(propertyName) : + results.getValuesAsWeightedStringList(index), WeightedStringListValueSwig::hasValue, WeightedStringListValueSwig::getNoValueMessage, WeightedStringListValueSwig::getValue, @@ -240,7 +269,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(String.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedUTF8StringList(propertyName), + () -> index == null ? + results.getValuesAsWeightedUTF8StringList(propertyName) : + results.getValuesAsWeightedUTF8StringList(index), WeightedUTF8StringListValueSwig::hasValue, WeightedUTF8StringListValueSwig::getNoValueMessage, WeightedUTF8StringListValueSwig::getValue, @@ -267,9 +298,10 @@ protected AspectPropertyValue getValueAsString(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - - try (StringValueSwig value = results.getValueAsString( - propertyName)) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(propertyName) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -288,8 +320,10 @@ protected AspectPropertyValue getValueAsJavaScript( new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (StringValueSwig value = results.getValueAsString( - propertyName)) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(propertyName) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(new JavaScript(value.getValue())); } @@ -307,8 +341,10 @@ protected AspectPropertyValue getValueAsInteger( AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (IntegerValueSwig value = results.getValueAsInteger( - propertyName)) { + Integer index = indexOf(propertyName); + try (IntegerValueSwig value = index == null ? + results.getValueAsInteger(propertyName) : + results.getValueAsInteger(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -325,8 +361,10 @@ protected AspectPropertyValue getValueAsBool(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (BoolValueSwig value = results.getValueAsBool( - propertyName)) { + Integer index = indexOf(propertyName); + try (BoolValueSwig value = index == null ? + results.getValueAsBool(propertyName) : + results.getValueAsBool(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -343,8 +381,10 @@ protected AspectPropertyValue getValueAsDouble(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (DoubleValueSwig value = results.getValueAsDouble( - propertyName)) { + Integer index = indexOf(propertyName); + try (DoubleValueSwig value = index == null ? + results.getValueAsDouble(propertyName) : + results.getValueAsDouble(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java index a0779d7dc..053521d41 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java @@ -62,6 +62,17 @@ public class IPIntelligenceOnPremiseEngine private volatile boolean propertiesPopulated = false; private final Random rand = new Random(); + /** + * Case insensitive map from each property name the engine was built with + * to its required property index, which is the place of the name in that + * list, the first where a name is given more than once, as the native + * engine gives it. Made from the names given at build and never from the + * data set, so an index keeps naming the same property after a reload + * from a data file whose property order differs. Empty when the engine + * was built with no names, in which case reads go by name. + */ + private final Map propertyIndexes; + /** * Construct a new instance of the {@link IPIntelligenceOnPremiseEngine}. * @param logger logger instance to use for logging @@ -70,6 +81,8 @@ public class IPIntelligenceOnPremiseEngine * @param properties native required properties configuration which define * the properties which the engine should be initialised * with + * @param propertyNames the property names {@code properties} was made + * from, in the same order * @param ipiDataFactory the factory to use when creating a * {@link IPIntelligenceDataHash} instance * @param tempDataFileDir the file where a temporary data file copy @@ -80,11 +93,13 @@ public class IPIntelligenceOnPremiseEngine AspectEngineDataFile dataFile, ConfigIpiSwig config, RequiredPropertiesConfigSwig properties, + List propertyNames, ElementDataFactory ipiDataFactory, String tempDataFileDir) { super(logger, ipiDataFactory, tempDataFileDir); this.config = config; this.propertiesConfigSwig = properties; + this.propertyIndexes = indexesOf(propertyNames); addDataFile(dataFile); } @@ -113,6 +128,37 @@ public MetaDataSwig getMetaData() { return engine.getMetaData(); } + /** + * Get the required property index for a property name, or null when the + * engine was not built with that name and the read has to go by name. + * @param propertyName name of the property + * @return required property index, or null + */ + Integer getPropertyIndex(String propertyName) { + return propertyIndexes.get(propertyName); + } + + /** + * Map each name to its place in the list, keeping the first place of a + * name given more than once in any letter case. + * @param names the property names in the order the engine is built with, + * or null + * @return unmodifiable case insensitive map from name to index + */ + private static Map indexesOf(List names) { + Map indexes = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + if (names != null) { + for (int i = 0; i < names.size(); i++) { + String name = names.get(i); + if (name != null && name.isEmpty() == false) { + indexes.putIfAbsent(name, i); + } + } + } + return Collections.unmodifiableMap(indexes); + } + @Override public List getProperties() { if (isFalse(propertiesPopulated)) { diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java index c403aee93..cf0b071fe 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java @@ -257,6 +257,7 @@ protected IPIntelligenceOnPremiseEngine newEngine(List properties) { dataFile, config, requiredProperties, + properties, new IpiDataFactory(loggerFactory), tempDir); } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java new file mode 100644 index 000000000..1aac78acd --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java @@ -0,0 +1,163 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.pipeline.engines.Constants; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for the property index rule on the IP Intelligence on-premise + * engine, which says that a property's index is its place in the list of + * names the engine was built with, that the index is fixed at build, and that + * every read by name goes through that index. + */ +public class PropertyIndexTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * Names in an order that is deliberately not the order the data file + * holds them in, so an index taken from the data file would be wrong. + */ + private static final List NAMES = Arrays.asList( + "RegisteredCountry", + "AccuracyRadiusMin", + "RegisteredName", + "Latitude"); + + private static File dataFile(String name) { + try { + return getFilePath(name); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private static IPIntelligenceOnPremiseEngine build( + File file, List names) throws Exception { + IPIntelligenceOnPremiseEngineBuilder builder = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.MaxPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + return builder.build(file.toString(), false); + } + + /** + * Every name the engine was built with has an index, and the index is the + * place of the name in the build time list, not its place in the data + * file. + */ + @Test + public void PropertyIndex_FromBuildTimeNames() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, NAMES)) { + for (int i = 0; i < NAMES.size(); i++) { + Integer index = engine.getPropertyIndex(NAMES.get(i)); + assertNotNull( + "Expected an index for '" + NAMES.get(i) + "'.", + index); + assertEquals( + "Expected '" + NAMES.get(i) + "' at index " + i + ".", + Integer.valueOf(i), + index); + } + } + } + + /** + * The lookup is case insensitive, because a caller may ask for a property + * in any casing. + */ + @Test + public void PropertyIndex_CaseInsensitive() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, NAMES)) { + assertEquals( + engine.getPropertyIndex("RegisteredName"), + engine.getPropertyIndex("registeredname")); + } + } + + /** + * A name given twice, in any letter case, has the index of its first + * place, as the native engine gives it. + */ + @Test + public void PropertyIndex_NameGivenTwiceHasFirstIndex() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, + Arrays.asList("RegisteredName", "Latitude", "registeredname"))) { + assertEquals( + Integer.valueOf(0), + engine.getPropertyIndex("RegisteredName")); + assertEquals( + Integer.valueOf(1), + engine.getPropertyIndex("Latitude")); + } + } + + /** + * A name the engine was not built with has no index, so the read goes by + * name. + */ + @Test + public void PropertyIndex_UnknownNameHasNoIndex() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, NAMES)) { + assertNull(engine.getPropertyIndex("NotAPropertyAtAll")); + } + } + + /** + * An engine built with no names has no indexes, so every read goes by + * name. + */ + @Test + public void PropertyIndex_NoNamesReadsByName() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, null)) { + assertNull(engine.getPropertyIndex("RegisteredName")); + } + } +} From d4d4dc60f2bbb4f03889df29bbb2f9883a03219f Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Wed, 11 Nov 2026 14:55:01 +0000 Subject: [PATCH 75/95] Make every cloud test call the live service and fail when it cannot (#59) * TEST: Create the 51Did for another address as a licensed caller anIdentifierForAnotherAddress_ReportsEveryFactor asked for a 51Did for a stated address with a resource key alone, and expected a mismatch when the same connection then presented it. The cloud service takes the address of such a caller from the connection, whatever address the caller states, so the verdict is verified and the test fails wherever a licence key is set. The test now creates the identifier as a licensed caller, the caller whose stated address the service uses, with both keys in the form body. The verdict is then a mismatch that names all nine factors and reports the browser address as one that differs. * TEST: Make every cloud test call the live service and fail when it cannot Tests named as cloud tests skipped themselves when no resource key was set, and the example test turned any failure into a skip, so a job that never reached the cloud service still reported success. The resource key the workflows pass never reached Maven either, because ci/run-unit-tests.ps1 took no Keys parameter. - DidClientLiveTests fails with a message naming every variable tried and the fodid product the key must carry, instead of four assume calls and an assumeNoException that swallowed a missing creator context. - The cloud engine example test no longer catches and skips, and the example now pairs a 51Degrees resource key with the 51Degrees cloud endpoint, so a key found in the environment is sent to the service that issued it. - New CloudRequestEngineLiveTests, because every other test in that module answers the engine from a mock and nothing in this repository called the live service through CloudRequestEngine. - ci/run-unit-tests.ps1 takes Keys and passes the key to Maven, and the workflows now supply the key entitled to the bespoke products. - The 51Did workflow runs against the key carrying the fodid product and fails when no key is available. Carried into 51degrees-java from pipeline-java, where it was opened as a draft. The nightly, publish and pull request workflows here already pass the bespoke key, and ci/run-unit-tests.ps1 already hands it to Maven as TestResourceKey, which the tests read, so those parts are not needed. The 51Did workflow keeps its loop over the keys with a paired licence key, drops the basic key, which does not carry fodid, and fails when no key is available. --------- Co-authored-by: James Rosewell --- .github/workflows/cloud-51did.yml | 19 +- .../CloudRequestEngineLiveTests.java | 139 ++++++++++++++ .../flowelements/LiveCloudKeys.java | 95 ++++++++++ .../developerexamples/cloudengine/Main.java | 60 ++++++- .../cloudengine/ExampleTests.java | 29 ++- .../pipeline/did/DidClientLiveTests.java | 169 ++++++++++-------- .../fiftyone/pipeline/did/LiveCloudKeys.java | 114 ++++++++++++ .../engines/services/HttpClientTests.java | 11 ++ 8 files changed, 538 insertions(+), 98 deletions(-) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java create mode 100644 pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index 69731a8d9..e99cdcb13 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -3,6 +3,9 @@ name: Cloud 51Did Tests # Runs the live cloud 51Did integration test (pipeline/pipeline.did, # DidClientLiveTests) once for every _51DEGREES_RESOURCE_KEY* key passed in # below. Each run sets the single _51DEGREES_RESOURCE_KEY the test reads. +# Only keys that carry the fodid product are passed in, because the tests ask +# the cloud for a 51Did and fail, saying so, when a key gets an answer with no +# identifier in it. That is entitlement working rather than a fault. # # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly @@ -10,10 +13,10 @@ name: Cloud 51Did Tests # A new resource key therefore needs a line here as well as the organisation # secret. # -# The test skips itself when no key is set, so a run with no secrets is green -# and proves nothing. That is deliberate, because a pull request from a fork -# has no secrets, and it is the reason the loop below reports which key each -# run used. +# A run with no key available fails. A cloud test that passes without +# calling the cloud reports success having proved nothing, and that is worse +# than a red job. A pull request from a fork gets no secrets, so run this +# workflow from a branch in this repository. on: workflow_dispatch: @@ -51,7 +54,6 @@ jobs: - name: Cloud 51Did test for each resource key secret shell: pwsh env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} # Without a license key the tests that redeem a sealed verdict skip. @@ -63,8 +65,11 @@ jobs: $keys = Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } if (-not $keys) { - Write-Host 'No _51DEGREES_RESOURCE_KEY* secrets available; skipping.' - exit 0 + throw ('No _51DEGREES_RESOURCE_KEY* secret is available, so the ' + + 'live 51Did tests cannot call the cloud service. The keys ' + + 'carry the fodid product these tests need. A pull request ' + + 'from a fork gets no secrets, so run this workflow from a ' + + 'branch in this repository.') } $failed = $false foreach ($k in $keys) { diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java new file mode 100644 index 000000000..1ef246c77 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java @@ -0,0 +1,139 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Calls the live 51Degrees cloud service through the cloud request + * engine, end to end. + *

    + * Every other test in this module answers the engine from a mock, which + * checks the request the engine builds and the way it reads a reply, and + * proves nothing about the service on the other end. A field the service + * stops writing, a header it starts refusing or an endpoint that moves + * all leave those tests green. This one makes the three calls the engine + * really makes, which are accessible properties and evidence keys when it + * is built, and the JSON request when it processes, so a change in the + * service is seen here. + *

    + * {@code FOD_CLOUD_API_URL} points the engine at another host, which the + * builder reads for itself. + */ +public class CloudRequestEngineLiveTests { + + /** + * A real browser User-Agent, so the service has something to detect + * rather than a string it will report as unknown. + */ + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + @Test + public void build_Process_ReadsDeviceDataFromTheLiveService() + throws Exception { + String resourceKey = + LiveCloudKeys.find(LiveCloudKeys.RESOURCE_KEY_NAMES); + if (resourceKey == null) { + fail(LiveCloudKeys.noResourceKey()); + } + + CloudRequestEngineBuilder builder = new CloudRequestEngineBuilder( + LoggerFactory.getILoggerFactory(), new HttpClientDefault()) + .setResourceKey(resourceKey); + String licenceKey = + LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); + if (licenceKey != null) { + builder.setLicenseKey(licenceKey); + } + + // Building asks the service for its accessible properties and its + // evidence keys, so a service that cannot be reached, or a key it + // refuses, fails here. + CloudRequestEngine engine = builder.build(); + + Map products = + engine.getPublicProperties(); + assertNotNull(products, + "The live cloud service returned no accessible properties at " + + "all, which is the service or the key rather than this " + + "package."); + assertFalse(products.isEmpty(), + "The live cloud service listed no products for this resource " + + "key. A key carries the products its account holds, so this " + + "is an entitlement problem and not a fault in the service. " + + "Create a key at https://configure.51degrees.com."); + assertTrue(products.containsKey("device"), + "The live cloud service listed the products " + + String.join(", ", products.keySet()) + " for this resource " + + "key, and none of them is the device product this test asks " + + "for. That is the key not carrying the device product rather " + + "than a fault in the service."); + + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(engine) + .build()) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", USER_AGENT).process(); + + CloudRequestData requestData = data.getFromElement(engine); + assertNotNull(requestData, + "The cloud request engine put no data in the flow, so " + + "the request to the live cloud service produced " + + "nothing to read."); + String json = requestData.getJsonResponse(); + assertNotNull(json, + "The live cloud service returned no JSON body for a " + + "request carrying a User-Agent."); + + JSONObject answer = new JSONObject(json); + assertTrue(answer.has("device"), + "The live cloud service answered with " + json + + ", which carries no device element. The resource key " + + "lists the device product, so this is the service " + + "rather than entitlement."); + JSONObject device = answer.getJSONObject("device"); + assertTrue(device.length() > 0, + "The live cloud service returned an empty device " + + "element for a real browser User-Agent."); + } + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java new file mode 100644 index 000000000..55be783d3 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java @@ -0,0 +1,95 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +/** + * Where the tests that call the live 51Degrees cloud service read their + * resource key and licence key from, and what they say when there is + * none. + *

    + * Every name is tried as an environment variable first and then as a + * system property, so a developer can export one and a build can pass one + * on the command line with {@code -D}. There is no fallback to a skip, + * because a cloud test that quietly passes with no key reports success + * having called nothing. + */ +final class LiveCloudKeys { + + /** + * The resource key names, in the order they are tried. + */ + static final String[] RESOURCE_KEY_NAMES = { + "_51DEGREES_RESOURCE_KEY", + "RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "TestResourceKey", + "SUPER_RESOURCE_KEY", + }; + + /** + * The licence key names, in the order they are tried. A licence key + * is optional, because an account can hold its products on the + * resource key alone. + */ + static final String[] LICENCE_KEY_NAMES = { + "_51DEGREES_LICENSE_KEY", + "LICENSE_KEY", + "_51DEGREES_LICENSE_KEY_BESPOKE", + "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", + "TestLicenseKey", + }; + + private LiveCloudKeys() { + } + + /** + * The first value set under any of the given names, or null when none + * of them is set anywhere. + */ + static String find(String[] names) { + for (String name : names) { + String value = System.getenv(name); + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + if (value != null && value.trim().isEmpty() == false) { + return value.trim(); + } + } + return null; + } + + /** + * What to fail with when no resource key is set, naming every place + * that was looked and what the key has to be entitled to. + */ + static String noResourceKey() { + return "This test calls the live 51Degrees cloud service, so it " + + "needs a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", RESOURCE_KEY_NAMES) + " is set, " + + "as an environment variable or as a system property, tried in " + + "that order. The key has to carry the device product, because " + + "the test asks the service for device detection properties. " + + "Create a key at https://configure.51degrees.com."; + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 5b7c1b325..a1011c612 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -46,6 +46,13 @@ public class Main { "https://51degrees.com/starsign/api/"; private static final String STAR_SIGN_RESOURCE_KEY = "cloudexample"; + /** + * The 51Degrees cloud service, used when a 51Degrees resource key is + * set and no other endpoint is given. + */ + private static final String FIFTYONE_CLOUD_ENDPOINT = + "https://cloud.51degrees.com/api/v4/"; + /** * The cloud endpoint, taken from FOD_CLOUD_API_URL when that is set, * which is the variable every 51Degrees cloud example honours and the @@ -61,7 +68,14 @@ public class Main { static String endpoint() { String value = env("FOD_CLOUD_API_URL"); if (value == null) { - return STAR_SIGN_ENDPOINT; + // A resource key and an endpoint travel together, because a + // key is only known to the service that issued it. So the + // 51Degrees cloud is used whenever a 51Degrees resource key + // was found, and the star sign service, with the star sign + // key, only when there was none. + return configuredResourceKey() == null + ? STAR_SIGN_ENDPOINT + : FIFTYONE_CLOUD_ENDPOINT; } while (value.endsWith("/")) { value = value.substring(0, value.length() - 1); @@ -70,21 +84,49 @@ static String endpoint() { } /** - * The resource key from _51DEGREES_RESOURCE_KEY, or the older - * RESOURCE_KEY, so the example can be pointed at a host whose keys - * are its own. Otherwise the key of the star sign service. + * The names a 51Degrees resource key is read from, in the order they + * are tried. Each is read as an environment variable first and then + * as a system property, so a developer can export one and a build + * can pass one on the command line with -D. */ - static String resourceKey() { - String value = env("_51DEGREES_RESOURCE_KEY"); - if (value == null) { - value = env("RESOURCE_KEY"); + static final String[] RESOURCE_KEY_NAMES = { + "_51DEGREES_RESOURCE_KEY", + "RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "TestResourceKey", + "SUPER_RESOURCE_KEY", + }; + + /** + * The 51Degrees resource key that was found, or null when none of the + * names above is set anywhere. + */ + static String configuredResourceKey() { + for (String name : RESOURCE_KEY_NAMES) { + String value = env(name); + if (value != null) { + return value; + } } + return null; + } + + /** + * The resource key the example runs with, which is a 51Degrees key + * where one was found and the key of the star sign service where + * none was. + */ + static String resourceKey() { + String value = configuredResourceKey(); return value == null ? STAR_SIGN_RESOURCE_KEY : value; } static String env(String name) { String value = System.getenv(name); - return value == null || value.trim().isEmpty() ? null : value; + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + return value == null || value.trim().isEmpty() ? null : value.trim(); } public static class Example { diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java index 36c491b54..626622038 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java @@ -25,19 +25,32 @@ import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assertions.fail; +/** + * Runs the cloud engine example against the live cloud service. + *

    + * Nothing is caught here. The example builds a cloud request engine, + * which asks the service for its accessible properties and its evidence + * keys, and then processes evidence through it, so any failure to reach + * the service, or any answer the pipeline cannot use, has to be seen. + * Catching and skipping would report success for a run that never + * reached the service at all. + */ public class ExampleTests { @Test public void SimpleCloudEngine_Test() throws Exception { - try { - new Main.Example().run(); - } - catch (Exception e) { - // Only assume, as this example will fail if there is no internet - // connection. - assumeFalse(true, "Exception thrown"); + if (Main.configuredResourceKey() == null) { + fail("This test runs the cloud engine example against the live " + + "51Degrees cloud service, so it needs a resource key. None " + + "of " + String.join(", ", Main.RESOURCE_KEY_NAMES) + " is " + + "set, as an environment variable or as a system property, " + + "tried in that order. Any key works here, because the " + + "example reports a product the key does not carry rather " + + "than failing on it. Create a key at " + + "https://configure.51degrees.com."); } + new Main.Example().run(); } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java index 6ea863aa2..a91f840a1 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java @@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -41,15 +42,21 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** - * Tests against the live cloud, skipped unless {@code _51DEGREES_RESOURCE_KEY} - * (or the older {@code RESOURCE_KEY}) is set. {@code FOD_CLOUD_API_URL} - * points them at another host, and {@code _51DEGREES_LICENSE_KEY} (or - * {@code LICENSE_KEY}) supplies the licence key where the account holds - * one. Each test creates a 51Did through the cloud {@code json} endpoint, - * which is one use against the resource key, plus one for each cloud call - * it then makes. + * Tests against the live cloud service. They need a resource key, read + * from the names {@link LiveCloudKeys} lists, and they fail rather than + * skip when there is none, because a run that called nothing must not + * report success. {@code FOD_CLOUD_API_URL} points them at another host, + * and a licence key is taken from the names {@link LiveCloudKeys} lists + * where the account holds one. Each test creates a 51Did through the + * cloud {@code json} endpoint, which is one use against the resource key, + * plus one for each cloud call it then makes. + *

    + * The resource key has to carry the fodid product. A key without it gets + * an answer from the service with no identifier in it, which these tests + * report as an entitlement failure naming the product. */ public class DidClientLiveTests { @@ -59,12 +66,12 @@ public class DidClientLiveTests { @Before public void init() { - resourceKey = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); - Assume.assumeTrue( - "Set _51DEGREES_RESOURCE_KEY to run the live 51Did cloud tests.", - resourceKey != null); - licenceKey = env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); - client = new DidClient(resourceKey, licenceKey); + resourceKey = LiveCloudKeys.find(LiveCloudKeys.RESOURCE_KEY_NAMES); + if (resourceKey == null) { + fail(LiveCloudKeys.noResourceKey()); + } + client = new DidClient( + resourceKey, LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES)); } @Test @@ -88,10 +95,17 @@ public void redeem_GarbageResult_IsUnreadable() throws Exception { .join(); } catch (CompletionException reported) { if (reported.getCause() instanceof DidNotSupportedException) { - Assume.assumeNoException( - "The host does not offer the creator context.", - reported.getCause()); - return; + // Reported rather than skipped. The redeem endpoint is + // part of the cloud service these tests exist to check, + // so a host that does not offer it is a real result and + // has to be seen. + throw new AssertionError( + "The cloud service at " + client.getEndpoint() + + " answered that it does not offer the creator " + + "context, so the redeem endpoint could not be " + + "called. Point FOD_CLOUD_API_URL at a service that " + + "offers it, or leave it unset for the 51Degrees " + + "cloud.", reported.getCause()); } throw reported; } @@ -173,13 +187,14 @@ public void verifyFullThenRedeemWithTheLicence_ReadsTheVerdict() /** * Every factor the service compares, read by name. The identifier is - * created for a stated address rather than for the connection that - * asked, so when the same connection then presents it the address the - * identifier carries is not the one being checked and the verdict is a - * mismatch. That is the one mismatch a test can arrange without a - * browser, and a mismatch is the only verdict that carries the - * breakdown, since a verified verdict matched everywhere and has - * nothing to diagnose. + * created by a licensed caller that states another address, so when + * this connection then presents it as a page the address it carries is + * not the one being checked and the verdict is a mismatch. That is the + * one mismatch a test can arrange without a browser. + *

    + * The address has to come from a licensed caller. The service takes + * the address of a caller that sends a resource key alone from the + * connection, whatever address that caller states. *

    * The names are written out rather than taken from the package, * because a list the package supplied would agree with itself whatever @@ -192,7 +207,7 @@ public void anIdentifierForAnotherAddress_ReportsEveryFactor() "Set _51DEGREES_LICENSE_KEY to redeem a sealed result.", client.hasLicenceKey()); - List identifiers = identifiersFor( + List identifiers = identifiersCreatedFor( "id.usage", "standard", CREATED_FOR); Assume.assumeTrue( "This resource key returned no marketing 51Did, so there was " @@ -273,9 +288,7 @@ private String verifyFull(String fodId) throws Exception { */ private FodId create() throws Exception { // id.usage is required. Without it the service takes the caller - // as not having asked for a 51Did at all and creates none, so this - // skipped on every run and the job reported success having tested - // nothing. + // as not having asked for a 51Did at all and creates none. String url = client.getEndpoint() + "json?resource=" + DidClient.encode(resourceKey) + "&id.usage=non-marketing&values=FODiD.IdProbGlobal"; @@ -291,9 +304,11 @@ private FodId create() throws Exception { String value = fodid == null ? null : fodid.optString("idprobglobal", null); - Assume.assumeTrue( - "The resource key does not return FODiD.IdProbGlobal.", - value != null); + if (value == null) { + fail(LiveCloudKeys.notEntitled( + "a request for FODiD.IdProbGlobal with " + + "id.usage=non-marketing")); + } return FodId.fromBase64(value); } @@ -314,15 +329,6 @@ private static String readAll(InputStream stream) throws IOException { } } - private static String env(String... names) { - for (String name : names) { - String value = System.getenv(name); - if (value != null && value.trim().isEmpty() == false) { - return value.trim(); - } - } - return null; - } /** * The versioned Model Terms for Marketing document a marketing 51Did is * created under. @@ -339,33 +345,53 @@ private static String env(String... names) { * Asks the {@code json} endpoint for a 51Did with the given query * parameter and returns every identifier it answered with. An empty * list means the resource key is not entitled to that usage, which the - * caller reports rather than fails. + * caller fails on, naming the product the key is missing. */ private List identifiersFor(String name, String value) throws Exception { - return identifiersFor(name, value, null); + String url = client.getEndpoint() + "json?resource=" + + DidClient.encode(resourceKey) + + "&" + name + "=" + DidClient.encode(value) + + "&values=FODiD.IdProbGlobal&values=FODiD.IdProbLic"; + HttpURLConnection connection = (HttpURLConnection) + URI.create(url).toURL().openConnection(); + connection.setRequestProperty("User-Agent", "pipeline.did live test"); + return readIdentifiers(connection); } /** - * As above, creating from the given address where one is given, so a - * test can create from one address and present the identifier from - * another. The service takes a supplied address only from a caller - * presenting a licence key, a server acting for a visitor, so the - * licence key goes with it. + * As above, asked by a licensed caller that states the address the + * identifier is created for, so a test can create for one address and + * present the identifier from another. Both keys travel in the form + * body, because a query string is written to access logs. */ - private List identifiersFor( + private List identifiersCreatedFor( String name, String value, String clientIp) throws Exception { - String url = client.getEndpoint() + "json?resource=" - + DidClient.encode(resourceKey) + String form = "resource=" + DidClient.encode(resourceKey) + + "&license=" + DidClient.encode(licenceKey) + "&" + name + "=" + DidClient.encode(value) - + (clientIp == null - ? "" - : "&client-ip=" + DidClient.encode(clientIp) - + "&license=" + DidClient.encode(licenceKey)) + + "&client-ip=" + DidClient.encode(clientIp) + "&values=FODiD.IdProbGlobal&values=FODiD.IdProbLic"; HttpURLConnection connection = (HttpURLConnection) - URI.create(url).toURL().openConnection(); + URI.create(client.getEndpoint() + "json").toURL() + .openConnection(); + connection.setRequestMethod("POST"); connection.setRequestProperty("User-Agent", "pipeline.did live test"); + connection.setRequestProperty("Content-Type", + "application/x-www-form-urlencoded; charset=utf-8"); + connection.setDoOutput(true); + try (OutputStream out = connection.getOutputStream()) { + out.write(form.getBytes(StandardCharsets.UTF_8)); + } + return readIdentifiers(connection); + } + + /** + * Every identifier in the answer to a request sent to the {@code json} + * endpoint. + */ + private static List readIdentifiers(HttpURLConnection connection) + throws Exception { int status = connection.getResponseCode(); String body = readAll(status >= 400 ? connection.getErrorStream() @@ -437,9 +463,7 @@ public void everyUsageReadsBackTheTermsAndFlagsTheServiceWrote() List identifiers = identifiersFor("id.usage", name); if (identifiers.isEmpty()) { - System.err.println("id.usage=" + name + ": no identifier " - + "returned, so this key is not entitled to that usage."); - continue; + fail(LiveCloudKeys.notEntitled("id.usage=" + name)); } for (int i = 0; i < identifiers.size(); i++) { assertAligned(name + "[" + i + "]", identifiers.get(i), @@ -450,14 +474,13 @@ public void everyUsageReadsBackTheTermsAndFlagsTheServiceWrote() } } - // Reported as a skip rather than written to stderr, because - // surefire captures test output and prints only the counts, so a - // run that proved nothing would otherwise look identical to one - // that proved everything. - Assume.assumeTrue( - "This resource key returned no marketing 51Did, so no terms " - + "address was read and this run did not prove it. Use a key " - + "entitled to the standard or personalized usage.", + // A run that read no marketing identifier read no terms address + // either, so it proved nothing and must not report success. + assertTrue( + "The live cloud service returned no marketing 51Did, so no " + + "terms address was read. The resource key has to carry the " + + "fodid product and be entitled to the standard or " + + "personalized usage.", checked > 0); } @@ -495,10 +518,8 @@ public void consentStringSetsTheUsageIsIndirectBit() throws Exception { // would prove the opposite of what it says. List identifiers = identifiersFor("tcstring", one[0]); if (identifiers.isEmpty()) { - System.err.println("consent string granting " + usage - + ": no identifier returned, so this key is not entitled " - + "to that marketing usage."); - continue; + fail(LiveCloudKeys.notEntitled( + "a consent string granting " + usage)); } for (int i = 0; i < identifiers.size(); i++) { // A consent string granting a marketing usage produces a @@ -511,10 +532,10 @@ public void consentStringSetsTheUsageIsIndirectBit() throws Exception { } // Same reasoning as the usage test above. - Assume.assumeTrue( - "This resource key returned no identifier for either consent " - + "string, so the usage is indirect bit was never read and " - + "this run did not prove it.", + assertTrue( + "The live cloud service returned no identifier for either " + + "consent string, so the usage-from-consent bit was never " + + "read and this run proved nothing.", proven > 0); } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java new file mode 100644 index 000000000..ba5efdc30 --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java @@ -0,0 +1,114 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +/** + * Where the tests that call the live 51Degrees cloud service read their + * resource key and licence key from, and what they say when there is + * none. + *

    + * Every name is tried as an environment variable first and then as a + * system property, so a developer can export one and a build can pass one + * on the command line with {@code -D}. There is no fallback to a skip, + * because a cloud test that quietly passes with no key reports success + * having called nothing. + */ +final class LiveCloudKeys { + + /** + * The resource key names, in the order they are tried. + *

    + * {@code _51DEGREES_RESOURCE_KEY} is first because the workflow that + * runs these tests once for every key the organisation holds sets + * that single name to each key in turn, and a name later in this list + * winning would make every one of those runs use the same key. + */ + static final String[] RESOURCE_KEY_NAMES = { + "_51DEGREES_RESOURCE_KEY", + "RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "TestResourceKey", + "SUPER_RESOURCE_KEY", + }; + + /** + * The licence key names, in the order they are tried. A licence key + * is optional, because an account can hold its products on the + * resource key alone. + */ + static final String[] LICENCE_KEY_NAMES = { + "_51DEGREES_LICENSE_KEY", + "LICENSE_KEY", + "_51DEGREES_LICENSE_KEY_BESPOKE", + "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", + "TestLicenseKey", + }; + + private LiveCloudKeys() { + } + + /** + * The first value set under any of the given names, or null when none + * of them is set anywhere. + */ + static String find(String[] names) { + for (String name : names) { + String value = System.getenv(name); + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + if (value != null && value.trim().isEmpty() == false) { + return value.trim(); + } + } + return null; + } + + /** + * What to fail with when no resource key is set, naming every place + * that was looked and what the key has to be entitled to. + */ + static String noResourceKey() { + return "These tests call the live 51Degrees cloud service, so they " + + "need a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", RESOURCE_KEY_NAMES) + " is set, " + + "as an environment variable or as a system property, tried in " + + "that order. The key has to carry the fodid product, because " + + "these tests ask the cloud for FODiD.IdProbGlobal and " + + "FODiD.IdProbLic. Create a key at " + + "https://configure.51degrees.com."; + } + + /** + * What to fail with when the cloud answered but returned no 51Did, + * which is the resource key not carrying the product rather than the + * service being broken. + */ + static String notEntitled(String detail) { + return "The live 51Degrees cloud service answered, but returned no " + + "51Did for " + detail + ". That is the resource key not " + + "carrying the fodid product rather than a fault in the " + + "service or in this package. Use a key entitled to fodid, " + + "from https://configure.51degrees.com."; + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java index f09b13400..e7cf616f5 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java @@ -35,6 +35,17 @@ import org.junit.Before; import org.junit.Test; +/** + * Calls the live 51Degrees cloud service, and fails rather than skips when + * it cannot be reached, because a test of an HTTP client that passes + * without making a request proves nothing. + *

    + * The resource key below is fixed on purpose and must not be replaced with + * a key from the environment. It is restricted to the 51degrees.com + * origin, which is what gives the two answers these tests need from one + * address, being 401 with no Origin header and 200 with one. A key without + * that restriction would answer 200 to both. + */ public class HttpClientTests { private HttpClient client; From a2d6cddcf8c01dd47288188e09def330775ba584 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 14:56:19 +0000 Subject: [PATCH 76/95] Share a repeated evidence set at most once in each interval (#61) * Test that a repeat is shared at most once in each interval The usage sharing tracker stores the time evidence was first shared and never moves it on. A repeat is held back for the interval after the first share, and after that it is shared on every request until the cache drops it. The specification describes the interval as a sliding window, which Python and Node follow by storing a new time whenever they share a repeat. The test shares evidence, waits past the interval, shares the repeat, and expects a repeat straight after that not to be shared. It fails until the tracker stores the time a repeat is shared. * Share a repeated evidence set at most once in each interval The tracker now stores the time it shares a repeat in the cached Date, so the interval starts again from that share. A repeat is shared at most once in each interval, which is what the specification's sliding window describes and what Python and Node already do. The check and the update are made under the Date's lock, so two requests cannot both share the same repeat. * Give the tracker tests that do not test expiry an interval no runner reaches The tests shared a 50 millisecond interval, so on a slow runner two requests that should count as one repeat could be more than 50 milliseconds apart and both be tracked. ShareUsageTracker_Session_DoNotTrack failed that way on MacOS_X64_Java_11, with the test taking 2.4 seconds. Only the test that sleeps past the interval to test expiry keeps the short one. ShareUsageTracker_Session_Track no longer sleeps, so the session cookie alone keeps its two requests apart. * Wait past the interval with a margin in the repeat evidence test --- .../fiftyone/trackers/ShareUsageTracker.java | 21 +++++-- .../trackers/ShareUsageTrackerTests.java | 63 ++++++++++++++++--- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index c6082300e..c1bd3bb09 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -38,7 +38,9 @@ /** * A tracker used by share usage to attempt to avoid repeatedly sending data - * relating to the same user session. + * relating to the same user session. Evidence that matches an entry is shared + * again once the interval has passed since it was last shared, so a repeat is + * shared at most once in each interval. * @see Specification */ public class ShareUsageTracker extends TrackerBase { @@ -64,7 +66,8 @@ public class ShareUsageTracker extends TrackerBase { * Construct a new instance. * @param configuration the cache configuration to use when building the * internal cache used by this tracker - * @param intervalMillis interval between matched in millis + * @param intervalMillis how long after evidence is shared a repeat of it is + * not shared, in milliseconds * @param filter the {@link EvidenceKeyFilter} that defines the evidence * values to use when creating a key from an {@link FlowData} */ @@ -83,14 +86,22 @@ public ShareUsageTracker( @Override protected Date newValue(FlowData data) { - // Stored meta-data values for this tracker are always true. + // The time the new evidence is shared. return new Date(); } @Override protected boolean match(FlowData data, Date value) { - Date now = new Date(); - return value.getTime() <= (now.getTime() - interval); + // Sharing a repeat stores the time in the cached value, so the next + // interval starts from this share. + long now = System.currentTimeMillis(); + synchronized (value) { + if (value.getTime() <= now - interval) { + value.setTime(now); + return true; + } + return false; + } } @Override diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java index dff24cbae..eaf8670d8 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java @@ -40,12 +40,29 @@ import static fiftyone.pipeline.core.Constants.*; import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.when; public class ShareUsageTrackerTests { - private long interval; + /** + * An interval no run of these tests comes near, for the tests that do + * not test expiry, so a slow runner cannot make a repeat count as new. + */ + private static final long NO_EXPIRY_INTERVAL_MILLIS = 60 * 60 * 1000; + + /** + * A short interval, for a test that sleeps past it to test expiry. + */ + private static final long EXPIRY_INTERVAL_MILLIS = 50; + + /** + * Added to the wait for an interval to pass, so that a sleep that returns + * early or a slow clock cannot leave a repeat inside the interval. + */ + private static final long EXPIRY_MARGIN_MILLIS = 250; private EvidenceKeyFilter evidenceKeyFilter; @@ -62,7 +79,6 @@ public void Init() { includedQueryStringParameters = new ArrayList<>(); sessionCookieName = DEFAULT_SESSION_COOKIE_NAME; - interval = 50; evidenceKeyFilter = new EvidenceKeyFilterShareUsage( blockedHttpHeaders, includedQueryStringParameters, @@ -73,7 +89,7 @@ public void Init() { 1000); shareUsageTracker = new ShareUsageTracker( cacheConfig, - interval, + NO_EXPIRY_INTERVAL_MILLIS, evidenceKeyFilter); evidenceData = new HashMap<>(); @@ -102,26 +118,56 @@ public void ShareUsageTracker_RepeatEvidence_BeforeSessionTimeout() { @Test public void ShareUsageTracker_RepeatEvidence_AfterSessionTimeout() throws InterruptedException { + ShareUsageTracker shareUsageTracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + EXPIRY_INTERVAL_MILLIS, + evidenceKeyFilter); int trackedEvents = 0; for (int i = 0; i < 2; i++) { if (shareUsageTracker.track(data)) { trackedEvents++; } - // Wait some time equal to the interval to elapse. - Thread.sleep(interval); + // Wait for the interval to pass, with a margin. + Thread.sleep(EXPIRY_INTERVAL_MILLIS + EXPIRY_MARGIN_MILLIS); } assertEquals(2, trackedEvents); } + /** + * A repeat is shared at most once in each interval. Sharing a repeat after + * the interval starts a new interval, so a repeat straight after it is not + * shared. + */ @Test - public void ShareUsageTracker_Session_Track() throws InterruptedException { - shareUsageTracker = new ShareUsageTracker( + public void ShareUsageTracker_RepeatEvidence_SharedOncePerInterval() + throws InterruptedException { + long interval = 500; + ShareUsageTracker shareUsageTracker = new ShareUsageTracker( new CacheConfiguration(new LruPutCache.Builder(), 1000), interval, evidenceKeyFilter); + assertTrue(shareUsageTracker.track(data), + "New evidence was not shared."); + Thread.sleep(interval + EXPIRY_MARGIN_MILLIS); + assertTrue(shareUsageTracker.track(data), + "A repeat after the interval was not shared."); + assertFalse(shareUsageTracker.track(data), + "A repeat straight after a repeat was shared was also shared, " + + "so the time it was shared was not stored."); + } + + @Test + public void ShareUsageTracker_Session_Track() { + // The interval does not expire, so only the session cookie keeps the + // two requests apart. + shareUsageTracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + NO_EXPIRY_INTERVAL_MILLIS, + evidenceKeyFilter); + int trackedEvents = 0; for (int i = 0; i < 2; i++) { final Map evidenceData = new HashMap<>(); @@ -138,7 +184,6 @@ public DataKey answer(InvocationOnMock invocationOnMock) throws Throwable { } } ); - Thread.sleep(interval); if (shareUsageTracker.track(data)) { trackedEvents++; } @@ -156,7 +201,7 @@ public void ShareUsageTracker_Session_DoNotTrack() { sessionCookieName); ShareUsageTracker shareUsageTracker = new ShareUsageTracker( new CacheConfiguration(new LruPutCache.Builder(), 1000), - interval, + NO_EXPIRY_INTERVAL_MILLIS, evidenceKeyFilter); int trackedEvents = 0; From cf7e5e28f6751b7afc43ef5dcfef399d3e820717 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 15:22:23 +0000 Subject: [PATCH 77/95] Pause usage sharing sends after repeated failures (#62) * Test that usage sharing pauses sending after failed sends Usage sharing in Java has no back-off. Every batch is sent while the service is failing, and is lost. These tests expect the back-off .NET is being given, at its default settings of ten failed sends within 100 seconds and a first pause of two seconds: - after ten failed sends nothing is sent during the pause, and the evidence processed during it is all sent once the pause ends and the service answers, with no more requests processed; - closing while paused returns at once and sends nothing. They fail until the back-off is added. * Pause usage sharing sends after repeated failures Java now has the same back-off as .NET, with the same five settings and defaults. - After setFailuresToEnterRecovery sends fail within setFailuresWindowSeconds, sending pauses for setInitialRecoveryDelaySeconds. A send fails when it throws or the answer is not 200. - When a pause ends one send is tried. If it fails the delay is multiplied by setRecoveryMultiplier, up to setMaxRecoveryDelaySeconds. If it succeeds the back-off ends. - Processing is not affected. Evidence is queued during a pause and new evidence is dropped only when the queue is full. - The consumer waits out the pause itself, on a latch that closing releases, and then sends until fewer than the minimum entries per message are queued, so the queue drains when a pause ends without more requests being processed. - Closing ends a pause at once and sends nothing while paused, logging how many entries were not sent. ShareUsageBackOff holds the rules and takes the time from the caller, so its tests set the time. The consumer calls the new protected waitWhilePaused before each message and recordSendResult after it. New constructors take the five settings, and the existing ones use the defaults. * Check the whole range of failures to enter recovery in the back-off The builder setter rejects a value outside 1 to 100, and the back-off now does too, so a class derived from ShareUsageBase gets the same check through its constructor. * Keep history out of the comments in the changed send code The comments in the functions this change touches described what the code used to do. They now say only what the code does and why. * Test the usage sharing options, the case of keys and the tracker's cache * Add the usage sharing options to share all Evidence and all query string parameters, read keys without regard to case, and keep one tracker cache * Test the timeouts on the legacy send * Test the back-off time comes from the monotonic clock * Time pauses on a monotonic clock and time out the legacy send * Test that an upload to a service that never answers times out * Time out an upload that is never answered * Test that discarding entries from a full queue is logged without a message per entry * Log entries discarded from a full queue without a message per entry * Test that a failed answer reports the status and the content * Test that a failed upload gives the content of the answer * Report the status and the content of a failed answer * Guard the failure deque against unboxing an empty first entry * Let the upload tests keep restricted headers allowed for the run * Remove a file added by mistake --- .../data/EvidenceKeyFilterShareUsage.java | 59 ++- .../EvidenceKeyFilterShareUsageTracker.java | 32 ++ .../fiftyone/flowelements/Constants.java | 46 ++ .../flowelements/ShareUsageBackOff.java | 169 +++++++ .../fiftyone/flowelements/ShareUsageBase.java | 219 ++++++++- .../flowelements/ShareUsageBuilder.java | 9 +- .../flowelements/ShareUsageBuilderBase.java | 143 ++++++ .../flowelements/ShareUsageElement.java | 131 +++++- .../fiftyone/trackers/ShareUsageTracker.java | 42 +- .../flowelements/ShareUsageBackOffTests.java | 199 ++++++++ .../ShareUsageElementBackOffTests.java | 436 ++++++++++++++++++ .../flowelements/ShareUsageOptionsTests.java | 282 +++++++++++ .../ShareUsageSendTimeoutTests.java | 133 ++++++ .../engines/caching/DataKeyedCacheBase.java | 16 +- .../engines/services/DataUploader.java | 7 + .../engines/services/DataUploaderHttp.java | 34 +- .../services/DataUploaderHttpTests.java | 132 ++++++ 17 files changed, 2025 insertions(+), 64 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index db1091fbe..3bfd42e8f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -47,9 +47,12 @@ * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be * shared using the constructor parameter. * 5) Not a header, cookie, session value or query string parameter. + * When all Evidence is shared, everything is included. When all query string + * parameters are shared, every query string parameter is included. * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email - * identifiers, the query string and the path are never included. + * identifiers, the query string, the path and the Cookie header are never + * included, whatever the configuration. */ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { @@ -65,6 +68,17 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final String sessionCookieName; + /** + * If true then every key is included, apart from those that are never + * shared. + */ + private final boolean shareAllEvidence; + + /** + * If true then every query string parameter is included. + */ + private final boolean shareAllQueryStringParameters; + /** * The content of HTTP headers in this array will not be included in the * request information sent to 51degrees. Any header names added here are @@ -113,6 +127,40 @@ public EvidenceKeyFilterShareUsage( List includedQueryStringParams, boolean includeSession, String sessionCookieName) { + this( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + sessionCookieName, + false, + false); + } + + /** + * Constructor. + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParams a list of the names of query string + * parameters that share usage should send + * to 51Degrees + * @param includeSession if true then the session cookie will be included in + * the filter + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param shareAllEvidence if true then all Evidence is included, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are included + */ + public EvidenceKeyFilterShareUsage( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession, + String sessionCookieName, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + this.shareAllEvidence = shareAllEvidence; + this.shareAllQueryStringParameters = shareAllQueryStringParameters; // The Cookie header carries every cookie, including the ones the // cookie rule leaves out. this.blockedHttpHeaders.add("cookies"); @@ -149,6 +197,10 @@ public boolean include(String key) { } } + if (shareAllEvidence) { + return true; + } + boolean result; int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); if (separator > 0) { @@ -174,8 +226,9 @@ public boolean include(String key) { case EVIDENCE_QUERY_PREFIX: // Only include query string parameters that start with the // 51Degrees prefix or have been specified in the - // constructor. - result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + // constructor, unless all of them are shared. + result = shareAllQueryStringParameters || + field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || includedQueryStringParams.contains(field); break; default: diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java index 9240b3267..61c2c348c 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java @@ -57,6 +57,38 @@ public EvidenceKeyFilterShareUsageTracker( sessionCookieName); } + /** + * Constructor. + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParams a list of the names of query string + * parameters that share usage should send + * to 51Degrees + * @param includeSession if true then the session cookie will be included in + * the filter + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param shareAllEvidence if true then all Evidence is included, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are included + */ + public EvidenceKeyFilterShareUsageTracker( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession, + String sessionCookieName, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + super( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters); + } + @Override public boolean include(String key) { if(key.equalsIgnoreCase(EVIDENCE_SESSIONID) || diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index b098ee136..48e1e1b88 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -93,6 +93,13 @@ public class Constants { * Default lost data count for share usage before it wraps */ static final int LOST_DATA_RESET_DEFAULT = 5000000; + + /** + * While the queue is full, a message is logged for the first entry + * discarded and then once for every this many more, never one for each + * entry. + */ + static final int QUEUE_FULL_LOG_EVERY = 1000; /** * The default URL to send usage data to. */ @@ -104,4 +111,43 @@ public class Constants { */ static final String EVIDENCE_HTTPHEADER_COOKIE_SUFFIX = "cookie"; + /** + * The default number of failed sends within the failures window that + * pauses sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT = 10; + + /** + * The fewest failed sends that can be set to pause sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN = 1; + + /** + * The most failed sends that can be set to pause sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX = 100; + + /** + * The default time in seconds over which failed sends are counted. + */ + public static final int SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT = 100; + + /** + * The default length in seconds of the first pause after failed sends. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT = 2.0; + + /** + * The default longest pause in seconds. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT = 300.0; + + /** + * The default multiplier applied to the pause each time the send tried + * when a pause ends fails. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT = 2.0; } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java new file mode 100644 index 000000000..d0bb4d0c3 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java @@ -0,0 +1,169 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * Decides when usage sharing may send. Sending pauses for the initial delay + * once the set number of sends fail within the window. A failed send after a + * pause multiplies the delay, up to the maximum, and a successful send ends + * the back-off. Only sending pauses, so evidence is still queued while + * paused. Each method takes the time from the caller, in milliseconds on + * the clock that {@link #now()} reads, so a change to the system time cannot + * lengthen or skip a pause. + */ +class ShareUsageBackOff { + + private final int failuresToEnterRecovery; + private final long failuresWindowMillis; + private final double initialDelaySeconds; + private final double maxDelaySeconds; + private final double multiplier; + + /** + * When each recent failed send happened, oldest first. Empty while + * backing off. + */ + private final Deque failures = new ArrayDeque<>(); + + /** + * The length of the current pause in seconds, or zero when not backing + * off. + */ + private double delaySeconds = 0; + + /** + * When sending may start again. + */ + private long resumeAt = Long.MIN_VALUE; + + /** + * @return the time in milliseconds on a monotonic clock, for the caller to + * pass to the other methods. Its zero is not a calendar time. + */ + static long now() { + return System.nanoTime() / 1000000; + } + + ShareUsageBackOff( + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialDelaySeconds, + double maxDelaySeconds, + double multiplier) { + if (failuresToEnterRecovery < + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN || + failuresToEnterRecovery > + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX) { + throw new IllegalArgumentException( + "failuresToEnterRecovery must be from " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN + + " to " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX + "."); + } + if (failuresWindowSeconds < 1) { + throw new IllegalArgumentException( + "failuresWindowSeconds must be 1 or more."); + } + if ((initialDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "initialDelaySeconds must be more than 0."); + } + if ((maxDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "maxDelaySeconds must be more than 0."); + } + if ((multiplier >= 1) == false) { + throw new IllegalArgumentException( + "multiplier must be 1 or more."); + } + this.failuresToEnterRecovery = failuresToEnterRecovery; + this.failuresWindowMillis = failuresWindowSeconds * 1000L; + this.initialDelaySeconds = initialDelaySeconds; + this.maxDelaySeconds = maxDelaySeconds; + this.multiplier = multiplier; + } + + /** + * @return true from the start of a pause until a send succeeds + */ + synchronized boolean isBackingOff() { + return delaySeconds > 0; + } + + /** + * @param now the time now in milliseconds + * @return the milliseconds left before sending may start again, or zero + * if it may start now + */ + synchronized long remaining(long now) { + return resumeAt > now ? resumeAt - now : 0; + } + + /** + * Records a failed send. + * @param now the time now in milliseconds + * @return the new pause in milliseconds if this failure started or + * lengthened one, otherwise zero + */ + synchronized long failed(long now) { + if (delaySeconds > 0) { + // The send tried when the last pause ended failed. + delaySeconds = Math.min( + delaySeconds * multiplier, + maxDelaySeconds); + } else { + failures.addLast(now); + // The emptiness check comes first so that nothing is unboxed + // from an empty deque. The constructor refuses a window below + // one second, so the deque always keeps the failure just added. + while (failures.isEmpty() == false && + failures.peekFirst() < now - failuresWindowMillis) { + failures.removeFirst(); + } + if (failures.size() < failuresToEnterRecovery) { + return 0; + } + failures.clear(); + delaySeconds = Math.min(initialDelaySeconds, maxDelaySeconds); + } + long pause = Math.max(1, Math.round(delaySeconds * 1000)); + resumeAt = now + pause; + return pause; + } + + /** + * Records a successful send. + * @return true if this ended a back-off + */ + synchronized boolean succeeded() { + if (delaySeconds == 0) { + return false; + } + delaySeconds = 0; + resumeAt = Long.MIN_VALUE; + return true; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 96b240773..9e417dcfe 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -228,6 +228,23 @@ public abstract class ShareUsageBase */ protected long lostDataReset = LOST_DATA_RESET_DEFAULT; + /** + * True from the first entry discarded because the queue is full until an + * entry is next queued. While true, a message is logged only for every + * {@link Constants#QUEUE_FULL_LOG_EVERY} entries discarded. + */ + private volatile boolean queueFullReported = false; + + /** + * Decides when sending may happen after failed sends. + */ + private final ShareUsageBackOff backOff; + + /** + * Released when the element starts closing, to end any pause at once. + */ + private final CountDownLatch closing = new CountDownLatch(1); + /** * Constructor * @param logger the logger to use @@ -396,6 +413,98 @@ protected ShareUsageBase( List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT, + Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, + false, + false); + } + + /** + * Constructor + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then the item is not shared + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param failuresToEnterRecovery the number of failed sends within the + * failures window that pauses sending + * @param failuresWindowSeconds the time in seconds over which failed sends + * are counted + * @param initialRecoveryDelaySeconds the length in seconds of the first + * pause + * @param maxRecoveryDelaySeconds the longest a pause can be, in seconds + * @param recoveryMultiplier what the pause is multiplied by each time the + * send tried when a pause ends fails + * @param shareAllEvidence if true then all Evidence is shared, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are shared + */ + protected ShareUsageBase( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialRecoveryDelaySeconds, + double maxRecoveryDelaySeconds, + double recoveryMultiplier, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { super(logger, null); if (minimumEntriesPerMessage > maximumQueueSize) { @@ -410,6 +519,12 @@ protected ShareUsageBase( } evidenceCollection = new LinkedBlockingQueue<>(maximumQueueSize); + backOff = new ShareUsageBackOff( + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier); this.addTimeout = addTimeout; this.takeTimeout = takeTimeout; @@ -430,7 +545,12 @@ protected ShareUsageBase( includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX); evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName); + blockedHttpHeaders, + includedQueryStringParameters, + false, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters); this.ignoreDataEvidenceFilter = ignoreDataEvidenceFilter; @@ -444,7 +564,12 @@ protected ShareUsageBase( cacheConfiguration, intervalMillis, new EvidenceKeyFilterShareUsageTracker( - blockedHttpHeaders, includedQueryStringParameters, trackSession, sessionCookieName)); + blockedHttpHeaders, + includedQueryStringParameters, + trackSession, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters)); } properties = new ArrayList<>(); @@ -609,14 +734,15 @@ protected void managedResourcesCleanup() { // This is set before anything is submitted, so a consumer can never // observe it as false and then exit leaving a partial batch behind. stopping = true; - // Queue a final consumer without consulting isRunning(). That check is - // subject to a time-of-check/time-of-use race: a consumer that has - // finished its loop with a sub-threshold remainder still reports as - // running until its Future flips to done, which used to stop close from - // queueing the drain that would have sent the remainder. The executor is - // single threaded, so the task queued here cannot run alongside a - // consumer that is already in flight - it runs after it - and shutdown() - // still allows an already submitted task to run to completion. + // Ends a pause at once, so closing never waits for one. Nothing is + // sent while paused. + closing.countDown(); + // Queue a final consumer without consulting isRunning(). A consumer + // that has finished its loop with a sub-threshold remainder still + // reports as running until its Future is done, so checking would skip + // the drain that sends the remainder. The executor is single threaded, + // so the task queued here runs after any consumer already in flight, + // and shutdown() still allows an already submitted task to finish. submitSendData(); try { executor.shutdown(); @@ -630,6 +756,13 @@ protected void managedResourcesCleanup() { // abandon its own wait immediately. logger.warn("Interrupted while awaiting close down"); } + if (evidenceCollection.isEmpty() == false && + backOff.remaining(ShareUsageBackOff.now()) > 0) { + logger.info( + "{} usage sharing entries were not sent, as sending was " + + "paused after failed sends when usage sharing closed.", + evidenceCollection.size()); + } } @Override @@ -670,6 +803,7 @@ private void processData(FlowData data, Map found) { getDataFromEvidence(found), addTimeout, TimeUnit.MILLISECONDS)) { + queueFullReported = false; logger.debug("Queued {} entries", evidenceCollection.size()); // If the collection has enough entries then start // taking data from it to be sent. @@ -677,11 +811,17 @@ private void processData(FlowData data, Map found) { trySendData(); } } else { - if (++lostData >= lostDataReset) { - logger.warn("Could not queue data for Share Usage. This could mean that" + - " the queue is too small, that the proportion of data shared is" + - " too high - or that the sending process has encountered" + - " problems sending."); + long lost = ++lostData; + if (queueFullReported == false || + lost % Constants.QUEUE_FULL_LOG_EVERY == 0) { + queueFullReported = true; + logger.warn("The usage sharing queue is full, so usage data is " + + "being discarded. This could mean that the queue is too " + + "small, that the proportion of data shared is too high, " + + "or that sending is failing. {} entries have been " + + "discarded.", lost); + } + if (lost >= lostDataReset) { lostData = 0; } } @@ -711,7 +851,9 @@ private ShareUsageData getDataFromEvidence(Map found) { continue; } - switch (entry.getKey()) { + // The keys are compared without regard to case, and the constants + // are lower case. + switch (entry.getKey().toLowerCase(Locale.ROOT)) { case fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY: // The client IP is dealt with separately for backwards // compatibility purposes. @@ -789,6 +931,51 @@ private void submitSendData() { protected abstract void sendUsageData() throws HttpException; + /** + * Waits until sending may start again after failed sends. A consumer + * calls this before each message it sends. + * @return false if the element started closing while sending was paused, + * in which case nothing more should be sent + */ + protected boolean waitWhilePaused() { + long pause; + while ((pause = backOff.remaining(ShareUsageBackOff.now())) > 0) { + try { + if (closing.await(pause, TimeUnit.MILLISECONDS)) { + return false; + } + } catch (InterruptedException e) { + // Not re-interrupting, for the reason given in + // managedResourcesCleanup. The interrupt comes from closing. + return false; + } + } + return true; + } + + /** + * Records whether the remote service accepted a message, so that repeated + * failures pause sending and a success ends the pause. + * @param accepted true if the message was accepted + */ + protected void recordSendResult(boolean accepted) { + if (accepted) { + if (backOff.succeeded()) { + logger.info("Usage sharing resumed after a send succeeded."); + } + } else { + long pause = backOff.failed(ShareUsageBackOff.now()); + if (pause > 0) { + logger.warn( + "Usage sharing paused for {} seconds after failed " + + "sends. Usage data is still queued, up to the " + + "maximum queue size, and sending starts again when " + + "the pause ends.", + pause / 1000.0); + } + } + } + /** * Get engine Version No. * @return engine version no diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java index 56a96e9ac..c73a86326 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java @@ -129,7 +129,14 @@ public ShareUsageElement build() throws IOException { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - null); + null, + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier, + shareAllEvidence, + shareAllQueryStringParameters); if (Objects.nonNull(this.httpClient)) { sue.httpClient = httpClient; } else if (Objects.nonNull(dataUploader)){ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index e376660cf..6e07f6dd3 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -59,6 +59,13 @@ public abstract class ShareUsageBuilderBase { protected final List includedQueryStringParameters = new ArrayList<>(); protected final List> ignoreDataEvidenceFilter = new ArrayList<>(); protected boolean trackSession; + protected int failuresToEnterRecovery = Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT; + protected int failuresWindowSeconds = Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT; + protected double initialRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; + protected double maxRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; + protected double recoveryMultiplier = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; + protected boolean shareAllEvidence; + protected boolean shareAllQueryStringParameters; /** * Constructor @@ -143,6 +150,56 @@ public ShareUsageBuilderBase setIncludedQueryStringParameter( return this; } + /** + * If set to true, every query string and HTTP form parameter is shared, + * rather than only those that start with 51D_ or are named in the + * included parameters. The whole query string and the parameters that are + * never shared are still not shared. + *

    + * Default value is false + * @param shareAll true to share all query string parameters + * @return this builder + */ + @DefaultValue(booleanValue = false) + public ShareUsageBuilderBase setShareAllQueryStringParameters( + boolean shareAll) { + this.shareAllQueryStringParameters = shareAll; + return this; + } + + /** + * Whether every query string parameter is shared. + * @return true if all query string parameters are shared + */ + public boolean isShareAllQueryStringParameters() { + return shareAllQueryStringParameters; + } + + /** + * If set to true, all Evidence is shared, which overrides the blocked + * HTTP headers and the rules for cookies, session values and query string + * parameters. The keys that are never shared (an email address, the salt + * for hashed email identifiers, the whole query string, the path and the + * Cookie header) are still not shared. + *

    + * Default value is false + * @param shareAll true to share all Evidence + * @return this builder + */ + @DefaultValue(booleanValue = false) + public ShareUsageBuilderBase setShareAllEvidence(boolean shareAll) { + this.shareAllEvidence = shareAll; + return this; + } + + /** + * Whether all Evidence is shared. + * @return true if all Evidence is shared + */ + public boolean isShareAllEvidence() { + return shareAllEvidence; + } + /** * By default, all HTTP headers (excluding a few such as 'cookies', if they * don't start with 51D_) are shared. Individual headers can be excluded from sharing by @@ -420,6 +477,92 @@ public ShareUsageBuilderBase setTrackSession(boolean track) { return this; } + /** + * Set the number of failed sends within the failures window that pauses + * sending. Usage data is still queued while sending is paused, up to the + * maximum queue size. + * @param failuresToEnterRecovery the number of failed sends, from 1 to 100 + * @return this builder + */ + @DefaultValue(intValue = Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT) + public ShareUsageBuilderBase setFailuresToEnterRecovery( + int failuresToEnterRecovery) { + if (failuresToEnterRecovery < Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN || + failuresToEnterRecovery > Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX) { + throw new IllegalArgumentException( + "failuresToEnterRecovery must be from " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN + " to " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX + + ". Received: " + failuresToEnterRecovery + "."); + } + this.failuresToEnterRecovery = failuresToEnterRecovery; + return this; + } + + /** + * Set the time in seconds over which failed sends are counted. + * @param failuresWindowSeconds the time in seconds, 1 or more + * @return this builder + */ + @DefaultValue(intValue = Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT) + public ShareUsageBuilderBase setFailuresWindowSeconds( + int failuresWindowSeconds) { + if (failuresWindowSeconds < 1) { + throw new IllegalArgumentException( + "failuresWindowSeconds must be 1 or more."); + } + this.failuresWindowSeconds = failuresWindowSeconds; + return this; + } + + /** + * Set the length in seconds of the first pause after failed sends. + * @param initialDelaySeconds the length in seconds, more than 0 + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT) + public ShareUsageBuilderBase setInitialRecoveryDelaySeconds( + double initialDelaySeconds) { + if ((initialDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "initialDelaySeconds must be more than 0."); + } + this.initialRecoveryDelaySeconds = initialDelaySeconds; + return this; + } + + /** + * Set the longest a pause can be, in seconds. + * @param maxDelaySeconds the length in seconds, more than 0 + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT) + public ShareUsageBuilderBase setMaxRecoveryDelaySeconds( + double maxDelaySeconds) { + if ((maxDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "maxDelaySeconds must be more than 0."); + } + this.maxRecoveryDelaySeconds = maxDelaySeconds; + return this; + } + + /** + * Set what the pause is multiplied by each time the send tried when a + * pause ends fails. 2 doubles the pause each time. + * @param multiplier the multiplier, 1 or more + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT) + public ShareUsageBuilderBase setRecoveryMultiplier(double multiplier) { + if ((multiplier >= 1) == false) { + throw new IllegalArgumentException( + "multiplier must be 1 or more."); + } + this.recoveryMultiplier = multiplier; + return this; + } + /** * Create the {@link ShareUsageElement}. * @return the newly created target instance diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 6fb71eae7..58109cb2a 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -43,6 +43,11 @@ import java.util.zip.GZIPOutputStream; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_MAX_EVIDENCE_LENGTH; /** @@ -306,6 +311,98 @@ public class ShareUsageElement extends ShareUsageBase { List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT, + SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, + false, + false); + } + + /** + * Constructor + * + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then the item is not shared + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param failuresToEnterRecovery the number of failed sends within the failures + * window that pauses sending + * @param failuresWindowSeconds the time in seconds over which failed sends are + * counted + * @param initialRecoveryDelaySeconds the length in seconds of the first pause + * @param maxRecoveryDelaySeconds the longest a pause can be, in seconds + * @param recoveryMultiplier what the pause is multiplied by each time the send + * tried when a pause ends fails + * @param shareAllEvidence if true then all Evidence is shared, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are shared + */ + ShareUsageElement( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialRecoveryDelaySeconds, + double maxRecoveryDelaySeconds, + double recoveryMultiplier, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { super( logger, sharePercentage, @@ -320,7 +417,14 @@ public class ShareUsageElement extends ShareUsageBase { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - tracker); + tracker, + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier, + shareAllEvidence, + shareAllQueryStringParameters); headers.put("Content-Type", "text/xml; charset=utf-8"); // assume default GZIP connection @@ -333,7 +437,8 @@ public class ShareUsageElement extends ShareUsageBase { } /** - * Take data from the queue until there are fewer entries than the minimum batch size + * Take data from the queue until there are fewer entries than the minimum + * batch size, waiting out any pause after failed sends before each message */ @Override protected void sendUsageData() { @@ -342,10 +447,14 @@ protected void sendUsageData() { List allData = new ArrayList<>(); // drain queue while minimum entries in it do { + // Closing while paused ends the wait, and nothing more is sent. + if (waitWhilePaused() == false) { + break; + } logger.debug(threadMarker, "Queue size is {}", evidenceCollection.size()); try { allData.clear(); - // for historical reasons max entries set to 2* + // A message holds at most twice the minimum entries. while (allData.size() < minEntriesPerMessage * 2) { ShareUsageData currentData = evidenceCollection.poll(takeTimeout, TimeUnit.MILLISECONDS); // no data available @@ -359,7 +468,13 @@ protected void sendUsageData() { // drained everything - and there is no point posting a document // with no entries in it. if (allData.isEmpty() == false) { - sendAsXML(allData); + boolean accepted = false; + try { + sendAsXML(allData); + accepted = true; + } finally { + recordSendResult(accepted); + } } } catch (InterruptedException interruptedException) { // Deliberately not re-interrupting: the interrupt comes from @@ -391,7 +506,10 @@ protected void sendAsXML(List allData) throws Exception { // check for completion int code = dataUploader.getResponseCode(); if (code != 200) { - throw new Exception(String.format("Share Usage response code was %d ", code)); + throw new Exception(String.format( + "Share Usage response code was %d with content '%s'", + code, + dataUploader.getResponseContent())); } logger.debug(threadMarker, "Send elements done"); } @@ -560,6 +678,9 @@ protected void legacySendAsXML(List allData) throws Exception { } HttpURLConnection connection = httpClient.connect(new URI(shareUsageUrl.trim()).toURL()); + // A send that is never answered must end, or it stops all sending. + connection.setConnectTimeout(httpSendTimeout); + connection.setReadTimeout(httpSendTimeout); String response = httpClient.postData(connection, headers, baos.toByteArray()); int responseCode = connection.getResponseCode(); String responseMessage = connection.getResponseMessage() + "data: '" + response + "'"; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index c1bd3bb09..8bfa52d3b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -22,7 +22,6 @@ package fiftyone.pipeline.engines.fiftyone.trackers; -import fiftyone.caching.PutCache; import fiftyone.pipeline.core.data.DataKey; import fiftyone.pipeline.core.data.DataKeyBuilder; import fiftyone.pipeline.core.data.DataKeyBuilderDefault; @@ -32,7 +31,6 @@ import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.trackers.TrackerBase; -import java.io.IOException; import java.util.Date; import java.util.Map; @@ -55,13 +53,6 @@ public class ShareUsageTracker extends TrackerBase { */ private final long interval; - /** - * The times keyed by the values usage sharing would share. The cache of - * the base class is keyed by the Evidence alone, which no longer holds - * the web request's values. - */ - private final PutCache sharedValuesCache; - /** * Construct a new instance. * @param configuration the cache configuration to use when building the @@ -78,10 +69,6 @@ public ShareUsageTracker( super(configuration); this.interval = intervalMillis; this.filter = filter; - // The base class has checked that the builder makes a PutCache. - this.sharedValuesCache = (PutCache) - configuration.getCacheBuilder().build( - null, configuration.getSize()); } @Override @@ -109,31 +96,14 @@ protected EvidenceKeyFilter getFilter() { return filter; } - @Override - public Date get(FlowData data) { - return sharedValuesCache.get(getSharedValuesKey(data)); - } - - @Override - public void put(FlowData data, Date value) { - sharedValuesCache.put(getSharedValuesKey(data), value); - } - - @Override - public void close() throws IOException { - try { - sharedValuesCache.close(); - } finally { - super.close(); - } - } - /** - * The key of the values {@link ShareUsageEvidence#find} finds that the - * filter includes, which take in the web request's values, as the - * session tracking hash is of the values that would be shared. + * The key is made from the values {@link ShareUsageEvidence#find} finds + * that the filter includes, which take in the web request's values, + * as the session tracking hash is of the values that would be shared. + * The Evidence alone does not hold them. */ - private DataKey getSharedValuesKey(FlowData data) { + @Override + protected DataKey generateKey(FlowData data) { DataKeyBuilder builder = new DataKeyBuilderDefault(); for (Map.Entry entry : ShareUsageEvidence.find(data.getEvidence()).entrySet()) { diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java new file mode 100644 index 000000000..1a628235c --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java @@ -0,0 +1,199 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The rules that decide when usage sharing may send after failed sends, + * with each test setting the time, and the builder settings for them. + */ +public class ShareUsageBackOffTests { + + private static final long START = 1767225600000L; + + private static long at(double seconds) { + return START + Math.round(seconds * 1000); + } + + /** + * Three failures within a ten second window start a two second pause, + * rising by two each time up to five. + */ + private static ShareUsageBackOff create( + int failuresToEnterRecovery, + double initialDelaySeconds, + double maxDelaySeconds) { + return new ShareUsageBackOff( + failuresToEnterRecovery, + 10, + initialDelaySeconds, + maxDelaySeconds, + 2); + } + + private static ShareUsageBackOff create() { + return create(3, 2, 5); + } + + /** + * The time the back-off is given comes from a clock that a change to the + * system time does not move. + */ + @Test + public void ShareUsageBackOff_Now_FollowsTheMonotonicClock() { + long before = System.nanoTime() / 1000000; + long now = ShareUsageBackOff.now(); + long after = System.nanoTime() / 1000000; + + assertTrue(before <= now && now <= after); + } + + @Test + public void ShareUsageBackOff_PausesAfterFailuresWithinWindow() { + ShareUsageBackOff backOff = create(); + + assertEquals(0, backOff.failed(at(0))); + assertEquals(0, backOff.failed(at(1))); + assertEquals(0, backOff.remaining(at(1)), + "Sending paused before enough sends failed."); + assertFalse(backOff.isBackingOff()); + + assertEquals(2000, backOff.failed(at(2))); + assertTrue(backOff.isBackingOff()); + assertEquals(1000, backOff.remaining(at(3))); + assertEquals(0, backOff.remaining(at(4))); + } + + @Test + public void ShareUsageBackOff_FailuresOutsideWindowDoNotCount() { + ShareUsageBackOff backOff = create(); + + backOff.failed(at(0)); + backOff.failed(at(1)); + + assertEquals(0, backOff.failed(at(11.5)), + "Failures older than the window were counted."); + assertFalse(backOff.isBackingOff()); + } + + /** + * The window counts failures, not failures in a row, so a high failure + * rate pauses sending even when some sends succeed. + */ + @Test + public void ShareUsageBackOff_SuccessesBetweenFailuresDoNotReset() { + ShareUsageBackOff backOff = create(); + + backOff.failed(at(0)); + assertFalse(backOff.succeeded()); + backOff.failed(at(1)); + assertFalse(backOff.succeeded()); + + assertEquals(2000, backOff.failed(at(2))); + } + + @Test + public void ShareUsageBackOff_FailedSendAfterPauseMultipliesDelay() { + ShareUsageBackOff backOff = create(1, 2, 5); + + assertEquals(2000, backOff.failed(at(0))); + assertEquals(4000, backOff.failed(at(2))); + assertEquals(5000, backOff.failed(at(6)), + "The delay grew past the maximum."); + assertEquals(5000, backOff.failed(at(11))); + assertEquals(5000, backOff.remaining(at(11))); + } + + @Test + public void ShareUsageBackOff_SuccessEndsBackOff() { + ShareUsageBackOff backOff = create(2, 2, 5); + backOff.failed(at(0)); + backOff.failed(at(1)); + backOff.failed(at(3)); + + assertTrue(backOff.succeeded(), + "A success did not end the back-off."); + assertFalse(backOff.isBackingOff()); + assertEquals(0, backOff.remaining(at(4))); + + // The next pause needs the full number of failures again, and starts + // from the initial delay. + assertEquals(0, backOff.failed(at(8))); + assertEquals(2000, backOff.failed(at(9))); + } + + @Test + public void ShareUsageBackOff_MaximumBelowInitialLimitsFirstPause() { + ShareUsageBackOff backOff = create(1, 10, 3); + + assertEquals(3000, backOff.failed(at(0))); + } + + @Test + public void ShareUsageBackOff_InvalidSettingsAreRejected() { + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(0, 10, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(101, 10, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 0, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 0, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, Double.NaN, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 2, 0, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 2, 5, 0.5)); + } + + /** + * The builder settings that pause sending after failed sends reject + * values the pause cannot work with. + */ + @Test + public void ShareUsageBuilder_RecoverySettings_RejectValuesOutOfRange() { + ShareUsageBuilder builder = + new ShareUsageBuilder(LoggerFactory.getILoggerFactory()); + + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresToEnterRecovery(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresToEnterRecovery(101)); + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresWindowSeconds(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setInitialRecoveryDelaySeconds(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setMaxRecoveryDelaySeconds(-1)); + assertThrows(IllegalArgumentException.class, + () -> builder.setRecoveryMultiplier(0.5)); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java new file mode 100644 index 000000000..0a6405558 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -0,0 +1,436 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Usage sharing pauses sending after repeated failed sends, keeps queueing + * evidence while paused, and drains the queue once a send succeeds. These + * tests use the default settings, which pause sending for two seconds after + * ten sends fail within 100 seconds. + */ +public class ShareUsageElementBackOffTests { + + /** + * The longest a test waits for something it expects to happen. + */ + private static final long WAIT_LIMIT_MILLIS = 30000; + + private static final int DEFAULT_FAILURES = 10; + + private static final long DEFAULT_PAUSE_MILLIS = 2000; + + private static final Pattern CLIENT_IP = + Pattern.compile("([^<]*)"); + + /** + * Answers each send with the status the test sets, and records when each + * send was answered and the client IPs of the sends it accepted. + */ + private static class RecordingUploader implements DataUploader { + private final long start = System.nanoTime(); + private final List times = new ArrayList<>(); + private final List accepted = new ArrayList<>(); + private volatile int code = 500; + private volatile String content = ""; + private ByteArrayOutputStream body; + + @Override + public synchronized OutputStream getOutputStream() { + body = new ByteArrayOutputStream(); + return body; + } + + @Override + public synchronized int getResponseCode() { + int answer = code; + times.add(now()); + if (answer == 200) { + Matcher matcher = CLIENT_IP.matcher( + new String(body.toByteArray(), StandardCharsets.UTF_8)); + while (matcher.find()) { + accepted.add(matcher.group(1)); + } + } + return answer; + } + + public String getResponseContent() { + return content; + } + + long now() { + return (System.nanoTime() - start) / 1000000; + } + + synchronized int count() { + return times.size(); + } + + synchronized List times() { + return new ArrayList<>(times); + } + + synchronized List accepted() { + return new ArrayList<>(accepted); + } + } + + /** + * Shares every request. + */ + private static class ShareAll implements Tracker { + @Override + public boolean track(FlowData flowData) { + return true; + } + } + + private Pipeline pipeline; + private Logger logger = LoggerFactory.getLogger(ShareUsageElement.class); + private RecordingUploader uploader; + private ShareUsageElement element; + private int requests; + + /** + * Each entry is sent on its own, as the minimum entries per message is + * one. + */ + @BeforeEach + public void init() throws Exception { + pipeline = new PipelineBuilder().build(); + uploader = new RecordingUploader(); + requests = 0; + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + new ShareAll()); + element.addPipeline(pipeline); + element.dataUploader = uploader; + } + + /** + * Replaces the element with one that has its own back-off settings. The + * failures window is long enough that every failure in a test counts. + */ + private void useBackOff( + int failuresToEnterRecovery, + double initialDelaySeconds, + double maxDelaySeconds, + int maximumQueueSize) throws Exception { + element.close(); + element = new ShareUsageElement( + logger, + 1, + 1, + maximumQueueSize, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + new ShareAll(), + failuresToEnterRecovery, + 100, + initialDelaySeconds, + maxDelaySeconds, + 2, + false, + false); + element.addPipeline(pipeline); + element.dataUploader = uploader; + } + + @AfterEach + public void cleanUp() throws Exception { + element.close(); + pipeline.close(); + } + + /** + * Processes a request whose evidence no other request has, and returns + * its client IP. + */ + private String process() throws Exception { + int number = requests++; + String ip = "10.0." + (number / 250) + "." + (number % 250 + 1); + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_CLIENTIP_KEY, ip); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + } + return ip; + } + + private static void waitFor( + BooleanSupplier condition, + Supplier message) throws InterruptedException { + long end = System.currentTimeMillis() + WAIT_LIMIT_MILLIS; + while (condition.getAsBoolean() == false) { + if (System.currentTimeMillis() > end) { + fail(message.get()); + } + Thread.sleep(20); + } + } + + /** + * Processes requests one at a time, waiting for each to be sent, so each + * is a separate failed send. + */ + private void failSends(int count) throws Exception { + for (int i = 1; i <= count; i++) { + process(); + final int expected = i; + waitFor( + () -> uploader.count() == expected && + element.isRunning() == false, + () -> "Send " + expected + " was not made."); + } + } + + /** + * Ten failed sends pause sending. Nothing is sent during the pause, the + * evidence processed during it is queued, and all of it is sent once the + * pause ends and the service answers, with no more requests processed to + * start the sending. + */ + @Test + public void ShareUsageElement_BackOff_PausesThenDrainsQueue() + throws Exception { + failSends(DEFAULT_FAILURES); + long pausedAt = uploader.now(); + + List queued = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + queued.add(process()); + } + Thread.sleep(700); + assertEquals(DEFAULT_FAILURES, uploader.count(), + "Sends were made during the pause that ten failed sends should " + + "have started."); + + uploader.code = 200; + waitFor( + () -> uploader.accepted().containsAll(queued), + () -> "The evidence processed during the pause was not all " + + "sent once the service answered. Sent: " + uploader.accepted()); + long waited = uploader.times().get(DEFAULT_FAILURES) - pausedAt; + assertTrue(waited >= DEFAULT_PAUSE_MILLIS - 500, + "Sending resumed " + waited + " ms after the pause started, " + + "before the two second pause ended."); + } + + /** + * Closing while paused returns at once rather than waiting for the pause + * to end, and sends nothing. + */ + @Test + public void ShareUsageElement_BackOff_CloseWhilePausedDoesNotWaitOrSend() + throws Exception { + failSends(DEFAULT_FAILURES); + process(); + process(); + + long start = System.currentTimeMillis(); + element.close(); + long took = System.currentTimeMillis() - start; + + assertTrue(took < 1500, + "Closing took " + took + " ms, as it waited for the pause."); + assertEquals(DEFAULT_FAILURES, uploader.count(), + "A send was made during the pause or when closing."); + } + + /** + * Each failed send after a pause multiplies the delay, and the delay + * stops growing at the maximum. + */ + @Test + public void ShareUsageElement_BackOff_DelayGrowsToMaximum() + throws Exception { + useBackOff(1, 0.5, 1.0, 100); + for (int i = 0; i < 40; i++) { + process(); + } + + waitFor( + () -> uploader.count() >= 4, + () -> "Fewer than four sends were made."); + List times = uploader.times(); + long first = times.get(1) - times.get(0); + long second = times.get(2) - times.get(1); + long third = times.get(3) - times.get(2); + String shown = first + ", " + second + ", " + third + " ms"; + assertTrue(first >= 400, + "The first pause was not the initial delay. Gaps: " + shown); + assertTrue(second >= 900, + "The second pause was not doubled. Gaps: " + shown); + assertTrue(third >= 900 && third < 1900, + "The third pause was not held at the maximum. Gaps: " + shown); + } + + /** + * When the queue fills during a pause, new evidence is dropped and what + * was queued first is sent once sending resumes. + */ + @Test + public void ShareUsageElement_BackOff_FullQueueDropsNewEvidence() + throws Exception { + useBackOff(1, 2, 2, 4); + failSends(1); + + List kept = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + kept.add(process()); + } + List dropped = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + dropped.add(process()); + } + uploader.code = 200; + + waitFor( + () -> uploader.accepted().containsAll(kept), + () -> "The evidence queued during the pause was not all sent. " + + "Sent: " + uploader.accepted()); + Thread.sleep(500); + for (String ip : dropped) { + assertTrue(uploader.accepted().contains(ip) == false, + "Evidence processed when the queue was full was sent."); + } + } + + /** + * A logger that records the warnings it is given. + */ + private static Logger recordingLogger(List warnings) { + return (Logger) Proxy.newProxyInstance( + Logger.class.getClassLoader(), + new Class[] {Logger.class}, + (proxy, method, args) -> { + if (method.getName().equals("warn")) { + warnings.add(Arrays.deepToString(args)); + return null; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == String.class) { + return "test"; + } + return null; + }); + } + + /** + * Evidence discarded because the queue is full is logged, but not with a + * message for each entry discarded. + */ + @Test + public void ShareUsageElement_FullQueue_LogsWithoutAMessagePerEntry() + throws Exception { + List warnings = new CopyOnWriteArrayList<>(); + logger = recordingLogger(warnings); + useBackOff(1, 2, 2, 4); + failSends(1); + warnings.clear(); + + for (int i = 0; i < 10; i++) { + process(); + } + + long full = warnings.stream() + .filter(warning -> warning.contains("full")) + .count(); + assertTrue(full >= 1, "Discarding entries was not logged."); + assertTrue(full < 6, "A message was logged for each entry discarded."); + } + + /** + * A send answered with anything other than 200 reports the status code + * and the content of the answer. + */ + @Test + public void ShareUsageElement_FailedAnswer_ReportsStatusAndContent() { + uploader.content = "the service says no"; + + Exception failure = assertThrows( + Exception.class, + () -> element.sendAsXML(new ArrayList<>())); + + assertTrue(failure.getMessage().contains("500")); + assertTrue(failure.getMessage().contains("the service says no")); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java new file mode 100644 index 000000000..65a920482 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -0,0 +1,282 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.caching.Cache; +import fiftyone.caching.CacheBuilder; +import fiftyone.caching.LruPutCache; +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; +import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; +import fiftyone.pipeline.engines.services.DataUploader; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPathFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the usage sharing options "Share all Evidence" and "Share all query + * string parameters" from a configuration file, the case of the keys the + * element reads and the number of caches a tracker holds. + * @see Specification + */ +public class ShareUsageOptionsTests { + + /** + * Keeps what the element sends so a test can read it. + */ + private static class CapturingUploader implements DataUploader { + private final ByteArrayOutputStream sent = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return sent; + } + + @Override + public int getResponseCode() { + return 200; + } + + Document document() throws Exception { + return DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new InputSource( + new ByteArrayInputStream(sent.toByteArray()))); + } + } + + /** + * Counts the caches the tracker asks for. + */ + private static class CountingCacheBuilder implements CacheBuilder { + final AtomicInteger built = new AtomicInteger(); + private final LruPutCache.Builder inner = new LruPutCache.Builder(); + + @Override + public Cache build(Cache c, int cacheSize) { + built.incrementAndGet(); + return inner.build(c, cacheSize); + } + } + + /** + * Options for a usage sharing element that sends every request at once, + * as a configuration file holds them. + */ + private static PipelineOptions options(String... nameAndValue) { + ElementOptions element = new ElementOptions(); + element.builderName = "ShareUsage"; + element.buildParameters.put("SharePercentage", "1"); + element.buildParameters.put("MinimumEntriesPerMessage", "1"); + element.buildParameters.put("RepeatEvidenceIntervalMinutes", "0"); + element.buildParameters.put("BlockedHttpHeader", "x-blocked"); + for (int i = 0; i < nameAndValue.length; i += 2) { + element.buildParameters.put(nameAndValue[i], nameAndValue[i + 1]); + } + PipelineOptions options = new PipelineOptions(); + options.elements.add(element); + return options; + } + + private static Map evidence() { + Map evidence = new HashMap<>(); + evidence.put("header.user-agent", "agent"); + evidence.put("header.x-blocked", "blocked"); + evidence.put("header.cookie", "a=b"); + evidence.put("cookie.plain", "cookie"); + evidence.put("session.plain", "session"); + evidence.put("query.plain", "query"); + evidence.put("query.id.email", "a@example.com"); + evidence.put("id.email", "a@example.com"); + evidence.put("id.salt", "salt"); + evidence.put("server.request-path", "/path"); + evidence.put("server.request-query", "?plain=query"); + return evidence; + } + + /** + * Builds the element from the options, gives it the Evidence and returns + * what it sends. + */ + private static Document send( + PipelineOptions options, + Map evidence) throws Exception { + CapturingUploader uploader = new CapturingUploader(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .buildFromConfiguration(options)) { + ShareUsageElement element = + pipeline.getElement(ShareUsageElement.class); + assertNotNull(element); + element.dataUploader = uploader; + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + assertNotNull(element.getSendDataFuture()); + element.getSendDataFuture().get(); + } + } + return uploader.document(); + } + + private static boolean shared( + Document sent, + String category, + String name) throws Exception { + return !XPathFactory.newInstance().newXPath().evaluate( + "//Device/" + category + "[@Name='" + name + "']", + sent).isEmpty(); + } + + @Test + public void defaults_ShareOnlyWhatTheRulesAllow() throws Exception { + Document sent = send(options(), evidence()); + + assertTrue(shared(sent, "header", "user-agent")); + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "session", "plain")); + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() + throws Exception { + Document sent = send(options("ShareAllEvidence", "true"), evidence()); + + assertTrue(shared(sent, "header", "user-agent")); + assertTrue(shared(sent, "header", "x-blocked")); + assertTrue(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "session", "plain")); + assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "id", "email")); + assertFalse(shared(sent, "id", "salt")); + assertFalse(shared(sent, "header", "cookie")); + assertFalse(shared(sent, "server", "request-path")); + assertFalse(shared(sent, "server", "request-query")); + } + + @Test + public void shareAllEvidence_False_FromConfiguration_KeepsTheRules() + throws Exception { + Document sent = send(options("ShareAllEvidence", "false"), evidence()); + + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void shareAllQueryStringParameters_FromConfiguration_SharesEveryParameter() + throws Exception { + Document sent = send( + options("ShareAllQueryStringParameters", "true"), + evidence()); + + assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "server", "request-query")); + // The other rules still apply. + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "session", "plain")); + assertTrue(shared(sent, "header", "user-agent")); + } + + @Test + public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules() + throws Exception { + Document sent = send( + options("ShareAllQueryStringParameters", "false"), + evidence()); + + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void builder_ShareOptionsDefaultToFalse() { + ShareUsageBuilder builder = new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()); + + assertFalse(builder.isShareAllEvidence()); + assertFalse(builder.isShareAllQueryStringParameters()); + } + + @Test + public void keysAreReadWithoutRegardToCase() throws Exception { + Map evidence = new HashMap<>(); + evidence.put("Server.Client-IP", "1.2.3.4"); + evidence.put("Query.Session-ID", "session-1"); + evidence.put("Query.Sequence", "5"); + + Document sent = send(options(), evidence); + + assertEquals("1.2.3.4", + sent.getElementsByTagName("ClientIP").item(0).getTextContent()); + assertEquals("session-1", + sent.getElementsByTagName("SessionId").item(0).getTextContent()); + assertEquals("5", + sent.getElementsByTagName("Sequence").item(0).getTextContent()); + } + + @Test + public void tracker_BuildsOneCache() { + CountingCacheBuilder builder = new CountingCacheBuilder(); + + try (ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(builder, 10), + 1000, + new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME))) { + assertEquals(1, builder.built.get()); + } catch (java.io.IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java new file mode 100644 index 000000000..780cb80bb --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java @@ -0,0 +1,133 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A send through an {@link HttpClient} that never answers must end, or it + * would stop all sending. + */ +public class ShareUsageSendTimeoutTests { + + /** + * Opens a connection that records the timeouts set on it. + */ + private static class RecordingHttpClient implements HttpClient { + private HttpURLConnection connection; + + @Override + public HttpURLConnection connect(URL url) { + connection = new HttpURLConnection(url) { + @Override + public int getResponseCode() { + return 200; + } + + @Override + public String getResponseMessage() { + return ""; + } + + @Override + public void disconnect() { } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public void connect() { } + }; + return connection; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return ""; + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return ""; + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return ""; + } + } + + @Test + public void ShareUsageElement_LegacySend_ConnectionHasTimeouts() + throws Exception { + RecordingHttpClient client = new RecordingHttpClient(); + ShareUsageElement element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME, + flowData -> true); + element.httpClient = client; + try { + element.legacySendAsXML(new ArrayList<>()); + + assertEquals( + element.httpSendTimeout, + client.connection.getConnectTimeout(), + "The connection has no connect timeout."); + assertEquals( + element.httpSendTimeout, + client.connection.getReadTimeout(), + "The connection has no read timeout."); + assertTrue(element.httpSendTimeout > 0); + } finally { + element.close(); + } + } +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java index fd16b2c1b..348f613ac 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java @@ -70,14 +70,26 @@ public DataKeyedCacheBase(CacheConfiguration configuration) { */ protected abstract EvidenceKeyFilter getFilter(); + /** + * Creates the key the internal cache holds the data of a {@link FlowData} + * under. By default this is the key the {@link FlowData} generates from + * the Evidence that {@link #getFilter()} includes. A class whose key is + * made from other values overrides this method. + * @param flowData the {@link FlowData} to make a key for + * @return the key to use in the internal cache + */ + protected DataKey generateKey(FlowData flowData) { + return flowData.generateKey(getFilter()); + } + @Override public V get(FlowData flowData) { - return internalCache.get(flowData.generateKey(getFilter())); + return internalCache.get(generateKey(flowData)); } @Override public void put(FlowData flowData, V value) { - internalCache.put(flowData.generateKey(getFilter()), value); + internalCache.put(generateKey(flowData), value); } @Override diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java index 38df1b000..b790a6415 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java @@ -44,5 +44,12 @@ public interface DataUploader { */ int getResponseCode() throws IOException; + /** + * @return the content of the answer, which is logged when the code is + * not 200, or an empty string if the uploader cannot give it + */ + default String getResponseContent() { + return ""; + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java index 777584bf5..9ad54e674 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java @@ -23,9 +23,11 @@ package fiftyone.pipeline.engines.services; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.zip.GZIPOutputStream; @@ -34,13 +36,19 @@ public class DataUploaderHttp implements DataUploader{ private final String url; private final Map headers; private final int timeout; + + /** + * The most characters of an answer's content that are given. + */ + private static final int LOGGED_CONTENT_LENGTH = 500; HttpURLConnection connection; /** * the URL to which data is to be uploaded * @param url the URL rto which data should be uploaded * @param headers HTTP headers for the connection - * @param timeout connection timeout in milliseconds + * @param timeout milliseconds to wait to connect, and then to wait for + * an answer */ public DataUploaderHttp(String url, Map headers, int timeout) { this.url = url; @@ -55,6 +63,7 @@ public DataUploaderHttp(String url, Map headers, int timeout) { public OutputStream getOutputStream() throws Exception{ connection = (HttpURLConnection) new URI(url.trim()).toURL().openConnection(); connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); connection.setRequestMethod("POST"); for (Map.Entry header : headers.entrySet()) { connection.setRequestProperty(header.getKey(), header.getValue()); @@ -67,5 +76,28 @@ public OutputStream getOutputStream() throws Exception{ public int getResponseCode() throws IOException { return connection.getResponseCode(); } + + /** + * Reads the content of a failed answer, as far as the most characters + * that are logged. + */ + @Override + public String getResponseContent() { + try (InputStream error = connection.getErrorStream()) { + if (error == null) { + return ""; + } + byte[] content = new byte[LOGGED_CONTENT_LENGTH]; + int length = 0; + int read; + while (length < content.length && (read = error.read( + content, length, content.length - length)) > 0) { + length += read; + } + return new String(content, 0, length, StandardCharsets.UTF_8); + } catch (IOException e) { + return ""; + } + } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java new file mode 100644 index 000000000..d9094b438 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java @@ -0,0 +1,132 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.sun.net.httpserver.HttpServer; + +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.util.HashMap; + +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * An upload to a service that accepts the connection and never answers must + * end, or it would stop all usage sharing. + */ +public class DataUploaderHttpTests { + + /** + * How long the uploader waits for an answer. + */ + private static final int TIMEOUT_MILLIS = 500; + + /** + * How long the test service holds a connection open without answering. + */ + private static final int HOLD_MILLIS = 30000; + + /** + * The test fails if the upload has not ended by then. + */ + private static final int LIMIT_MILLIS = 15000; + + /** + * The JDK decides whether a request may carry a restricted header, Origin + * among them, from this property when it first loads its HTTP connection + * class, and the value it reads then stands for the rest of the run. + * HttpClientDefault.connect sets it and relies on being the first to open + * a connection. These tests open one as well, so they set the same value + * first and leave that class working whichever test runs earlier. + */ + @BeforeClass + public static void AllowRestrictedHeaders() { + System.setProperty("sun.net.http.allowRestrictedHeaders", "true"); + } + + @Test(timeout = LIMIT_MILLIS) + public void DataUploaderHttp_NoAnswer_TimesOut() throws Exception { + try (ServerSocket server = new ServerSocket(0)) { + Thread holder = new Thread(() -> { + try (Socket socket = server.accept()) { + Thread.sleep(HOLD_MILLIS); + } catch (Exception ignored) { + // The test has ended and closed the server. + } + }); + holder.setDaemon(true); + holder.start(); + DataUploaderHttp uploader = new DataUploaderHttp( + "http://localhost:" + server.getLocalPort() + "/usage", + new HashMap<>(), + TIMEOUT_MILLIS); + try (OutputStream out = uploader.getOutputStream()) { + out.write(new byte[] {1}); + } + + try { + uploader.getResponseCode(); + fail("The upload was answered."); + } catch (SocketTimeoutException expected) { + // The wait for an answer ended. + } + } + } + + @Test(timeout = LIMIT_MILLIS) + public void DataUploaderHttp_FailedAnswer_GivesTheContent() + throws Exception { + byte[] answer = "the service says no".getBytes(StandardCharsets.UTF_8); + HttpServer server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext("/usage", exchange -> { + exchange.sendResponseHeaders(500, answer.length); + exchange.getResponseBody().write(answer); + exchange.close(); + }); + server.start(); + try { + DataUploaderHttp uploader = new DataUploaderHttp( + "http://localhost:" + server.getAddress().getPort() + "/usage", + new HashMap<>(), + TIMEOUT_MILLIS * 10); + try (OutputStream out = uploader.getOutputStream()) { + out.write(new byte[] {1}); + } + + assertEquals(500, uploader.getResponseCode()); + assertEquals( + "the service says no", + uploader.getResponseContent()); + } finally { + server.stop(0); + } + } +} From bf66e3d167dce9d85ec6a5ea94a0396587ad6f15 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 17:24:43 +0000 Subject: [PATCH 78/95] Show the root logo in every README, link the main contributing guide, and fix the README links (#49) * Show the root logo in every README, link the main contributing guide, and fix the README links Six READMEs showed an older 51Degrees wordmark from 51degrees.com rather than the logo the root README shows. Each now shows the root logo as a Markdown image from raw.githubusercontent.com, using .github/images/logo-160.png, a 160 pixel copy of .github/images/logo.png, the size the root README shows it at. The root CONTRIBUTING.md linked the guide in common-ci. It now links the 51Degrees contributing guide, which leads on to the full guide in common-ci. The Developer Documentation link at the top of six READMEs went to the main page of the product's generated API reference, which has no content. They now go to the product's documentation. The root README's table links both that and the class list of the Java API reference, and the pipeline README's sentence about the API reference links the class list. The dependencies page that the Device Detection and IP Intelligence READMEs linked is gone, and the next sentence in each already links the tested versions page, so the sentence is removed. The location README's Maven link went to search.maven.org, which answers 403, and now goes to the artifact on central.sonatype.com. * Link each product's quick start from the READMEs The documentation links went to section pages of the 51Degrees documentation, such as _device_detection__index.html, which only list the pages in the section. They now go to the page in each section that is most helpful to a developer and the most recently kept up to date. Those are the Device Detection, IP Intelligence and Reverse Geocoding quick starts, which show how to choose between the cloud and on-premise and how to start in each language, and Pipeline Basics for the Pipeline API, which has no quick start. The campaign tags of each link are unchanged. --- .github/images/logo-160.png | Bin 0 -> 7620 bytes CONTRIBUTING.md | 5 +++-- README.md | 8 ++++---- device-detection/README.md | 5 ++--- examples/device-detection/README.md | 4 ++-- examples/ip-intelligence/README.md | 4 ++-- ip-intelligence/README.md | 5 ++--- location/README.md | 6 +++--- pipeline/README.md | 6 +++--- 9 files changed, 21 insertions(+), 22 deletions(-) create mode 100644 .github/images/logo-160.png diff --git a/.github/images/logo-160.png b/.github/images/logo-160.png new file mode 100644 index 0000000000000000000000000000000000000000..78952d50981b33d2283303662741f634132d74a8 GIT binary patch literal 7620 zcmbuEbxd4O`0f{9e6g~)6xSA~xI4w&ij-0u%HmR3oZ?!n#fodug=KMvQi?8C+^x7? zzPY)%`Tcj3o18f(lbOkTCNs}7^X3z+rKyCELx}?b0Pt0m<#nFF-~KB=%;$IPggrL^ zKohGXFQezPaG2|xNv+@dB(<$~vCO~Jz?=n$fU3w6%L1@rm5lUJ3|t(BAaYU+3{nMc z`QR8fwr^~HMCj)~LJG_jKu~ZsS16QQ1_(_IB<9O1hW(f|8Gk4&_iwXTVR_HOpY$&< z&u`~kacuW(2hw+fy$htHN+Amz2&TqZ#-%1az*0vemIiQh*M^Xuu$-Wt08hSw7+3O( zABn?4SOM%%w^SW`#0_HLGlgGHW0PDBS~H3ix$Gy-Vmt(yY$*fE`CmEU_f#Ew)L9hW zVP3TACyLkLb`w{}vT`|t(>luQ0?^v>M>rl@g zAm>=U0mXc5-kIt7y{B zjq5`QQHAd+RRG%KTB$m8m_eH`Gu?#j8Me~V8By7&rZa9C+{Vv9}P4Vv;WF)1wKbHq}ys>jrFi~}rcuocHhYb8X_ zfJ>QDbyO3=W{m8aOiU=Hx$(cHr({TC3>`13j7^RM*#Gw7m~_8HA2 zJlAbL02+*+rwksvaM7}r_=h)BZznkM+!>OGA+Oc`G_lS|`v(B0kwdrv_ruBCgTWEc zQ)PYMsrnb<_19?*&?K>CpoMlVG#XMxZgIl30ydSxUqz1HVt*dqW?z1?N4+~(PqQ)e zRv+P`L^Q`zV-+!Z575lHw#!@x{)mfZ>Rqhn9Mpel2}(n@+?rv0jMDj>N{?Q>PcWIp z#X;T-T02B`+9R#L8u7yAFK?-Uqxo-7b_zRI^oGsRr5O+)F>WnIc3#CK)H%OO#z@g9 zJFRd~&*Nw{VLn<(bc3H@nHsNxT|+N#E12ku-%+z4b%3x09B}@2c)o?=?jj8YJZPDJ zf$D^7bH+5_($y%}tN~_o$8l zj6T*!mDAH7bru%!$dog)x*as5Oh`jF9of>$}y~&;)d}9>^FIxw zbitVZhkklOkw8q)-&cHDU@IB@B0!1v?O(S{BQ8ZI;xg1)!gQUo-u^FC#vOiwS5hZ{ z3okEfza55v>jOgfHEdRTD8fDG3D41}$w@ZxVd(AP(!H)8i`+?VkMjLaWRzPAF$rp^ zS$uc?Qyk3w<@fajB_V+WJwtIPz|wMM242f!T_Mk0J<;T=R?8K&+daC1dQrV0S$Wwg zE#TzbFK^~6EDPYV=!#FqlEV#{OA!w<4BGQ+wb7@ z9{f&XEhBg+%oR_SZ|ekQ)MdKZr3RP9s=z7x#6O;KDHEF9h(^0qN*&hs&JNf3&fk|_ zh>6&!X{;|JC=Y*s_Fvx!ymr6Uh7SHanYoDfmdnMS6)M}y(-t+SKyDB-HlxOy^ms;Q zH1U{nU7P{xT#J!(U4tFE<9!^c>*C1Xt>P}rBa-(4HNT6~SXY{r! z2m-k>y_o81o`OlX_+Z*;drF*0>_}Nu)}UnCt%;5@mzzn}8`2CKZJn}IW?xXXS>kb03{ zL!g8I4LPKPtNgKiB#~RAhtMKiI!!fG_eV%7I$@?-(3l?-Sa^n^_kD*f-sbhI4N$d= zPZq1&veNiw6}Kr69aRv+VWfWN;J|kGw_SVdN3w9)&(CWDwQN6fWyt>e_4=rB#lEIV zNK<0d)@+t$NeW`hw@uD?WyN}U`+1m0NHHk0~q|d<+Dx(@pp3-(r%$8RA)I}JT z<5iLo1@0H&mFJXNHN+84IrJ_DO7RmAiQWm=>VI+eC(9I$Jm5X*jv9FN<51-!q!pbY z2P>ZOV)~d~wyUb2HUIt(YxC_xpLKMxT8780#MQ~2)MBJgj53ps4gQE-sxs54VJCzc zi`ETLRh7~&3=JwPzPhDyLn0Z#Og`t6y^?__!%ak=0Y_V7MK$+tHawu@JeA?D)Tv+8 zn>6T$>$T={ggLeGL-!f2wQHB)m7iLIQ!TanUjgZY1%|Ram3F?Ny(&4Wz&*3W{a{h2ddOMg3mFtyD zIG_xBB_+Ko;MwXF8l+=pQZB4{_z{p9cqrXZn-ZPt$+9)tA%3m;LWF?Bl6pp3pOqnu z#*Tbngdma*!qy4fcDY<|@3PsC@N-bVVaRna=ZupGFJ>d2n?nv7;lv*bt z+I=(qC(b$Wf^sMPAlT?(^vsNuNI`ODb|mJ=UIz460Y=ftr|Y_b!U-8@}6o z<29(S@2z!>u_ZeR{pM<;1jDTb-Dbsj{58OUe*lZD{pAaWvHTd{TiF zCp?CX#v8y9{@RysYm`EZtDsUR-~YnAsx7TiSRIuVJ*GLtUzH4O?hXR_|%UyMa^MI|EI5*}({=OqN#IBO;w&>Zs9BMf6Og8YwCv?79-@ z9w~xVOnP`oHBYv1xDUU3SY9Ab0j;yNF%8KbYn9p7f-EI zNd|jJ3kj#RKR_xoEEt2ga|4g2y`uB3h!}BStU@4+`+Kz+yaPd-Y32ZFsjVap{3v;v za#np)L~4O$(>|RJf%(o)E_uGs9oA%7jzI$N!?fbJ3nq6$C{h=u8v8pjQi-XtJTJMD zy_aQqB-17@nX~Z(YqKX(A}%* z!K08%VS6ft9vFIcP>vVcSr&^&w4G}_Sm9biOoLCS43%~@@eblfkUrRg zji&#O^3u*N%HJ#Z!500T%yUBZ3@ScdsEc{DaVjaV?8nNpK31{2uuacm{~&FuQ2e6n z*|HmC^`1(@`yp=GG)UoCqq*RBM`nXPvA7jam-l$jTn8B{OAAhOVai&{Q_=O{)0$HN zt~NOxFq1+1M*QZE?vigkH-I_)`~Kcui5>@j=2y;6))%o~@{0`F%`i&NUzkQeh6hgp zkSw3pu&tDF2ql#_&Zj!?EZA*zNjTCNJsw%xFDn^1h!m$4?CClF;TBH*_(%ZT+GwN4 zIK`Ee-5Hc~6ZQg$uUW~vom?KC_$TzVo)i|rJp`X6%(zx(0yGZ?hOr;&Lw`yc?%K zgkpmOGj|loi6ypkD6jP>c)nS|?(KY8ImJB0J0vw+$KQ9sxpTxi(~J;cci<+ZkK*w_ z$?8&Fc7XP8m*KeIQn`!600y!vQ#?8%-#k*(hO%4>T#KrfiRoe}?80H54LR-wyA35X zXKo?DTI>q+Agl2MW3&Yr#II4ouj%OKfa}i_Ue|lww%!f-sJ6r|c$J0yRM9QH zl$ed{-b$&#VaBU5Wm6?bIU5tP-lKI=UJp?wgY9pDWp)INm;j1>CfoaI6dg3)1AoxI#LG@Q+5spyZ zx7VMa(m1u|3SE#o8rL{y!=wf?59Y|wP0%YQ23=x8il2fCnZm8{vGH?Z-V+~+L|>G; z@R#Y$qlbQD(278~llhJu+UI*?h6E)i-VRTfCM%pRa7xVXiv+=@L9A>jjo@P1hsmbF~a^YsYT(I2OBm*-bfqw=P68}5sEOBQz6$~X2X3_QvL6qEf> z^-5em2HGM9>))ifzew@%9Nh1YW2{xL@5)Iv%axIT(4EJKR%ByxH$q2nr(Eyw=f#ds zxw_l%nc!ZcTcb_o^u080_$0v9_Gy{N>bu0cAmduV=GDIPxCF!#jU-$c$5b-g`SG?L72mFexIoVlJ3Eua!4W!9rDgKdfGt!+v zB(aTGVBbOKxsWQ8lJpyQMA`C`Ca3ugXi`}Uz+7&`==1B*LaQBZiK1f{&t*Qh;^TyC zifXY@>zM|v`Om0G+DA?*VHc((WjvD*A&r#rYM$piF6nacUfH_!emoy|v!Iq8<87s; z`%`DrNu&2$23mm*+1HJIYxQAV1CbF2oPWWxR(gm3rdE9CBm;0n_Q&no>*}*dCg!pm zuJWTQ6bax}Xhs&!7kfB`3F{YTloE9!S3K_x)$;tE2`o)t#yt`c+BnYiFHgSdk2m(`g6A!VhlnPPGbo; zAj-q|G`*?|>CxjzE2*e(M9ffcYVP=mTNq;tlLqXne^nu2J&iSWeb)_5KWzq=eavoE zp0j<3uy}9zzyuRi%E|#wlH&gY393YBsISOwa%aRlMd!6^7>>UWESL6n#r6rXbnC~$(q z*&L*{+qfAkV?pOcMabS&pH#q3&yG2at=NFn4j?k6p(?=NETA`qK5 z1qepBL0MqwoojeKDoL_V%~zZ^Phzcp!oTVzfpO&P&fiqwvl03*|B|A*H|*suq#6I) zN+noyo%9t`87Dk!_?&U?L3C+P=elX3?l!Qdo=cDCZBIgVe~w_F{F}m&)5RuA-0~_$ zX`$@CCYtd#|31Ml#$12R&52Z#7iCaIZmGw)U-)04TB*%g+q4ZXjpsz+gAr$T1MRRbYG%n}|8{;!cG$+x$S zsd+);Nbyqf!q+(+ro#%3#csL|w&IFs{jNmDK4&Mygrb8y~m|w2l$nc$JL_JCAqf^ujSv*+Wr^TW@^apG*UhqBp@j0ey*+Lu@H*! zpOt2o7-xM&l0h^szG9-6g^o2pwZ&W=PIV_uB_osc{Sj&FyYFpxkpR88?HCPAC~Nc=wj+)5dqf&0J@T z!zAE)WL}gqaB(5Tj4T#()+WAu8t<*;LEZh=ZH!hBC#tXDK+u^|w?zHj%rgJH>vE8K z-l9o^%A`-TpMEyH;G}XTtJ5z_>9Ou}UxiM4uqOXh_RE7LyeaVOXbgNjxz)Cq_QT|6 zGXs~Q21$LZp<6sEf1H5()g_~rV|3Sa8@z1prVhD1datfwA6rk6!|Yna>aOQ>gWK$k zwfoCVbS8hvyMvj%M(Yb?t7l6FY(cdm^#oS!JW&0U+rCurJh`y3G~0 zIbR}@f?NC8p2#%+k6IUr`+ zc=MgQ2LM51;e*!ejd&p{r^9o3eQN&RJH^6GaT9<6tXqBZuQxBe&(dKck_11Z@>U+8$hqyh z`+M}aV~4%3pYGe`(|XtrbEHlRp33v^E$+v>d|Cbh#$}wb^?kO@=c7gp=SqjYU67F} z;z=6U2=b%Gdqd=;-+rhl_Hj=p!&VJygQOJoQI%*gu~v7E`9(~ah?gx!3bRuqaJ5tY zwC!x*O=dKmYwu;4M@A#+=OM&>3Sd228lA^dFPd~+@vC~NBsXU(Ywp%dOoG2E;@L&T z&-{2c{KCTNc=KXh+!eCrW1~i!)^MwC*UR)ms0^1vKoRZv=?zPELzFrx8pN-w#;cXA zT5&4CH-6_fE0fc34HR3!T6hHH)gfxoxf;Q7*rlq+s0F^ddd%W!x)jpw9oeGAmzd&@UF%d^(GiCoBvBwKD`*N)+Ky~ zInTTPdm=6Rxq4K3Jj(x}EdLRoXW;Wchx~5<^FN#ZpIA^K`nRe`8OJzV75bjbL)WU} zKHw5M+z{1K@EPHtubN%ZAR(@LJt%>GV)#8Z25lsST{4YQzG^ac zzo!{<&Jc&e@3ZL+Y^*YU<^#+UWswm&2rNzoBqKNx^3E8IkkAVx`Y4KDk@t$&B!0kn z@Z|J*k>D^OXxv^?l>u7#%+x@gXf6n3%i?dFV!`hEfF>7^?BYKEHt-38_DCZgUYOcO zbq!d?i~e(Dor>KsrJ@*@Q6^A9L!{*miq_6cRH Date: Wed, 11 Nov 2026 17:37:42 +0000 Subject: [PATCH 79/95] Give the specification's JSON Builder error text with no sequence number, and test the behaviour (#104) * Take the sequence number check out, to show the new tests failing The JSON Builder specification says the element MUST fail with an error and MUST NOT set its JSON value when no Sequence Element has produced a sequence number. JsonBuilderElement already does that, and nothing tested it, so the regression audit for this release read it as missing. This commit adds the tests and takes the check out, so the build shows them failing. The next commit puts the check back, unchanged, and the tests pass. What reaches rc is the tests alone. * Put the sequence number check back The check is restored exactly as it is on rc, so what this draft adds to rc is the tests alone. The previous commit took it out only to show the build failing without it. * Test that the cause says a Sequence Element has to come first The JSON Builder specification says the error SHOULD say that a Sequence Element needs to be placed before the JSON Builder Element in the Pipeline. The exception the pipeline records says so, and the cause it carries does not, reading "Sequence number not present in evidence. this is mandatory." This test pins the cause's wording, and fails until the message changes. * Say in the cause that a Sequence Element has to come before the JSON builder The cause read "Sequence number not present in evidence. this is mandatory.", which neither says what to do nor reads as one sentence. It now carries the wording 51degrees-go and 51degrees-rust use, so the same misconfiguration reads the same whichever language a deployment uses. The exception the pipeline records is unchanged. * Pin the error message the JSON Builder specification gives specifications#139 turns the SHOULD into a MUST and gives the exact text, so that the message reads the same in every language. It also says that where an implementation wraps the error in another, the one the caller reads carries the message, which in Java is the PipelineConfigurationException rather than its cause. The constant holds the text and the tests assert it word for word on both the recorded error and the cause. They fail until the element uses it. * Give the specification's message when there is no sequence number The PipelineConfigurationException the caller reads said "Make sure there is a SequenceElement placed before this JsonBuilderElement in the pipeline" and its cause said something else again. Both now carry the text the specification gives, which every language carries. One more test writes that text out beside the constant, so that changing the constant fails rather than moving the other assertions with it. * Pin the JSON Builder links at the specification commit that merged specifications#139 merged at 13:19 as d72eb074, with one more commit saying that the message is the whole text of the error and that a prefix a language's error type writes is not part of it. The links moved from the draft head and from 67ead698, which predates the message. No code changes. * Say in the helper what it now asserts The helper checked that the message named a sequence element and now checks the whole message, so its note says so. --- .../pipeline/jsonbuilder/Constants.java | 14 +- .../flowelements/JsonBuilderElement.java | 10 +- .../jsonbuilder/JsonBuilderSequenceTests.java | 268 ++++++++++++++++++ 3 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java index e1c422eeb..f72d41ee8 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java @@ -25,5 +25,17 @@ public class Constants { public static final Integer MAX_JAVASCRIPT_ITERATIONS = 10; - + + /** + * What the element says when no Sequence Element has produced a sequence + * number. The + * JSON + * Builder specification gives this text and every language gives the + * same, so that the message reads the same whichever one a deployment + * uses. + */ + public static final String NO_SEQUENCE_NUMBER_MESSAGE = + "The JSON Builder needs a sequence number. Place a Sequence Element " + + "before the JSON Builder in the Pipeline."; + } diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java index fcf920332..72f037049 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java @@ -203,9 +203,10 @@ private String buildJson(FlowData data, PipelineConfig config) throws Exception try { sequenceNumber = getSequenceNumber(data); } catch (Exception e) { - throw new PipelineConfigurationException("Make sure there is a " - + "SequenceElement placed before this JsonBuilderElement " - + "in the pipeline", e); + // The caller reads this one, so it carries the message the + // specification gives, and the cause carries the same. + throw new PipelineConfigurationException( + Constants.NO_SEQUENCE_NUMBER_MESSAGE, e); } Map allProperties = getAllProperties(data, config); @@ -225,8 +226,7 @@ private int getSequenceNumber(FlowData data) throws Exception { "query.sequence", Integer.class); if(sequence.hasValue() == false) { - throw new Exception("Sequence number not present in evidence. " + - "this is mandatory."); + throw new Exception(Constants.NO_SEQUENCE_NUMBER_MESSAGE); } return sequence.getValue(); } diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java new file mode 100644 index 000000000..0a0329e18 --- /dev/null +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java @@ -0,0 +1,268 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.jsonbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; + +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The JsonBuilderElement needs a sequence number, and fails without one. + *

    + * The + * JSON + * Builder specification says the element MUST fail with an error and MUST + * NOT set its JSON value when no Sequence Element has produced a sequence + * number, that a query.sequence value taken from a query string is text and so + * does not count, and that an integer a caller added to query.sequence does + * count in Java because the element reads the integer in Evidence whoever + * added it. + *

    + * The error MUST carry the message the page gives, which is + * {@link Constants#NO_SEQUENCE_NUMBER_MESSAGE}, and where the error is wrapped + * in another the one the caller reads carries it. + */ +public class JsonBuilderSequenceTests { + + private final TestLoggerFactory loggerFactory; + + public JsonBuilderSequenceTests() { + ILoggerFactory internalLogger = mock(ILoggerFactory.class); + when(internalLogger.getLogger(anyString())).thenReturn(mock(Logger.class)); + loggerFactory = new TestLoggerFactory(internalLogger); + } + + /** + * Build a pipeline of an engine and the JSON builder, with a sequence + * element in front when asked for. + * @param withSequenceElement whether to place a sequence element first + * @param suppress whether the pipeline suppresses processing exceptions + * @return the built pipeline, which closes its elements + * @throws Exception if the engine or the pipeline cannot be built + */ + private Pipeline buildPipeline( + boolean withSequenceElement, + boolean suppress) throws Exception { + EmptyEngine engine = new EmptyEngineBuilder(loggerFactory).build(); + JsonBuilderElement jsonBuilder = + new JsonBuilderElementBuilder(loggerFactory).build(); + PipelineBuilder builder = new PipelineBuilder(loggerFactory); + if (withSequenceElement) { + SequenceElement sequenceElement = + new SequenceElementBuilder(loggerFactory).build(); + builder = builder.addFlowElement(sequenceElement); + } + return builder + .addFlowElement(engine) + .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) + .setSuppressProcessException(suppress) + .build(); + } + + /** + * Process one request with the pipeline's exceptions suppressed, and + * assert that the element recorded one error carrying the specification's + * message and set no JSON. + * @param sequence the query.sequence Evidence to add, or null for none + * @throws Exception if the pipeline cannot be built + */ + private void assertFailsWithoutASequenceNumber(Object sequence) + throws Exception { + try (Pipeline pipeline = buildPipeline(false, true); + FlowData flowData = pipeline.createFlowData()) { + if (sequence != null) { + flowData.addEvidence("query.sequence", sequence); + } + flowData.process(); + + Collection errors = flowData.getErrors(); + assertEquals(1, errors.size(), + "Expected the JsonBuilderElement to record one error."); + String message = errors.iterator().next().getThrowable().getMessage(); + assertEquals(Constants.NO_SEQUENCE_NUMBER_MESSAGE, message, + "Expected the error the caller reads to carry the message " + + "the specification gives, word for word."); + + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNull(result == null ? null : result.getJson(), + "Expected no JSON to have been set."); + } + } + + /** + * The message is written out here as well as in the element, so that + * changing the element's constant fails this test rather than moving the + * other assertions along with it. + */ + @Test + public void JsonBuilder_TheMessage_IsTheOneTheSpecificationGives() { + assertEquals( + "The JSON Builder needs a sequence number. Place a Sequence " + + "Element before the JSON Builder in the Pipeline.", + Constants.NO_SEQUENCE_NUMBER_MESSAGE); + } + + /** + * With no sequence element and no Evidence there is no sequence number, + * so the element fails and sets no JSON. + */ + @Test + public void JsonBuilder_NoSequenceElement_Fails() throws Exception { + assertFailsWithoutASequenceNumber(null); + } + + /** + * A query string carries text. Text is not a sequence number, so the + * element fails as it does with no Evidence at all. + */ + @Test + public void JsonBuilder_SequenceFromQueryString_Fails() throws Exception { + assertFailsWithoutASequenceNumber("4"); + } + + /** + * Text that is not a number at all fails the same way. + */ + @Test + public void JsonBuilder_SequenceThatIsNotANumber_Fails() throws Exception { + assertFailsWithoutASequenceNumber("abc"); + } + + /** + * The pipeline's own exception handling decides whether the error is + * thrown or recorded. With exceptions not suppressed the failure reaches + * the caller, and still no JSON is set. + */ + @Test + public void JsonBuilder_NoSequenceElement_ThrowsWhenNotSuppressed() + throws Exception { + try (Pipeline pipeline = buildPipeline(false, false); + FlowData flowData = pipeline.createFlowData()) { + AggregateException thrown = assertThrows( + AggregateException.class, + flowData::process); + // The aggregate carries each element's error as a suppressed + // exception. + assertEquals(1, thrown.getSuppressed().length, + "Expected one suppressed exception in the aggregate."); + String message = thrown.getSuppressed()[0].getMessage(); + assertEquals(Constants.NO_SEQUENCE_NUMBER_MESSAGE, message, + "Expected the error the caller reads to carry the message " + + "the specification gives, word for word."); + + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNull(result == null ? null : result.getJson(), + "Expected no JSON to have been set."); + } + } + + /** + * The error the element records carries the one getSequenceNumber threw + * as its cause, and the cause says the same thing, because it is the line + * a caller meets at the top of the stack trace. + */ + @Test + public void JsonBuilder_NoSequenceElement_TheCauseSaysWhatIsNeeded() + throws Exception { + try (Pipeline pipeline = buildPipeline(false, true); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + + Collection errors = flowData.getErrors(); + assertEquals(1, errors.size(), + "Expected the JsonBuilderElement to record one error."); + Throwable cause = + errors.iterator().next().getThrowable().getCause(); + assertNotNull(cause, "Expected the error to carry a cause."); + assertEquals(Constants.NO_SEQUENCE_NUMBER_MESSAGE, + cause.getMessage(), + "Expected the cause to carry the same message as the error " + + "the caller reads."); + } + } + + /** + * An integer a caller added to query.sequence is a sequence number, even + * with no sequence element, because the element reads the integer in + * Evidence whoever added it. + */ + @Test + public void JsonBuilder_IntegerFromTheCaller_BuildsTheJson() + throws Exception { + try (Pipeline pipeline = buildPipeline(false, false); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.sequence", 3); + flowData.process(); + + assertEquals(0, flowData.getErrors().size(), + "Expected no error with an integer sequence number."); + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNotNull(result, "Expected the JsonBuilderData to be present."); + assertNotNull(result.getJson(), "Expected the JSON to be set."); + } + } + + /** + * With a sequence element in front the JSON is built, which is the order + * the web integration assembles. + */ + @Test + public void JsonBuilder_WithSequenceElement_BuildsTheJson() + throws Exception { + try (Pipeline pipeline = buildPipeline(true, false); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + + assertEquals(0, flowData.getErrors().size(), + "Expected no error with a sequence element in the pipeline."); + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNotNull(result, "Expected the JsonBuilderData to be present."); + assertNotNull(result.getJson(), "Expected the JSON to be set."); + } + } +} From 528e0df31854b3d3b729c1e1561dc2f02c027c4e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 17:47:44 +0000 Subject: [PATCH 80/95] Give the pipeline developer examples the test resource key (#108) The integration step runs the pipeline developer examples' ExampleTests again without the key the unit test run passes, and the cloud engine example's test now fails when it has no key rather than skipping, so every full build failed there. The step now passes -DTestResourceKey as the example build below it does. --- ci/run-integration-tests.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 99621cbd9..74b354718 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -33,7 +33,10 @@ $failed = @() Write-Host "::group::Pipeline developer examples" Push-Location "$RepoName/pipeline" try { - mvn test @MavenOptions -DfailIfNoTests=false "-Dsurefire.failIfNoSpecifiedTests=false" -Dtest=ExampleTests + # The cloud engine example calls the live cloud service, so it is given + # the test resource key, as the unit test run gives it. + mvn test @MavenOptions -DfailIfNoTests=false "-Dsurefire.failIfNoSpecifiedTests=false" -Dtest=ExampleTests ` + "-DTestResourceKey=$TestResourceKey" if ($LASTEXITCODE -ne 0) { $failed += 'pipeline developer examples' } } finally { Pop-Location From c34b30d03d9cfd3632494485367e495ab59c80f0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 19:34:01 +0000 Subject: [PATCH 81/95] Move cxx to the 51degrees-cxx release 4.5.105 and read by index on an engine built with no names (#82) * Move cxx to the 51degrees-cxx rc commit 865cee97 * Remove the ownership workaround from the IP Intelligence tests The configuration's collections are borrowed. Test that each one the configuration hands out is borrowed, and stop forcing the flag by reflection in the two tests that read them. * Generate the IP Intelligence SWIG wrapper again for cxx 865cee97 SWIG 4.5.1 now marks the configuration's collection getters as borrowed, as the corrected MetaData.i asks. The Device Detection wrapper is unchanged apart from comments recording the include path, so it is left as it was. * Move cxx to the 51degrees-cxx rc commit b4f769ad Takes the eight cxx commits after 865cee97, from 51Degrees/51degrees-cxx#118, 51Degrees/51degrees-cxx#139, 51Degrees/51degrees-cxx#143, 51Degrees/51degrees-cxx#136, 51Degrees/51degrees-cxx#131, 51Degrees/51degrees-cxx#141, 51Degrees/51degrees-cxx#144 and 51Degrees/51degrees-cxx#145. * Move cxx to the 51degrees-cxx rc commit cfad8978 Takes the two cxx commits after b4f769ad, from 51Degrees/51degrees-cxx#130 and 51Degrees/51degrees-cxx#146. This is the cxx commit for the release. * Move cxx to the 51degrees-cxx release commit 7c4ac436 * Test that an engine built with no names reads by index An engine built with no names should take its names from the native engine after the first data file loads, give every property of that file an index the native results agree with, and keep those indexes through a reload from a file whose properties are a superset in another order. The read by name against the read by index is measured for such an engine as it is for one built with names. * Read by index on an engine built with no names 51degrees-cxx 7c4ac436 keeps the first data file's property names, in their order, across every reload of an engine created with no names, and its C++ reads by index work on such an engine. The engine therefore makes its map of names to indexes once, from the native engine's required properties after the first data file loads, and never again. A property only a later data file has is not in the map, so it reads by name and gives no value until the engine is built again. --- cxx | 2 +- .../flowelements/DeviceDataHashDefault.java | 2 +- .../DeviceDetectionHashEngine.java | 38 +++- .../flowelements/PropertyIndexTests.java | 193 ++++++++++++++++-- .../IPIntelligenceDataHashDefault.java | 2 +- .../IPIntelligenceOnPremiseEngine.java | 38 +++- .../onpremise/interop/swig/ConfigIpiSwig.java | 8 +- .../onpremise/flowelements/BuilderTests.java | 75 ++++--- .../flowelements/PropertyIndexTests.java | 103 +++++++++- .../OnPremisePerformanceProfileTests.java | 36 +--- 10 files changed, 414 insertions(+), 83 deletions(-) diff --git a/cxx b/cxx index c6d4e4d90..7c4ac436e 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit c6d4e4d907e8da83845c095e80374280f8db29c5 +Subproject commit 7c4ac436e47cd15beed304cad2b8151d50a1efd1 diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index 3dad6384f..691761125 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -94,7 +94,7 @@ public class DeviceDataHashDefault /** * Get the required property index for a property name, or null when the - * engine was not built with that name, so the read falls back to the name. + * engine has no index for that name, so the read falls back to the name. * @param propertyName name of the property * @return required property index, or null */ diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java index 2da75263e..0b5c93fb1 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java @@ -68,10 +68,20 @@ public class DeviceDetectionHashEngine * list, the first where a name is given more than once, as the native * engine gives it. Made from the names given at build and never from the * data set, so an index keeps naming the same property after a reload - * from a data file whose property order differs. Empty when the engine - * was built with no names, in which case reads go by name. + * from a data file whose property order differs. When the engine was + * built with no names it is made instead from the names the native engine + * reports after the first data file loads, which are the properties of + * that file sorted by name and which a reload does not change. */ - private final Map propertyIndexes; + private volatile Map propertyIndexes; + + /** + * True once {@link #propertyIndexes} holds the names the indexes are + * fixed on, which is at construction when the engine was built with + * names and after the first data file loads when it was not. A reload + * never makes the map again. + */ + private volatile boolean propertyIndexesFixed; /** * Construct a new instance of the {@link DeviceDetectionHashEngine}. @@ -100,6 +110,7 @@ public class DeviceDetectionHashEngine this.config = config; this.propertiesConfigSwig = properties; this.propertyIndexes = indexesOf(propertyNames); + this.propertyIndexesFixed = this.propertyIndexes.isEmpty() == false; addDataFile(dataFile); } @@ -130,7 +141,9 @@ public MetaDataSwig getMetaData() { /** * Get the required property index for a property name, or null when the - * engine was not built with that name and the read has to go by name. + * index list has no such name and the read has to go by name. That covers + * a name the engine was not built with, and a property only a data file + * loaded after the first has. * @param propertyName name of the property * @return required property index, or null */ @@ -138,6 +151,22 @@ Integer getPropertyIndex(String propertyName) { return propertyIndexes.get(propertyName); } + /** + * Fix the indexes on the names the native engine reports, which it does + * only for an engine built with no names, and only after its first data + * file has loaded. Those names are the properties of that file sorted by + * name, and the native engine keeps them and their order through every + * later reload, so the map is made once and a reload never changes it. + */ + private void fixPropertyIndexes() { + if (propertyIndexesFixed == false) { + try (VectorStringSwig names = engine.getRequiredProperties()) { + propertyIndexes = indexesOf(new ArrayList<>(names)); + } + propertyIndexesFixed = true; + } + } + /** * Map each name to its place in the list, keeping the first place of a * name given more than once in any letter case. @@ -395,6 +424,7 @@ protected void unmanagedResourcesCleanup() { } private void setEngineMetaData() { + fixPropertyIndexes(); evidenceKeys = getKeysFromEngine(engine); evidenceKeyFilter = new EvidenceKeyFilterWhitelist( evidenceKeys, diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java index 4c222ce41..0302a3a95 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java @@ -23,6 +23,7 @@ package fiftyone.devicedetection.hash.engine.onpremise.flowelements; import fiftyone.devicedetection.hash.engine.onpremise.data.DeviceDataHash; +import fiftyone.devicedetection.hash.engine.onpremise.data.PropertyMetaDataHash; import fiftyone.devicedetection.hash.engine.onpremise.interop.Swig; import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ConfigHashSwig; @@ -46,7 +47,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; @@ -179,14 +184,108 @@ public void PropertyIndex_UnknownNameHasNoIndex() throws Exception { } /** - * An engine built with no names has no indexes, so every read goes by - * name. + * An engine built with no names has an index for every property of the + * first data file it loads, no two properties share an index, and the + * native results name that property at that index. */ @Test - public void PropertyIndex_NoNamesReadsByName() throws Exception { + public void PropertyIndex_NoNamesFromFirstFile() throws Exception { File file = dataFile(LITE_HASH_DATA_FILE_PATH); - try (DeviceDetectionHashEngine engine = build(file, null)) { - assertNull(engine.getPropertyIndex("IsMobile")); + List names = dataSetNames(file); + try (DeviceDetectionHashEngine engine = build(file, null); + NativeEngine nativeEngine = new NativeEngine(file, null); + ResultsHashSwig results = nativeEngine.engine.process(MOBILE)) { + Set seen = new HashSet<>(); + for (String name : names) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + assertTrue( + "Index " + index + " given to more than one name.", + seen.add(index)); + assertEquals( + "Expected the native results to name '" + name + + "' at index " + index + ".", + name, + results.getPropertyName(index)); + } + } + } + + /** + * An engine built with no names keeps its first data file's indexes + * through a reload from a data file whose properties are a superset in a + * different order. Each property both files have then reads what an + * engine built on the new file reads, and a property only the new file + * has gets no index. + */ + @Test + public void PropertyIndex_NoNamesSurvivesReload() throws Exception { + File lite = dataFile(LITE_HASH_DATA_FILE_PATH); + File bigger = dataFile(TAC_HASH_DATA_FILE_NAME); + List names = dataSetNames(lite); + names.retainAll(dataSetNames(bigger)); + String added = nameOnlyIn(bigger, lite); + + DeviceDetectionHashEngine engine = build(lite, null); + DeviceDetectionHashEngine fresh = build(bigger, null); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + Pipeline freshPipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(fresh) + .build()) { + + Map before = new HashMap<>(); + for (String name : names) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + before.put(name, index); + } + + engine.refreshData( + engine.getDataFiles().get(0).getIdentifier(), + Files.readAllBytes(bigger.toPath())); + + for (String name : names) { + assertEquals( + "Index of '" + name + "' moved on reload.", + before.get(name), + engine.getPropertyIndex(name)); + } + if (added != null) { + assertNull( + "Expected no index for '" + added + "', which only " + + "the new file has.", + engine.getPropertyIndex(added)); + } + + try (FlowData data = pipeline.createFlowData(); + FlowData expected = freshPipeline.createFlowData()) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + expected.addEvidence("header.user-agent", MOBILE); + expected.process(); + DeviceData device = data.get(DeviceData.class); + DeviceData want = expected.get(DeviceData.class); + for (String name : names) { + AspectPropertyValue got = + (AspectPropertyValue) device.get(name); + AspectPropertyValue wanted = + (AspectPropertyValue) want.get(name); + assertEquals( + "Expected '" + name + "' to have a value as a " + + "fresh engine gives it.", + wanted.hasValue(), + got.hasValue()); + if (wanted.hasValue()) { + assertEquals( + "Expected '" + name + "' to read as a fresh " + + "engine reads it.", + wanted.getValue(), + got.getValue()); + } + } + } } } @@ -344,6 +443,25 @@ private static String nameOnlyIn(File has, File lacks) throws Exception { return null; } + /** + * The names of the properties the data file holds, sorted, leaving out + * the match metrics the engine adds, which are not in the data file and + * so have no index. + */ + private static List dataSetNames(File file) throws Exception { + List names = new ArrayList<>(); + try (DeviceDetectionHashEngine engine = build(file, null)) { + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + if (property instanceof PropertyMetaDataHash) { + names.add(property.getName()); + } + } + } + Collections.sort(names); + return names; + } + private static List propertyNames(File file) throws Exception { List names = new ArrayList<>(); try (DeviceDetectionHashEngine engine = build(file, null)) { @@ -361,19 +479,38 @@ private static List propertyNames(File file) throws Exception { * index from the same native results, and {@link #READS} times through * the typed accessor, which is the whole path a caller takes, and prints * the median nanoseconds of one read over {@link #ROUNDS} rounds after a - * warm-up. Every read must give the same value. The times are printed - * rather than asserted, because runners vary. + * warm-up, for an engine built with {@link #NAMES}. Every read must give + * the same value. The times are printed rather than asserted, because + * runners vary. */ @Test public void PropertyIndex_ReadByIndexAgainstByName() throws Exception { + measureReads(NAMES); + } + + /** + * The same measurement for an engine built with no names, which reads by + * the index of the property in its first data file. + */ + @Test + public void PropertyIndex_NoNamesReadByIndexAgainstByName() + throws Exception { + measureReads(null); + } + + private static void measureReads(List built) throws Exception { File file = dataFile(LITE_HASH_DATA_FILE_PATH); - DeviceDetectionHashEngine engine = build(file, NAMES); - int index = engine.getPropertyIndex("IsMobile"); + DeviceDetectionHashEngine engine = build(file, built); + Integer found = engine.getPropertyIndex("IsMobile"); + assertNotNull("Expected an index for 'IsMobile'.", found); + int index = found; ConfigHashSwig config = new ConfigHashSwig(); config.setHighPerformance(); RequiredPropertiesConfigSwig required; try (VectorStringSwig names = new VectorStringSwig()) { - names.addAll(NAMES); + if (built != null) { + names.addAll(built); + } required = new RequiredPropertiesConfigSwig(names); } EngineHashSwig nativeEngine = @@ -425,8 +562,9 @@ public void PropertyIndex_ReadByIndexAgainstByName() throws Exception { } assertEquals("Every read gives the same value", 0, different); System.out.println(String.format( - "READ COST IsMobile reads=%d rounds=%d byNameNs=%s " + - "byIndexNs=%s typedNs=%s java=%s os=%s", + "READ COST IsMobile names=%s reads=%d rounds=%d " + + "byNameNs=%s byIndexNs=%s typedNs=%s java=%s os=%s", + built == null ? "none" : "given", READS, ROUNDS, spread(byName), spread(byIndex), spread(typed), System.getProperty("java.version"), System.getProperty("os.name"))); @@ -438,6 +576,37 @@ READS, ROUNDS, spread(byName), spread(byIndex), } } + /** + * A native engine on its own, so a test can read the native results + * directly. Built with no names when the list is null. + */ + private static class NativeEngine implements AutoCloseable { + private final ConfigHashSwig config = new ConfigHashSwig(); + private final RequiredPropertiesConfigSwig required; + private final EngineHashSwig engine; + + NativeEngine(File file, List names) { + config.setHighPerformance(); + RequiredPropertiesConfigSwig made; + try (VectorStringSwig vector = new VectorStringSwig()) { + if (names != null) { + vector.addAll(names); + } + made = new RequiredPropertiesConfigSwig(vector); + } + required = made; + engine = new EngineHashSwig( + file.getAbsolutePath(), config, required); + } + + @Override + public void close() { + engine.delete(); + required.delete(); + config.delete(); + } + } + /** * Reads IsMobile by name through the overload that takes the name's * bytes, which is how a read goes when the engine has no index for it. diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index a93045407..6d1300fd8 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -94,7 +94,7 @@ public class IPIntelligenceDataHashDefault /** * Get the required property index for a property name, or null when the - * engine was not built with that name, so the read falls back to the name. + * engine has no index for that name, so the read falls back to the name. * @param propertyName name of the property * @return required property index, or null */ diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java index 053521d41..6955ef9e9 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java @@ -68,10 +68,20 @@ public class IPIntelligenceOnPremiseEngine * list, the first where a name is given more than once, as the native * engine gives it. Made from the names given at build and never from the * data set, so an index keeps naming the same property after a reload - * from a data file whose property order differs. Empty when the engine - * was built with no names, in which case reads go by name. + * from a data file whose property order differs. When the engine was + * built with no names it is made instead from the names the native engine + * reports after the first data file loads, which are the properties of + * that file sorted by name and which a reload does not change. */ - private final Map propertyIndexes; + private volatile Map propertyIndexes; + + /** + * True once {@link #propertyIndexes} holds the names the indexes are + * fixed on, which is at construction when the engine was built with + * names and after the first data file loads when it was not. A reload + * never makes the map again. + */ + private volatile boolean propertyIndexesFixed; /** * Construct a new instance of the {@link IPIntelligenceOnPremiseEngine}. @@ -100,6 +110,7 @@ public class IPIntelligenceOnPremiseEngine this.config = config; this.propertiesConfigSwig = properties; this.propertyIndexes = indexesOf(propertyNames); + this.propertyIndexesFixed = this.propertyIndexes.isEmpty() == false; addDataFile(dataFile); } @@ -130,7 +141,9 @@ public MetaDataSwig getMetaData() { /** * Get the required property index for a property name, or null when the - * engine was not built with that name and the read has to go by name. + * index list has no such name and the read has to go by name. That covers + * a name the engine was not built with, and a property only a data file + * loaded after the first has. * @param propertyName name of the property * @return required property index, or null */ @@ -138,6 +151,22 @@ Integer getPropertyIndex(String propertyName) { return propertyIndexes.get(propertyName); } + /** + * Fix the indexes on the names the native engine reports, which it does + * only for an engine built with no names, and only after its first data + * file has loaded. Those names are the properties of that file sorted by + * name, and the native engine keeps them and their order through every + * later reload, so the map is made once and a reload never changes it. + */ + private void fixPropertyIndexes() { + if (propertyIndexesFixed == false) { + try (VectorStringSwig names = engine.getRequiredProperties()) { + propertyIndexes = indexesOf(new ArrayList<>(names)); + } + propertyIndexesFixed = true; + } + } + /** * Map each name to its place in the list, keeping the first place of a * name given more than once in any letter case. @@ -385,6 +414,7 @@ protected void unmanagedResourcesCleanup() { } private void setEngineMetaData() { + fixPropertyIndexes(); evidenceKeys = getKeysFromEngine(engine); evidenceKeyFilter = new EvidenceKeyFilterWhitelist( evidenceKeys, diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java index e86fa537f..522943998 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java @@ -81,7 +81,7 @@ public CollectionConfigSwig getStrings() { } public CollectionConfigSwig getComponents() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getComponents(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getComponents(swigCPtr, this), false); } public CollectionConfigSwig getMaps() { @@ -89,15 +89,15 @@ public CollectionConfigSwig getMaps() { } public CollectionConfigSwig getProperties() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProperties(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProperties(swigCPtr, this), false); } public CollectionConfigSwig getValues() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getValues(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getValues(swigCPtr, this), false); } public CollectionConfigSwig getProfiles() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProfiles(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProfiles(swigCPtr, this), false); } public CollectionConfigSwig getGraphs() { diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java index 30b9e0025..405a4a227 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java @@ -92,6 +92,43 @@ public void Builder_MaxPerformance_ReplacesBalanced() throws Exception { collections(PerformanceProfiles.Balanced), inMemory)); } + /** + * Check that every collection the configuration hands out is borrowed. + * The collections belong to the configuration, so a proxy that owned + * one would free it a second time when the proxy is deleted. + */ + @Test + public void Config_CollectionsAreBorrowed() throws Exception { + ConfigIpiSwig config = new ConfigIpiSwig(); + try { + config.setBalanced(); + long[] before = collections(config); + CollectionConfigSwig[] all = { + config.getStrings(), + config.getComponents(), + config.getMaps(), + config.getProperties(), + config.getValues(), + config.getProfiles(), + config.getGraphs(), + config.getProfileGroups(), + config.getProfileOffsets(), + config.getPropertyTypes(), + config.getGraph() + }; + Field own = CollectionConfigSwig.class + .getDeclaredField("swigCMemOwn"); + own.setAccessible(true); + for (CollectionConfigSwig collection : all) { + assertFalse(own.getBoolean(collection)); + collection.delete(); + } + assertArrayEquals(before, collections(config)); + } finally { + config.delete(); + } + } + /** * The native configuration the builder hands to the engine it builds. */ @@ -132,17 +169,17 @@ private static long[] collections(PerformanceProfiles profile) { */ private static long[] collections(ConfigIpiSwig config) { CollectionConfigSwig[] all = { - borrowed(config.getStrings()), - borrowed(config.getComponents()), - borrowed(config.getMaps()), - borrowed(config.getProperties()), - borrowed(config.getValues()), - borrowed(config.getProfiles()), - borrowed(config.getGraphs()), - borrowed(config.getProfileGroups()), - borrowed(config.getProfileOffsets()), - borrowed(config.getPropertyTypes()), - borrowed(config.getGraph()) + config.getStrings(), + config.getComponents(), + config.getMaps(), + config.getProperties(), + config.getValues(), + config.getProfiles(), + config.getGraphs(), + config.getProfileGroups(), + config.getProfileOffsets(), + config.getPropertyTypes(), + config.getGraph() }; long[] settings = new long[all.length * 2]; for (int i = 0; i < all.length; i++) { @@ -151,20 +188,4 @@ private static long[] collections(ConfigIpiSwig config) { } return settings; } - - /** - * The generated wrappers mark four ConfigIpiSwig getters as owned until - * they are regenerated with the C library's corrected MetaData.i. - */ - private static CollectionConfigSwig borrowed(CollectionConfigSwig proxy) { - try { - Field own = CollectionConfigSwig.class - .getDeclaredField("swigCMemOwn"); - own.setAccessible(true); - own.setBoolean(proxy, false); - return proxy; - } catch (ReflectiveOperationException e) { - throw new IllegalStateException(e); - } - } } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java index 1aac78acd..9ae3290c2 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java @@ -22,20 +22,32 @@ package fiftyone.ipintelligence.engine.onpremise.flowelements; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ConfigIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import org.junit.Test; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; /** @@ -59,6 +71,8 @@ public class PropertyIndexTests { "RegisteredName", "Latitude"); + private static final String ADDRESS = "185.28.167.77"; + private static File dataFile(String name) { try { return getFilePath(name); @@ -150,14 +164,95 @@ public void PropertyIndex_UnknownNameHasNoIndex() throws Exception { } /** - * An engine built with no names has no indexes, so every read goes by - * name. + * An engine built with no names has an index for every property of the + * first data file it loads, no two properties share an index, and the + * native results name that property at that index. */ @Test - public void PropertyIndex_NoNamesReadsByName() throws Exception { + public void PropertyIndex_NoNamesFromFirstFile() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, null); + NativeEngine nativeEngine = new NativeEngine(file); + ResultsIpiSwig results = nativeEngine.engine.process(ADDRESS)) { + List names = propertyNames(engine); + assertTrue("Expected the data file to have properties.", + names.isEmpty() == false); + Set seen = new HashSet<>(); + for (String name : names) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + assertTrue( + "Index " + index + " given to more than one name.", + seen.add(index)); + assertEquals( + "Expected the native results to name '" + name + + "' at index " + index + ".", + name, + results.getPropertyName(index)); + } + } + } + + /** + * An engine built with no names keeps the indexes it took from its first + * data file through a reload, because they are made once and not from + * the reloaded data set. + */ + @Test + public void PropertyIndex_NoNamesSurvivesReload() throws Exception { File file = dataFile(LITE_IPI_DATA_FILE_NAME); try (IPIntelligenceOnPremiseEngine engine = build(file, null)) { - assertNull(engine.getPropertyIndex("RegisteredName")); + Map before = new HashMap<>(); + for (String name : propertyNames(engine)) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + before.put(name, index); + } + engine.refreshData(engine.getDataFiles().get(0).getIdentifier()); + for (Map.Entry entry : before.entrySet()) { + assertEquals( + "Index of '" + entry.getKey() + "' moved on reload.", + entry.getValue(), + engine.getPropertyIndex(entry.getKey())); + } + } + } + + private static List propertyNames( + IPIntelligenceOnPremiseEngine engine) { + List names = new ArrayList<>(); + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + names.add(property.getName()); + } + return names; + } + + /** + * A native engine on its own with no names, so a test can read the + * native results directly. + */ + private static class NativeEngine implements AutoCloseable { + private final ConfigIpiSwig config = new ConfigIpiSwig(); + private final RequiredPropertiesConfigSwig required; + private final EngineIpiSwig engine; + + NativeEngine(File file) { + config.setMaxPerformance(); + RequiredPropertiesConfigSwig made; + try (VectorStringSwig vector = new VectorStringSwig()) { + made = new RequiredPropertiesConfigSwig(vector); + } + required = made; + engine = new EngineIpiSwig( + file.getAbsolutePath(), config, required); + } + + @Override + public void close() { + engine.delete(); + required.delete(); + config.delete(); } } } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java index 3af3ff389..01664c283 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java @@ -171,17 +171,17 @@ private static long[] collections(PerformanceProfiles profile) { */ private static long[] collections(ConfigIpiSwig config) { CollectionConfigSwig[] all = { - borrowed(config.getStrings()), - borrowed(config.getComponents()), - borrowed(config.getMaps()), - borrowed(config.getProperties()), - borrowed(config.getValues()), - borrowed(config.getProfiles()), - borrowed(config.getGraphs()), - borrowed(config.getProfileGroups()), - borrowed(config.getProfileOffsets()), - borrowed(config.getPropertyTypes()), - borrowed(config.getGraph()) + config.getStrings(), + config.getComponents(), + config.getMaps(), + config.getProperties(), + config.getValues(), + config.getProfiles(), + config.getGraphs(), + config.getProfileGroups(), + config.getProfileOffsets(), + config.getPropertyTypes(), + config.getGraph() }; long[] settings = new long[all.length * 2]; for (int i = 0; i < all.length; i++) { @@ -190,18 +190,4 @@ private static long[] collections(ConfigIpiSwig config) { } return settings; } - - /** - * The generated wrappers mark four ConfigIpiSwig getters as owned until - * they are regenerated with the C library's corrected MetaData.i. - */ - private static CollectionConfigSwig borrowed(CollectionConfigSwig proxy) { - try { - field(CollectionConfigSwig.class, "swigCMemOwn") - .setBoolean(proxy, false); - return proxy; - } catch (ReflectiveOperationException e) { - throw new IllegalStateException(e); - } - } } From 9c17f012b5cd424ce31355a1c583935c28fefb46 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 19:35:22 +0000 Subject: [PATCH 82/95] Make the data update follow the specification (#79) * Test the data update logging, the If-Modified-Since time and the licence key check The tests read the messages and levels from the Logging section of the data update specification, check that update on start-up works with auto update off and that the file system watcher follows the auto update switch, and that an engine whose update URL is the Distributor's is refused without a licence key. Tests that built an engine with the defaults now switch auto update off or give a key, as the specification's defaults table says. * Make the data update follow the specification Send the data file's last modified time as If-Modified-Since when the file has no publication date. Register the data file with the update service when update on start-up or auto update is on, and let the auto update switch govern the file system watcher. Log the actions and errors with the messages and levels in the Logging section of the data update specification. Refuse to build with update on start-up or auto update on, the Distributor's URL and no licence key, naming the option to turn off. * Test that HTTP status failures are warnings * Log a 429, a 403 and any other failing HTTP status as a warning The Logging section of the data update specification makes a failed download a warning, with the error detail from the server. * Test that the update on start-up log names the engine type * Name the engine in the update on start-up log before the engine exists The builder gives the data file its engine's type name, taken from the builder's own name, and the update service logs it until the engine is set. * Test the masked license key, the write error, the programmatic file comparison and the license key check across data files - DataUpdateServiceLoggingTests gives the data file a license key in its URL and checks that every message, at every level, and every exception logged with one, shows only the key's last 4 characters, for a 304, a 200, a failed connection, a failed download, a 403 and no response. - A failure writing the temporary file logs the Errors table's text for a write error. - With automatic update off and the file system watcher flag at its default, a programmatic update still compares the file on disk. - A builder with two data files, the second without a license key, refuses the build before it registers either. * Show the license keys only masked in every data update message The data update URL carries the license keys in its query. The messages for checking, downloading, a 304 reply, a failed connection and a failed download now name the URL with each key masked to its last 4 characters, and the error detail and the exception logged with a failure are masked too, because an HTTP library's exception often names the URL. The helper that named a data file by its engine for the old messages is no longer used and is removed. * Log a failure writing the temporary file as the write error A failure while writing the downloaded data to the temporary file was logged as the "Error downloading file" warning. The download stream now records whether a read failed, so a failure that is not a read logs the Errors table's "Error writing to data file location or temp file location" text, "An error occurred when writing to . Error detail: ", at error level, and returns AUTO_UPDATE_ERR_READING_STREAM, whose documentation covers a failed output stream. A failed read is still the download warning. * Compare the file on disk on a programmatic update, and when no watcher exists checkForUpdate skipped the comparison of the data file with the temporary file whenever the file system watcher flag was on. The watcher is created only when automatic update is on, so with automatic update off and the flag at its default, a programmatic update never looked at the file on disk. The data update specification's Programmatic update section says the file at the configured location is used "irrespective of whether Automatic Update from File is enabled". A programmatic update now always compares the files, and the timer compares them whenever no watcher exists, rather than whenever the flag is off. * Check every data file's license key before registering any data file preCreateEngine checked each data file's update service and license key inside the loop that registers the data files, so a builder with several data files could register the first, and run its update on start-up, before refusing the build for a later one. Every data file is now checked first, and a refused build registers none. * Expect the one refresh error in the test that rc added for a failed refresh DataUpdateService_CheckForUpdate_FileRefreshFails_TimerSetAgain, which rc added with the change that refreshes the engine once for an in-memory update, allowed for a warning on each of the 10 refresh tries. This draft logs each try at debug and one error after the last, the Errors table's "An error occurred while applying a data file update to " row, so the test now allows that one error. The full build of the merge failed on it. * Correct the grammar of the missing data update service message * Test that a failed refresh logs the Errors table's message data-updates.md gives two rows for AUTO_UPDATE_REFRESH_FAILED, being "An error occurred when applying a data update to engine '' after tries." for the file, which is tried 10 times, and the same without the tries for data held in memory. * Log the Errors table's message when a refresh fails A refresh from the file, tried 10 times, logs "An error occurred when applying a data update to engine '' after 10 tries.", and a refresh from data held in memory logs the same without the tries, as data-updates.md's two AUTO_UPDATE_REFRESH_FAILED rows give them, and as .NET and Go word them. The exception is still passed to the logger. --- ...DetectionOnPremisePipelineBuilderTest.java | 8 + ...elligenceOnPremisePipelineBuilderTest.java | 7 + ...tyOneOnPremiseAspectEngineBuilderBase.java | 12 + ...OnPremiseAspectEngineBuilderBaseTests.java | 127 ++++ .../data/AspectEngineDataFileDefault.java | 18 + .../OnPremiseAspectEngineBuilderBase.java | 88 ++- .../services/DataUpdateServiceDefault.java | 281 +++++++-- .../OnPremiseAspectEngineBuilderBaseTest.java | 58 ++ .../DataUpdateServiceLoggingTests.java | 550 ++++++++++++++++++ .../services/DataUpdateServiceTests.java | 55 +- 10 files changed, 1129 insertions(+), 75 deletions(-) create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java index 21b9e5bd4..aed639665 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java @@ -64,6 +64,7 @@ public void setup() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); builder = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false); } @@ -122,6 +123,7 @@ public void setEngineData_PropertyPropagatedToDataFileConfiguration() throws Exc try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(new byte[]{}, Enums.DeviceDetectionAlgorithm.Hash) .setEngineData(data) + .setAutoUpdate(false) .setShareUsage(false) .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); @@ -144,6 +146,7 @@ public void setEngineData_FilenameIsSet_PropertyNotPropagated() throws Exception public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false) .build()) { List flowElements = pipeline.getFlowElements(); @@ -156,6 +159,7 @@ public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { public void setShareUsage_True_SharedFlowElementAdded() throws Exception { try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(true) .build()) { List flowElements = pipeline.getFlowElements(); @@ -168,6 +172,7 @@ public void setShareUsage_True_SharedFlowElementAdded() throws Exception { public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); @@ -189,6 +194,7 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); try (Pipeline ignored = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .setDataUpdateService(dataUpdateServiceDefault) .build()) { verify(dataUpdateServiceDefault).registerDataFile(any()); @@ -199,6 +205,8 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey("licence_key") + .setDataUpdateService(mock(DataUpdateServiceDefault.class)) .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getUpdateOnStartup()); diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java index d220eb776..9e37ada63 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java @@ -65,6 +65,7 @@ public void setup() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); builder = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false); } @@ -121,6 +122,7 @@ public void setFilename_OtherExtension_ExceptionThrown() { public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false) .build()) { List flowElements = pipeline.getFlowElements(); @@ -133,6 +135,7 @@ public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { public void setShareUsage_True_SharedFlowElementAdded() throws Exception { try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(true) .build()) { List flowElements = pipeline.getFlowElements(); @@ -145,6 +148,7 @@ public void setShareUsage_True_SharedFlowElementAdded() throws Exception { public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); @@ -166,6 +170,7 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); try (Pipeline ignored = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .setDataUpdateService(dataUpdateServiceDefault) .build()) { verify(dataUpdateServiceDefault).registerDataFile(any()); @@ -176,6 +181,8 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey("licence_key") + .setDataUpdateService(mock(DataUpdateServiceDefault.class)) .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getUpdateOnStartup()); diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java index bcd634359..b6ea858ec 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; import fiftyone.pipeline.engines.data.AspectEngineDataFile; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFileDefault; @@ -80,6 +81,17 @@ public FiftyOneOnPremiseAspectEngineBuilderBase( setDataUpdateUrlFormatter(new FiftyOneUrlFormatter()); } + /** + * The Distributor serves nothing without a license key, so a key is + * required while the data update URL is the Distributor's. + * @param dataFileConfig the data file configuration to check + * @return true if the data update URL is the Distributor's + */ + @Override + protected boolean isLicenseKeyRequired(DataFileConfiguration dataFileConfig) { + return DATA_UPDATE_URL.equals(dataFileConfig.getDataUpdateUrl()); + } + /** * Create a new empty data file instance to be populated with the details of * the data file to be used. diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java index c1ff832dc..bb7231527 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java @@ -22,12 +22,15 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.engines.services.DataUpdateService; +import org.mockito.ArgumentCaptor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; @@ -38,7 +41,10 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -118,6 +124,7 @@ public void DataFile_SetEngine() throws Exception { ImplementedBuilder builder = new ImplementedBuilder( LoggerFactory.getILoggerFactory(), updateService); + builder.setAutoUpdate(false); builder.build("", false); assertNotNull(engine.getDataFileMetaData()); @@ -145,4 +152,124 @@ public void DataFile_UrlFormatter() throws Exception { "https://distributor.51degrees.com/api/v2/download?LicenseKeys="+licenseKey+"&Download=True&Type="+downloadType, engine.getDataFileMetaData().getFormattedUrl()); } + + private ImplementedBuilder newBuilder() { + return new ImplementedBuilder( + LoggerFactory.getILoggerFactory(), + updateService); + } + + /** + * The data file is registered with the update service when update on + * start-up is on, even though auto update is off. + */ + @Test + public void UpdateOnStartup_AutoUpdateOff_DataFileRegistered() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey(licenseKey); + + builder.build("", false); + + verify(updateService, times(1)).registerDataFile(any()); + } + + /** + * The engine does not exist when the data file is registered, so the + * data file carries the engine's type name for the update service's log. + */ + @Test + public void UpdateOnStartup_DataFileCarriesTheEngineTypeName() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey(licenseKey); + + builder.build("", false); + + ArgumentCaptor registered = + ArgumentCaptor.forClass(AspectEngineDataFile.class); + verify(updateService).registerDataFile(registered.capture()); + assertEquals("Implemented", + ((AspectEngineDataFileDefault) registered.getValue()) + .getEngineTypeName()); + } + + @Test + public void AutoUpdateAndUpdateOnStartupOff_DataFileNotRegistered() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(false); + + builder.build("", false); + + verify(updateService, never()).registerDataFile(any()); + } + + /** + * The default update URL is the Distributor's, which serves nothing + * without a license key. + */ + @Test + public void AutoUpdate_DefaultUrlNoLicenseKey_BuildRefused() { + ImplementedBuilder builder = newBuilder().setAutoUpdate(true); + + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> builder.build("", false)); + + assertTrue(e.getMessage().contains("setAutoUpdate(false)")); + assertFalse(e.getMessage().contains("setDataUpdateOnStartup(false)")); + verify(updateService, never()).registerDataFile(any()); + } + + @Test + public void UpdateOnStartup_DefaultUrlNoLicenseKey_BuildRefused() { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(true); + + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> builder.build("", false)); + + assertTrue(e.getMessage().contains("setDataUpdateOnStartup(false)")); + assertFalse(e.getMessage().contains("setAutoUpdate(false)")); + } + + @Test + public void AutoUpdate_LicenseKeySet_BuildAllowed() throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(true) + .setDataUpdateLicenseKey(licenseKey); + + builder.build("", false); + + verify(updateService, times(1)).registerDataFile(any()); + } + + @Test + public void AutoUpdate_OtherUrlNoLicenseKey_BuildAllowed() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(true) + .setDataUpdateUrl("https://example.com/data.file"); + + builder.build("", false); + + verify(updateService, times(1)).registerDataFile(any()); + } + + @Test + public void AutoUpdateOff_NoLicenseKey_BuildAllowed() throws Exception { + ImplementedBuilder builder = newBuilder().setAutoUpdate(false); + + builder.build("", false); + + verify(updateService, never()).registerDataFile(any()); + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java index 100d23207..55ecaa049 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java @@ -41,6 +41,7 @@ public class AspectEngineDataFileDefault implements AspectEngineDataFile { private DataFileConfiguration configuration; private DataUpdateService dataUpdateService = null; private String tempDataDirPath; + private String engineTypeName; @Override public String getIdentifier() { @@ -87,6 +88,23 @@ public void setTempDataFilePath(String path) { this.tempDataFilePath = path; } + /** + * The type name of the engine this data file is for, which the update + * service logs until the engine is set. + * @return the engine type name, or null if it is not known + */ + public String getEngineTypeName() { + return engineTypeName; + } + + /** + * Set the type name of the engine this data file is for. + * @param engineTypeName the engine type name + */ + public void setEngineTypeName(String engineTypeName) { + this.engineTypeName = engineTypeName; + } + @Override public String getTempDataDirPath() { if (tempDataDirPath == null && diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java index 37c9dc7c9..a0995b9fb 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.engines.flowelements; import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.configuration.DataFileConfiguration; import fiftyone.pipeline.engines.data.AspectEngineDataFile; @@ -181,28 +182,95 @@ protected void preCreateEngine() { // Any files that have the 'update on startup' flag set // will be updated now. // Create the auto-update configuration and register it. + for (DataFileConfiguration dataFileConfig : dataFileConfigs) { + checkUpdateConfiguration(dataFileConfig); + } for (DataFileConfiguration dataFileConfig : dataFileConfigs) { AspectEngineDataFile dataFile = newAspectEngineDataFile(); dataFile.setIdentifier(dataFileConfig.getIdentifier()); dataFile.setConfiguration(dataFileConfig); dataFile.setTempDataDirPath(tempDir); + if (dataFile instanceof AspectEngineDataFileDefault) { + // By convention an engine is named for its builder. + String builderName = getClass().getSimpleName(); + ((AspectEngineDataFileDefault) dataFile).setEngineTypeName( + builderName.endsWith("Builder") ? + builderName.substring( + 0, builderName.length() - "Builder".length()) : + builderName); + } - if (dataFileConfig.getAutomaticUpdatesEnabled()) { - if (dataUpdateService == null) { - throw new RuntimeException( - "Auto update enabled by data update service does not exist. " + - "This can be corrected by passing an DataUpdateService " + - "instance to the engine builder constructor. " + - "If building from configuration, this can be corrected " + - "by adding an DataUpdateService instance to the " + - "the pipeline builder via addService() method."); - } + if (dataFileConfig.getAutomaticUpdatesEnabled() || + dataFileConfig.getUpdateOnStartup()) { dataUpdateService.registerDataFile(dataFile); } dataFiles.add(dataFile); } } + /** + * Refuse a data file configuration that needs a data update service or + * a license key it does not have. Every data file is checked before any + * is registered, so a refused build leaves nothing registered. + */ + private void checkUpdateConfiguration(DataFileConfiguration dataFileConfig) { + boolean autoUpdate = dataFileConfig.getAutomaticUpdatesEnabled(); + boolean updateOnStartup = dataFileConfig.getUpdateOnStartup(); + if (autoUpdate == false && updateOnStartup == false) { + return; + } + String features = (autoUpdate ? "auto update" : "") + + (autoUpdate && updateOnStartup ? " and " : "") + + (updateOnStartup ? "update on startup" : ""); + if (dataUpdateService == null) { + throw new RuntimeException( + "The data update service does not exist but is " + + "needed for " + features + ". " + + "This can be corrected by passing a DataUpdateService " + + "instance to the engine builder constructor. " + + "If building from configuration, this can be corrected " + + "by adding a DataUpdateService instance to the " + + "pipeline builder via addService() method."); + } + if (isLicenseKeyRequired(dataFileConfig) && + hasLicenseKey(dataFileConfig) == false) { + throw new PipelineConfigurationException( + "No license key is set, and the data update URL " + + "serves nothing without one, so the engine cannot " + + "be built with " + features + " on. Set a license " + + "key with setDataUpdateLicenseKey, or turn off " + + (autoUpdate ? "auto update with setAutoUpdate(false)" : "") + + (autoUpdate && updateOnStartup ? " and " : "") + + (updateOnStartup ? + "update on startup with setDataUpdateOnStartup(false)" : "") + + "."); + } + } + + /** + * Whether the data update URL in the configuration serves nothing + * without a license key. Engines whose update source needs a key + * override this, and the engine then refuses to build with a data update + * option on and no key set. + * @param dataFileConfig the data file configuration to check + * @return true if the update source needs a license key + */ + protected boolean isLicenseKeyRequired(DataFileConfiguration dataFileConfig) { + return false; + } + + private static boolean hasLicenseKey(DataFileConfiguration dataFileConfig) { + List keys = dataFileConfig.getDataUpdateLicenseKeys(); + if (keys != null) { + for (String key : keys) { + if (key != null && key.isEmpty() == false) { + return true; + } + } + } + return false; + } + @SuppressWarnings("unchecked") @Override protected void configureEngine(TEngine engine) throws Exception { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index a2c4af853..b21f1a317 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -47,6 +47,7 @@ import java.net.URI; import java.net.URLDecoder; import java.nio.file.*; +import java.time.Instant; import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -217,10 +218,14 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { null; boolean newDataAvailable = false; if (dataFile != null) { - // Only check the file system if the file system watcher - // is not enabled and the engine is using a temporary file. + logger.info("Checking for update"); logger.debug("CheckForUpdate {}", dataFile.getDataFilePath()); - if (dataFile.getConfiguration().getFileSystemWatcherEnabled() == false && + // Compare the data file with the temporary file the engine uses. + // A programmatic update always does, whether or not Automatic + // Update from File is enabled. The timer does only when no file + // system watcher exists to notice a new file. + if ((manualUpdate || + dataFile.getConfiguration().getWatchKey() == null) && dataFile.getDataFilePath() != null && dataFile.getDataFilePath().isEmpty() == false && dataFile.getTempDataFilePath() != null && @@ -241,7 +246,11 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (newDataAvailable == false && dataFile.getConfiguration().getDataUpdateUrl() != null && dataFile.getConfiguration().getDataUpdateUrl().isEmpty() == false) { - result = checkForUpdateFromUrl(dataFile); + // A failed update is tried again only when the timer is on. + result = checkForUpdateFromUrl( + dataFile, + manualUpdate == false && + dataFile.getAutomaticUpdatesEnabled()); newDataAvailable = result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS || result == AutoUpdateStatus.AUTO_UPDATE_SUCCESS; @@ -335,15 +344,18 @@ private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { return AutoUpdateStatus.AUTO_UPDATE_SUCCESS; } + logger.info("Attempting to refresh Engine '{}' with new data", + getEngineType(dataFile)); // Try to update the file multiple times to ensure the file is not locked. int tries = 0; + Exception lastFailure = null; while (tries < 10) { try { dataFile.getEngine().refreshData(dataFile.getIdentifier()); return AUTO_UPDATE_SUCCESS; } catch (Exception ex) { - logger.warn("File Update: Error applying a data update to engine '{}'", - dataFile.getEngine().getClass().getSimpleName(), ex); + lastFailure = ex; + logger.debug("Refresh attempt {} failed", tries + 1, ex); try { Thread.sleep(200); } catch (InterruptedException ignored) { @@ -351,6 +363,7 @@ private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { } tries++; } + logRefreshFailure(dataFile, lastFailure, tries); return AUTO_UPDATE_REFRESH_FAILED; } @@ -446,22 +459,19 @@ public AutoUpdateStatus updateFromMemory(AspectEngineDataFile dataFile, byte[] d .writeBytes(data); } catch (Exception ex) { logger.error("An error occurred when writing to " + - "'" + dataFile.getDataFilePath() + - "'. The engine will be updated to use the new data " + - "but the file on disk will still contain old data.", - ex); + dataFile.getDataFilePath() + + ". Error detail: " + ex.getMessage(), ex); } } try { // Refresh the engine using the new data. + logger.info("Attempting to refresh Engine '{}' with new data", + getEngineType(dataFile)); dataFile.getEngine().refreshData(dataFile.getIdentifier(), data); result = AUTO_UPDATE_SUCCESS; } catch (Exception ex) { - logger.error("An error occurred when applying a " + - "data update to engine '" + dataFile.getEngine().getClass().getSimpleName() + - "'.", - ex); + logRefreshFailure(dataFile, ex, null); result = AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; } @@ -482,6 +492,7 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { // We also want to do this synchronously so that execution // will block until the engine is ready. if (dataFile.getConfiguration().getUpdateOnStartup() && isFalse(alreadyRegistered)) { + logger.info("Updating on start-up"); logger.debug("Checking for Update on Startup {}", dataFile.getDataFilePath()); AutoUpdateStatus status = checkForUpdate(dataFile, false); if (ObjectUtils.notEqual(status, AUTO_UPDATE_SUCCESS) && @@ -537,11 +548,15 @@ public void run() { } } - // If file system watcher is enabled then set it up. - if (aspectDataFile.getConfiguration().getFileSystemWatcherEnabled() && + // If file system watcher is enabled then set it up. The + // automatic update switch governs the watcher as well as the + // polling timer. + if (aspectDataFile.getAutomaticUpdatesEnabled() && + aspectDataFile.getConfiguration().getFileSystemWatcherEnabled() && aspectDataFile.getConfiguration().getWatchKey() == null && aspectDataFile.getDataFilePath() != null && aspectDataFile.getDataFilePath().isEmpty() == false) { + logger.info("Creating file system watcher"); logger.debug("Initialising watch service"); final Path aspectDataFilePath = Paths.get(aspectDataFile.getDataFilePath()).toAbsolutePath(); @@ -714,15 +729,18 @@ private static List licenseKeysToMask( * If the data currently used by the engine is the newest available then * nothing will be downloaded. * - * @param dataFile the data file to use - * @param tempFile the temp file to write the data to + * @param dataFile the data file to use + * @param tempFile the temp file to write the data to + * @param automatic true if the update will be tried again if it fails * @return the {@link AutoUpdateStatus} value indicating the result */ private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, - FileWrapper tempFile) { + FileWrapper tempFile, + boolean automatic) { String url = dataFile.getFormattedUrl(); + String engineType = getEngineType(dataFile); // The URL carries the license keys, so every message names it with // each key masked. List licenseKeys = licenseKeysToMask(dataFile, url); @@ -730,20 +748,20 @@ private AutoUpdateStatus downloadFile( logger.debug("downloadFile from {}", loggedUrl); try { + logger.info("Checking for update from '{}' for Engine '{}'", + loggedUrl, engineType); HttpURLConnection connection = httpClient.connect(new URI(url.trim()).toURL()); if (connection == null) { - logger.error("No response from data update service at '{}' for engine '{}'", - loggedUrl, Objects.nonNull(dataFile.getEngine()) ? - dataFile.getEngine().getClass().getSimpleName() : "No Engine"); + logger.warn(connectFailure(loggedUrl, engineType, automatic, + "No response from the data update service")); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } try { // Set last-modified header to ensure that a file will only // be downloaded if it is newer than the data we already have. + long ifModifiedSince = 0; if (dataFile.getConfiguration().getVerifyModifiedSince() == true) { - long ifModifiedSince = dataFile - .getDataPublishedDateTime() - .getTime(); + ifModifiedSince = getIfModifiedSince(dataFile); connection.setIfModifiedSince(ifModifiedSince); } @@ -751,26 +769,43 @@ private AutoUpdateStatus downloadFile( switch (connection.getResponseCode()) { case (HttpURLConnection.HTTP_OK): - return downloadOkResponse(connection, dataFile, tempFile); + AutoUpdateStatus downloaded; + try { + downloaded = downloadOkResponse( + connection, dataFile, tempFile, automatic); + } catch (Exception e) { + // An HTTP library's exception often names the + // URL it requested. + logger.warn(downloadFailure(loggedUrl, engineType, + automatic, LicenseKeyMask.maskIn( + e.getMessage(), licenseKeys)), + LicenseKeyMask.maskIn(e, licenseKeys)); + return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; + } + if (downloaded == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { + logger.info("Downloaded new data from '{}' for Engine '{}'", + loggedUrl, engineType); + } + return downloaded; // Note: needed because TooManyRequests is not available // in some versions of the HttpStatusCode enum. case (429): - logger.error("Too many requests to '{}' for {}", - loggedUrl, getIdForLogging(dataFile)); + logger.warn(downloadFailure(loggedUrl, engineType, automatic, + "HTTP status code 429 (too many requests)")); return AutoUpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS; case HttpURLConnection.HTTP_NOT_MODIFIED: - logger.debug("No data update available from '{}' for {}", - loggedUrl, getIdForLogging(dataFile)); + logger.info("No data newer than {} found at '{}' for Engine '{}'", + Instant.ofEpochMilli(ifModifiedSince), loggedUrl, + engineType); return AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; case HttpURLConnection.HTTP_FORBIDDEN: - logger.error("Access denied to data update service at '{}' for {}", - loggedUrl, getIdForLogging(dataFile)); + logger.warn(downloadFailure(loggedUrl, engineType, automatic, + "HTTP status code 403 (access denied)")); return AutoUpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN; default: - logger.error("HTTP status code '{}' from data update service at '{}' for {}", - connection.getResponseCode(), loggedUrl, - getIdForLogging(dataFile)); + logger.warn(downloadFailure(loggedUrl, engineType, automatic, + "HTTP status code " + connection.getResponseCode())); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } finally { @@ -778,12 +813,90 @@ private AutoUpdateStatus downloadFile( } } catch (Exception e) { // An HTTP library's exception often names the URL it requested. - logger.error("Error while processing data file download", + logger.warn(connectFailure(loggedUrl, engineType, automatic, + LicenseKeyMask.maskIn(e.getMessage(), licenseKeys)), LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } + /** + * Get the time to send as If-Modified-Since. This is the publication date + * of the data file, or the last modified time of the data file on disk + * where the data file has no publication date. + */ + private long getIfModifiedSince(AspectEngineDataFile dataFile) { + Date published = dataFile.getDataPublishedDateTime(); + long publishedMillis = published == null ? 0 : published.getTime(); + if (publishedMillis > 0) { + return publishedMillis; + } + String path = dataFile.getDataFilePath(); + if (path != null && path.isEmpty() == false) { + return fileWrapperFactory.getLastModified(path); + } + return publishedMillis; + } + + private static String retryNote(boolean automatic) { + return automatic ? " Update will be attempted again later." : ""; + } + + /** + * The warning for a failed connection. The URL and the detail are + * given with each license key already masked. + */ + private static String connectFailure( + String loggedUrl, + String engineType, + boolean automatic, + String detail) { + return "An error occurred when connecting to " + loggedUrl + + " in order to check for data file updates for " + engineType + + "." + retryNote(automatic) + " Error detail: " + detail; + } + + /** + * The warning for a failed download. The URL and the detail are given + * with each license key already masked. + */ + private static String downloadFailure( + String loggedUrl, + String engineType, + boolean automatic, + String detail) { + return "An error occurred while downloading a data file update for " + + engineType + " from " + loggedUrl + "." + retryNote(automatic) + + " Error detail: " + detail; + } + + /** + * Log the Errors table's message for a refresh that failed, with the + * number of tries for a refresh from the file and without it for one + * from data held in memory. The exception is passed to the logger, so + * its detail is kept beside the message. + */ + private void logRefreshFailure( + AspectEngineDataFile dataFile, + Exception ex, + Integer tries) { + logger.error("An error occurred when applying a data update to " + + "engine '" + getEngineType(dataFile) + "'" + + (tries == null ? "." : " after " + tries + " tries."), ex); + } + + private String getEngineType(AspectEngineDataFile dataFile) { + if (dataFile.getEngine() != null) { + return dataFile.getEngine().getClass().getSimpleName(); + } + // The engine is built after update on start-up has run. + if (dataFile instanceof AspectEngineDataFileDefault && + ((AspectEngineDataFileDefault) dataFile).getEngineTypeName() != null) { + return ((AspectEngineDataFileDefault) dataFile).getEngineTypeName(); + } + return "No Engine"; + } + /** * Persist the body of a 200 OK download to {@code tempFile}. *

    @@ -792,11 +905,16 @@ private AutoUpdateStatus downloadFile( * {@link GZIPInputStream}, so the JDK's concatenated-gzip-member * decoder behaves correctly on top of {@link HttpURLConnection} on * every JDK from 7 upwards. See that method for the gory details. + *

    + * A failure reading the download is thrown, for the caller to report. + * A failure writing {@code tempFile} is logged here as the + * specification's write error. */ private AutoUpdateStatus downloadOkResponse( HttpURLConnection connection, AspectEngineDataFile dataFile, - FileWrapper tempFile) throws IOException { + FileWrapper tempFile, + boolean automatic) throws IOException { logger.debug("HTTP Status is OK"); boolean verifyMd5 = dataFile.getConfiguration().getVerifyMd5(); boolean decompress = dataFile.getConfiguration().getDecompressContent(); @@ -817,9 +935,25 @@ private AutoUpdateStatus downloadOkResponse( src = new GZIPInputStream(src); } - writeBodyTo(tempFile, src); + ReadFailureRecorder body = new ReadFailureRecorder(src); + try { + writeBodyTo(tempFile, body); + } catch (IOException e) { + if (body.readFailed) { + // The download failed, which the caller reports. + throw e; + } + logger.error("An error occurred when writing to " + + tempFile.getPath() + ". Error detail: " + e.getMessage(), e); + return AutoUpdateStatus.AUTO_UPDATE_ERR_READING_STREAM; + } if (verifyMd5 && !md5Matches(md, connection)) { + logger.warn("An error occurred during the integrity check of new " + + "data file for " + getEngineType(dataFile) + "." + + retryNote(automatic) + + " Error detail: the MD5 hash of the download does not " + + "match the Content-MD5 header"); return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; } return AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; @@ -857,6 +991,38 @@ public int available() throws IOException { return new BufferedInputStream(lying); } + /** + * Records whether a read of the download failed, so that a failure to + * write the temporary file can be told from a failure to download. + */ + private static final class ReadFailureRecorder extends FilterInputStream { + private boolean readFailed; + + ReadFailureRecorder(InputStream in) { + super(in); + } + + @Override + public int read() throws IOException { + try { + return super.read(); + } catch (IOException e) { + readFailed = true; + throw e; + } + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + try { + return super.read(b, off, len); + } catch (IOException e) { + readFailed = true; + throw e; + } + } + } + private void writeBodyTo(FileWrapper tempFile, InputStream body) throws IOException { if (tempFile.getPath() != null) { logger.debug("Copying download stream"); @@ -890,20 +1056,17 @@ public void close() { } } - private String getIdForLogging(AspectEngineDataFile dataFile) { - return dataFile.getEngine() == null ? - "data file '" + dataFile.getIdentifier() + "'" : - "engine '" + dataFile.getEngine().getClass().getSimpleName() + "'"; - } - - private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { + private AutoUpdateStatus checkForUpdateFromUrl( + AspectEngineDataFile dataFile, + boolean automatic) { if (dataFile.getDataFilePath() == null || dataFile.getDataFilePath().isEmpty()) { // There is no data file path specified so perform the // update entirely in memory. try { FileWrapper uncompressedData = new FileWrapperMemory(); - AutoUpdateStatus memoryDownloadResult = downloadFile(dataFile, uncompressedData); + AutoUpdateStatus memoryDownloadResult = + downloadFile(dataFile, uncompressedData, automatic); if (memoryDownloadResult != AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { return memoryDownloadResult; } @@ -911,6 +1074,8 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { if (dataFile.getEngine() != null) { // Tell the engine to refresh itself with // the new data. + logger.info("Attempting to refresh Engine '{}' with new data", + getEngineType(dataFile)); dataFile.getEngine().refreshData(dataFile.getIdentifier(), reader.toByteArray()); } else { @@ -922,21 +1087,26 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { // The engine, or the configuration, now holds the new data. return AUTO_UPDATE_SUCCESS; } catch (Exception ex) { - logger.error("An error occurred when applying a data update to engine '{}'" , - Objects.nonNull(dataFile.getEngine())? - dataFile.getEngine().getClass().getName(): "no engine",ex); + logRefreshFailure(dataFile, ex, null); return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; } } - + if (dataFile.getTempDataDirPath() == null || + dataFile.getTempDataDirPath().isEmpty()) { + logger.error("The data file '" + dataFile.getIdentifier() + + "' is checking for updates but does not have a temporary " + + "file path configured."); + return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + } Path uncompressedTempFile = Paths.get( dataFile.getTempDataDirPath(), dataFile.getIdentifier() + "-" + randomUUID() + ".tmp"); FileWrapper uncompressedData = fileWrapperFactory.build(uncompressedTempFile.toString()); try { // Check if there is an update and download it if there is - AutoUpdateStatus result = downloadFile(dataFile, uncompressedData); + AutoUpdateStatus result = + downloadFile(dataFile, uncompressedData, automatic); if (result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { WatchKey watchKey = dataFile.getConfiguration().getWatchKey(); @@ -962,12 +1132,9 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { logger.debug("Reading file {}", d.read()); } } catch (Exception ex) { - logger.error("An error occurred when moving a data file to replace " + - "the existing one at {} for engine '{}'.", - dataFile.getDataFilePath(), - Objects.nonNull(dataFile.getEngine()) ? - dataFile.getEngine().getClass().getSimpleName() : "No engine defined", - ex); + logger.error("An error occurred when copying a data file to " + + "replace the existing one at '{}'.", + dataFile.getDataFilePath(), ex); result = AutoUpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME; } // Make sure to enable the file watcher again if needed. diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java index 68d84213d..196e6b9d3 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java @@ -22,7 +22,11 @@ package fiftyone.pipeline.engines.flowelements; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.services.DataUpdateService; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -35,11 +39,17 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; +import java.util.Collections; import java.util.List; import java.util.Set; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; @SuppressWarnings("rawtypes") public class OnPremiseAspectEngineBuilderBaseTest { @@ -131,4 +141,52 @@ public void CreateTemporaryFolder_VerifyPermissions() { assertTrue(tempDir.canWrite()); assertTrue(tempDir.canExecute()); } + + /** + * Every data file's license key is checked before any data file is + * registered, so a build refused for a later data file leaves nothing + * registered with the data update service. + */ + @Test + public void LaterDataFileHasNoLicenseKey_NothingRegistered() { + DataUpdateService updateService = mock(DataUpdateService.class); + OnPremiseAspectEngineBuilderBase builder = + new OnPremiseAspectEngineBuilderBase(null, updateService) { + @Override + public OnPremiseAspectEngineBuilderBase setPerformanceProfile( + Constants.PerformanceProfiles profile) { + return this; + } + + @Override + protected AspectEngine newEngine(List properties) { + return null; + } + + @Override + protected boolean isLicenseKeyRequired( + DataFileConfiguration dataFileConfig) { + return true; + } + }; + builder.addDataFile(autoUpdateConfiguration("First", "SOMELICENSEKEY")); + builder.addDataFile(autoUpdateConfiguration("Second", null)); + + assertThrows(PipelineConfigurationException.class, builder::buildEngine); + + verify(updateService, never()).registerDataFile(any()); + } + + private static DataFileConfiguration autoUpdateConfiguration( + String identifier, + String licenseKey) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setIdentifier(identifier); + config.setAutomaticUpdatesEnabled(true); + if (licenseKey != null) { + config.setDataUpdateLicenseKeys( + Collections.singletonList(licenseKey)); + } + return config; + } } \ No newline at end of file diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java new file mode 100644 index 000000000..4ec40a1b0 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -0,0 +1,550 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; +import ch.qos.logback.core.read.ListAppender; +import fiftyone.common.wrappers.io.FileWrapperFactoryDefault; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The messages and levels in the Logging section of the data update + * specification, and the If-Modified-Since time the update URL is sent. + * @see Specification + */ +@SuppressWarnings("unchecked") +public class DataUpdateServiceLoggingTests { + + private static final String UPDATE_URL = "http://test.com"; + // 13 September 2020, 12:26:40 UTC. + private static final long FILE_MODIFIED_MILLIS = 1600000000000L; + // A license key of the length and characters the Distributor issues. + private static final String LICENSE_KEY = "QF8ZR2KD7NMB4XWT9HJC3LPA"; + private static final String MASKED_KEY = "********************3LPA"; + private static final String KEYED_URL = UPDATE_URL + "?LicenseKeys=" + + LICENSE_KEY + "&Download=True&Type=HashV41"; + private static final String MASKED_URL = UPDATE_URL + "?LicenseKeys=" + + MASKED_KEY + "&Download=True&Type=HashV41"; + + private ch.qos.logback.classic.Logger logger; + private ListAppender appender; + private HttpClient httpClient; + private HttpURLConnection connection; + private DataUpdateServiceDefault dataUpdate; + private OnPremiseAspectEngine engine; + private String engineType; + private File dataFile; + + @Before + public void init() throws IOException { + logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger( + "DataUpdateServiceLoggingTests"); + logger.setLevel(Level.INFO); + appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + connection = mock(HttpURLConnection.class); + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + // The futures are real but their first run is a day away. + dataUpdate = new DataUpdateServiceDefault( + logger, httpClient, new FileWrapperFactoryDefault(), null); + + engine = mock(OnPremiseAspectEngine.class); + engineType = engine.getClass().getSimpleName(); + dataFile = File.createTempFile("logging-test", ".tmp"); + assertTrue(dataFile.setLastModified(FILE_MODIFIED_MILLIS)); + } + + @After + public void cleanup() { + dataUpdate.close(); + logger.detachAppender(appender); + dataFile.delete(); + } + + private AspectEngineDataFileDefault newDataFile( + boolean inMemory, + boolean autoUpdate, + boolean updateOnStartup) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdate); + config.setUpdateOnStartup(updateOnStartup); + config.setFileSystemWatcherEnabled(false); + config.setDataUpdateUrl(UPDATE_URL); + config.setVerifyMd5(false); + config.setDecompressContent(false); + config.setPollingIntervalSeconds(86400); + config.setMaxRandomisationSeconds(0); + if (inMemory == false) { + config.setDataFilePath(dataFile.getAbsolutePath()); + } + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setIdentifier("default"); + file.setEngine(engine); + file.setConfiguration(config); + file.setTempDataDirPath(System.getProperty("java.io.tmpdir")); + when(engine.getDataFileMetaData()).thenReturn(file); + return file; + } + + private List logged(Level level) { + List messages = new ArrayList<>(); + for (ILoggingEvent event : appender.list) { + if (event.getLevel() == level) { + messages.add(event.getFormattedMessage()); + } + } + return messages; + } + + private void assertLogged(Level level, String message) { + List messages = logged(level); + assertTrue( + "Expected '" + message + "' at " + level + " but got " + messages, + messages.contains(message)); + } + + @Test + public void IfModifiedSince_NoPublishedDate_UsesFileLastModified() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setVerifyModifiedSince(true); + + dataUpdate.checkForUpdate(engine); + + verify(connection).setIfModifiedSince(FILE_MODIFIED_MILLIS); + } + + @Test + public void IfModifiedSince_PublishedDate_UsedInsteadOfFileLastModified() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setVerifyModifiedSince(true); + file.setDataPublishedDateTime(new java.util.Date(1500000000000L)); + + dataUpdate.checkForUpdate(engine); + + verify(connection).setIfModifiedSince(1500000000000L); + } + + @Test + public void Check_NotModified_LogsTheSpecifiedMessages() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setVerifyModifiedSince(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Checking for update"); + assertLogged(Level.INFO, "Checking for update from '" + UPDATE_URL + + "' for Engine '" + engineType + "'"); + assertLogged(Level.INFO, "No data newer than 2020-09-13T12:26:40Z " + + "found at '" + UPDATE_URL + "' for Engine '" + engineType + "'"); + } + + @Test + public void Check_Downloaded_LogsDownloadAndRefresh() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + newDataFile(true, false, false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Downloaded new data from '" + UPDATE_URL + + "' for Engine '" + engineType + "'"); + assertLogged(Level.INFO, "Attempting to refresh Engine '" + + engineType + "' with new data"); + } + + @Test + public void UpdateFromMemory_LogsRefresh() { + AspectEngineDataFileDefault file = newDataFile(true, false, false); + + dataUpdate.updateFromMemory(file, new byte[]{1, 2, 3}); + + assertLogged(Level.INFO, "Attempting to refresh Engine '" + + engineType + "' with new data"); + } + + @Test + public void UpdateFromMemory_RefreshFails_LogsTheTableMessage() + throws Exception { + AspectEngineDataFileDefault file = newDataFile(true, false, false); + doThrow(new RuntimeException("no room")) + .when(engine).refreshData(anyString(), any(byte[].class)); + + dataUpdate.updateFromMemory(file, new byte[]{1, 2, 3}); + + assertLogged(Level.ERROR, "An error occurred when applying a data " + + "update to engine '" + engineType + "'."); + } + + @Test + public void UpdateOnStartup_LogsUpdatingOnStartUp() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.INFO, "Updating on start-up"); + } + + @Test + public void UpdateOnStartup_EngineNotBuiltYet_LogsTheEngineTypeName() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + file.setEngine(null); + file.setEngineTypeName("ExampleEngine"); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.INFO, "Checking for update from '" + UPDATE_URL + + "' for Engine 'ExampleEngine'"); + } + + @Test + public void UpdateOnStartup_AutoUpdateOff_StartsNoTimer() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + + dataUpdate.registerDataFile(file); + + assertEquals(null, file.getFuture()); + } + + @Test + public void FileSystemWatcher_LogsCreatingFileSystemWatcher() { + AspectEngineDataFileDefault file = newDataFile(false, true, false); + file.getConfiguration().setFileSystemWatcherEnabled(true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.INFO, "Creating file system watcher"); + assertNotNull(file.getConfiguration().getWatchKey()); + } + + @Test + public void ConnectFails_Manual_WarnsWithoutRetryText() throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("refused")); + newDataFile(false, false, false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + UPDATE_URL + " in order to check for data file updates for " + + engineType + ". Error detail: refused"); + } + + @Test + public void ConnectFails_Automatic_WarnsThatUpdateWillBeTriedAgain() + throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("refused")); + AspectEngineDataFileDefault file = newDataFile(false, true, true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + UPDATE_URL + " in order to check for data file updates for " + + engineType + ". Update will be attempted again later. " + + "Error detail: refused"); + } + + @Test + public void ConnectFails_AutoUpdateOff_DoesNotPromiseAnotherTry() + throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("refused")); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + UPDATE_URL + " in order to check for data file updates for " + + engineType + ". Error detail: refused"); + } + + private void assertStatusWarned(int status, String detail) + throws Exception { + when(connection.getResponseCode()).thenReturn(status); + newDataFile(true, false, false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred while downloading a data " + + "file update for " + engineType + " from " + UPDATE_URL + ". " + + "Error detail: " + detail); + assertTrue(logged(Level.ERROR).isEmpty()); + } + + @Test + public void TooManyRequests_WarnsAboutTheDownload() throws Exception { + assertStatusWarned(429, "HTTP status code 429 (too many requests)"); + } + + @Test + public void Forbidden_WarnsAboutTheDownload() throws Exception { + assertStatusWarned(403, "HTTP status code 403 (access denied)"); + } + + @Test + public void OtherStatus_WarnsAboutTheDownload() throws Exception { + assertStatusWarned(500, "HTTP status code 500"); + } + + /** + * The download is written to a temporary file in a folder that does not + * exist, so the write fails while the download itself succeeds. + */ + @Test + public void TemporaryFileCannotBeWritten_LogsTheWriteError() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + String missingDirectory = new File( + System.getProperty("java.io.tmpdir"), + "no-such-directory-" + System.nanoTime()).getAbsolutePath(); + file.setTempDataDirPath(missingDirectory); + // The engine's copy is the data file itself, so the file on disk is + // not newer and the update is downloaded. + file.setTempDataFilePath(dataFile.getAbsolutePath()); + + dataUpdate.checkForUpdate(engine); + + verify(connection).getInputStream(); + boolean found = false; + for (String message : logged(Level.ERROR)) { + found |= message.startsWith("An error occurred when writing to " + + missingDirectory) && message.contains(". Error detail: "); + } + assertTrue("No write error in " + logged(Level.ERROR) + + ", and the warnings are " + logged(Level.WARN), found); + assertTrue(logged(Level.WARN).isEmpty()); + } + + @Test + public void Md5DoesNotMatch_WarnsAboutIntegrityCheck() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + when(connection.getHeaderField("Content-MD5")).thenReturn("0"); + AspectEngineDataFileDefault file = newDataFile(true, false, false); + file.getConfiguration().setVerifyMd5(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred during the integrity " + + "check of new data file for " + engineType + ". Error detail: " + + "the MD5 hash of the download does not match the Content-MD5 " + + "header"); + } + + @Test + public void NoTemporaryFilePath_LogsError() { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.setTempDataDirPath(null); + when(engine.getTempDataDirPath()).thenReturn(null); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.ERROR, "The data file 'default' is checking for " + + "updates but does not have a temporary file path configured."); + } + + /** + * A data file whose formatted URL carries a license key, as the + * Distributor's URL does, with every level logged. + */ + private AspectEngineDataFileDefault newKeyedDataFile(boolean inMemory) { + logger.setLevel(Level.DEBUG); + AspectEngineDataFileDefault file = newDataFile(inMemory, false, false); + DataFileConfiguration config = file.getConfiguration(); + config.setDataUpdateLicenseKeys( + Collections.singletonList(LICENSE_KEY)); + config.setUrlFormatter(dataFile -> KEYED_URL); + return file; + } + + /** + * Fails if any log entry, or any throwable logged with one, holds the + * license key in full. + */ + private void assertKeyOnlyMasked() { + assertFalse(appender.list.isEmpty()); + for (ILoggingEvent event : appender.list) { + String text = event.getFormattedMessage(); + if (event.getThrowableProxy() != null) { + text += "\n" + + ThrowableProxyUtil.asString(event.getThrowableProxy()); + } + assertFalse( + "The license key is in full in: " + text, + text.contains(LICENSE_KEY)); + } + } + + @Test + public void LicenseKey_NotModified_LogsTheMaskedUrl() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newKeyedDataFile(false); + file.getConfiguration().setVerifyModifiedSince(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Checking for update from '" + MASKED_URL + + "' for Engine '" + engineType + "'"); + assertLogged(Level.INFO, "No data newer than 2020-09-13T12:26:40Z " + + "found at '" + MASKED_URL + "' for Engine '" + engineType + "'"); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_Downloaded_LogsTheMaskedUrl() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + newKeyedDataFile(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Downloaded new data from '" + MASKED_URL + + "' for Engine '" + engineType + "'"); + assertKeyOnlyMasked(); + } + + /** + * An HTTP library's exception often names the URL it requested, so the + * error detail and the logged exception are masked too. + */ + @Test + public void LicenseKey_ConnectFails_MasksTheUrlAndTheException() + throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("Connection refused: " + KEYED_URL)); + newKeyedDataFile(false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + MASKED_URL + " in order to check for data file updates for " + + engineType + ". Error detail: Connection refused: " + MASKED_URL); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_DownloadFails_MasksTheUrlAndTheException() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenThrow(new IOException("Premature EOF from " + KEYED_URL)); + newKeyedDataFile(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred while downloading a data " + + "file update for " + engineType + " from " + MASKED_URL + ". " + + "Error detail: Premature EOF from " + MASKED_URL); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_Forbidden_LogsTheMaskedUrl() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + newKeyedDataFile(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred while downloading a data " + + "file update for " + engineType + " from " + MASKED_URL + ". " + + "Error detail: HTTP status code 403 (access denied)"); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_NoResponse_LogsTheMaskedUrl() throws Exception { + when(httpClient.connect(any(URL.class))).thenReturn(null); + newKeyedDataFile(false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + MASKED_URL + " in order to check for data file updates for " + + engineType + ". Error detail: No response from the data update " + + "service"); + assertKeyOnlyMasked(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 4ed689fc2..a1650c1da 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -374,7 +374,12 @@ public void DataUpdateService_Register_FileSystemWatcher() throws IOException { dataUpdate.registerDataFile(file); // Assert - assertNotNull(config.getWatchKey()); + // The automatic update switch governs the file system watcher. + if (autoUpdateEnabled) { + assertNotNull(config.getWatchKey()); + } else { + assertNull(config.getWatchKey()); + } } finally { // Make sure we tidy up the temp file. File tempFile = new File(tempData); @@ -890,10 +895,12 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { // reconfigured to try again later. verify(futureFactory, atLeast(1)) .schedule(any(Runnable.class), anyLong()); - // We expect one error to be logged so make sure it's + // We expect one warning to be logged so make sure it's // ignored in cleanup and verify its presence. - ignoreErrors = 1; - assertEquals(1, logger.errorsLogged.size()); + ignoreWranings = 1; + assertEquals(1, logger.warningsLogged.size()); + assertTrue(logger.warningsLogged.get(0).contains(errorText)); + assertEquals(0, logger.errorsLogged.size()); assertEquals(AUTO_UPDATE_HTTPS_ERR, completeEventArgs[0].getStatus()); } finally { @@ -1019,6 +1026,32 @@ public void DataUpdateService_CheckForUpdate_FileUpdated() { anyLong()); } + /** + * A programmatic update compares the data file on disk whether or not + * Automatic Update from File is enabled, and with automatic update off + * no file system watcher exists whatever the watcher flag says. + */ + @Test + public void DataUpdateService_CheckForUpdate_FileUpdated_AutoUpdateOffWatcherFlagDefault() { + // Arrange + OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + configureFileUpdate(engine); + + // Act + dataUpdate.checkForUpdate(engine); + + // Assert + verify(engine, times(1)).refreshData((String)any()); + } + @Test public void DataUpdateService_CheckForUpdate_UrlNoUpdate() throws IOException { // Getting no data from the URL will cause an error to be logged @@ -1345,8 +1378,8 @@ public void DataUpdateService_CheckForUpdate_FileUpdated_TimerSetAgain() { */ @Test public void DataUpdateService_CheckForUpdate_FileRefreshFails_TimerSetAgain() { - // updatedFileAvailable warns on each of its 10 tries. - ignoreWranings = 10; + // updatedFileAvailable logs one error once its tries are used up. + ignoreErrors = 1; // Arrange OnPremiseAspectEngine engine = @@ -1375,6 +1408,12 @@ public void DataUpdateService_CheckForUpdate_FileRefreshFails_TimerSetAgain() { assertEquals( DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED, status); + String expected = "An error occurred when applying a data update " + + "to engine '" + engine.getClass().getSimpleName() + + "' after 10 tries."; + assertTrue( + "Expected '" + expected + "' but got " + logger.errorsLogged, + logger.errorsLogged.contains(expected)); verify(futureFactory, times(2 * timers)).schedule( any(Runnable.class), anyLong()); @@ -1688,8 +1727,8 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { assertTrue("The 'checkForUpdateComplete' " + "event was never fired", completed); - // Ignore the error that is logged due to the 429 - ignoreErrors = 1; + // Ignore the warning that is logged due to the 429 + ignoreWranings = 1; // Check that the timer has been set to go off again. assertNotNull(file.getFuture()); // Check that the timer has been set to expire in 10 seconds. From 53ccddde54314ee51263f5b4c8ede60271d4cd6d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 19:36:28 +0000 Subject: [PATCH 83/95] Give every environment variable one _51DEGREES_ name (#105) * Add the environment variable name check, red before the rename Reads every tracked file and fails on a name that starts with the digit, which a POSIX shell cannot set, and on an old name that is still read where each variable has one name. Pester tests give the check one line at a time and a workflow runs both on every pull request. This commit is the check alone, so it fails over the repository as it stands and names every place the next commit changes. * Give every environment variable one _51DEGREES_ name A POSIX shell cannot set a name that starts with a digit, so the three names that did move to the leading underscore, and the names read behind a _51DEGREES_ one are no longer read at all. 51DEGREES_IPI_PATH, 51DEGREES_DD_PATH and 51DEGREES_RESOURCE_KEY become _51DEGREES_IPI_PATH, _51DEGREES_DD_PATH and _51DEGREES_RESOURCE_KEY. FOD_CLOUD_API_URL becomes _51DEGREES_CLOUD_ENDPOINT, which is the name the Rust library already reads, and the constant holding it is renamed with it. The RESOURCE_KEY and LICENSE_KEY reads behind the _51DEGREES_ names in the 51Did and cloud engine examples and the live 51Did tests are removed. * See an old name declared as a key The old-name half needed a quote, a dollar sign, a brace or a getenv call in front of the name, so it could not see one declared as a key in a workflow env: block or listed at the start of a line in a readme. A name at the start of a line, past any spaces, tabs and a list dash, now counts. The tests state it, with a workflow line whose key is an old name, a readme list entry and a shell assignment on the failing side, and a workflow line whose key is not an old name, and a longer name that ends in one of these words, on the passing side. * See an old name braced as a PowerShell or lookup variable The check misses an old name written as ${env:NAME}, as $ENV:NAME, as Env:\NAME or as ${{ env.NAME }}, because its prefix set wants the dollar and the env: adjacent, and its brace branch wants the name straight after the brace. Both the CI scripts and the pipeline config lookup write the braced form, so an old name can come back through the file types the check watches. These cases fail until the prefix set takes the env token on its own. * Match the env token on its own, so a braced old name is seen The prefix set wanted the dollar and the env token adjacent, and its brace branch wanted the name straight after the brace, so ${env:NAME}, $ENV:NAME, Env:\NAME and a workflow ${{ env.NAME }} all went past. Matching the token on its own, in any case and with a following backslash allowed, closes all four, which is what the Go check already does. * Wrap the endpoint comment on a whole phrase --- .../workflows/environment-variable-names.yml | 30 +++++ AGENTS.md | 4 +- ci/check-environment-variable-names.Tests.ps1 | 92 +++++++++++++ ci/check-environment-variable-names.ps1 | 124 ++++++++++++++++++ ci/run-integration-tests.ps1 | 2 +- examples/device-detection/README.md | 2 +- .../examples/web/GettingStartedWebOnPrem.java | 2 +- .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 4 +- .../web/OnPremDataFileOptionTest.java | 4 +- examples/ip-intelligence/README.md | 6 +- .../examples/console/OfflineProcessing.java | 2 +- .../examples/shared/DataFileHelper.java | 2 +- .../examples/shared/KeyHelper.java | 2 +- .../examples/shared/KeyHelperTest.java | 2 +- ip-intelligence/README.md | 4 +- .../shared/testhelpers/FileUtils.java | 2 +- .../shared/testhelpers/KeyUtils.java | 2 +- .../CountryTranslationIntegrationTest.java | 4 +- pipeline/README.md | 4 +- .../CloudRequestEngineBuilder.java | 4 +- .../flowelements/Constants.java | 2 +- .../CloudRequestEngineLiveTests.java | 4 +- .../flowelements/LiveCloudKeys.java | 2 - .../developerexamples/cloudengine/Main.java | 17 ++- .../README.md | 8 +- .../fodid/CreatorContextDemoServer.java | 20 +-- pipeline/pipeline.did/README.md | 2 +- .../java/fiftyone/pipeline/did/DidClient.java | 2 +- .../pipeline/did/DidClientLiveTests.java | 10 +- .../fiftyone/pipeline/did/LiveCloudKeys.java | 2 - 30 files changed, 304 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/environment-variable-names.yml create mode 100644 ci/check-environment-variable-names.Tests.ps1 create mode 100644 ci/check-environment-variable-names.ps1 diff --git a/.github/workflows/environment-variable-names.yml b/.github/workflows/environment-variable-names.yml new file mode 100644 index 000000000..f5b3abd55 --- /dev/null +++ b/.github/workflows/environment-variable-names.yml @@ -0,0 +1,30 @@ +name: Environment variable names + +on: + pull_request: + push: + branches: + - main + - rc + workflow_dispatch: + +permissions: + contents: read + +jobs: + environment-variable-names: + name: No tracked file names a variable starting with a digit or an old name + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Test the check + shell: pwsh + run: Invoke-Pester -Path ./ci/check-environment-variable-names.Tests.ps1 -CI + + - name: Check the names + shell: pwsh + run: ./ci/check-environment-variable-names.ps1 diff --git a/AGENTS.md b/AGENTS.md index 971eb5d05..7a0e7b924 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.c ``` The example downloads the Lite file first when it is not there. In place of -the argument, the `51DEGREES_IPI_PATH` environment variable or system property +the argument, the `_51DEGREES_IPI_PATH` environment variable or system property can give the absolute path of a data file that is already there. On Java 24 or later, add `--enable-native-access=ALL-UNNAMED` before the class @@ -183,7 +183,7 @@ java -cp examples/device-detection/console/target/device-detection-java-examples It prints the same `Input values:` and `Results:` blocks as the on-premise example. The IP Intelligence cloud examples read their key from -`51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and +`_51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and `examples/ip-intelligence/README.md` gives the keys to create for them. ### Pitfalls diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 new file mode 100644 index 000000000..7b9273d80 --- /dev/null +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -0,0 +1,92 @@ +# Tests for ci/check-environment-variable-names.ps1. +# +# Each case is one line of text and what the check must make of it. The +# digit-initial name is matched in any case, because Windows reads an +# environment variable name in any case, so a lower case old name is still +# the old name there. The old names keep their case, for the reason the check +# gives. + +BeforeAll { + $script:Check = Join-Path $PSScriptRoot 'check-environment-variable-names.ps1' + . $script:Check + + function Get-Kind([string]$Line) { + @(Find-EnvironmentNameProblem $Line | ForEach-Object { $_.Kind }) + } +} + +Describe 'Find-EnvironmentNameProblem' { + + It 'finds the digit-initial name ' -ForEach @( + @{ Line = 'System.getenv("51DEGREES_IPI_PATH")' } + @{ Line = 'public static final String IPI = "51DEGREES_IPI_PATH";' } + @{ Line = '${51DEGREES_DD_PATH:-lite.hash}' } + @{ Line = "'51DEGREES_DD_PATH' = (Resolve-Path `$file).Path" } + @{ Line = 'System.getenv("51degrees_ipi_path")' } + @{ Line = '$env:51Degrees_Resource_Key = "key"' } + ) { + Get-Kind $Line | Should -Contain 'digit' + } + + It 'finds the old name ' -ForEach @( + @{ Line = 'System.getenv("RESOURCE_KEY")' } + @{ Line = 'env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY");' } + @{ Line = 'env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY");' } + @{ Line = 'public static final String FOD = "FOD_CLOUD_API_URL";' } + @{ Line = '$env:LICENSE_KEY = $Keys.TestResourceKey' } + @{ Line = '${env:LICENSE_KEY} = $Keys.TestResourceKey' } + @{ Line = '${Env:RESOURCE_KEY}' } + @{ Line = '$ENV:RESOURCE_KEY = "key"' } + @{ Line = 'Remove-Item Env:\LICENSE_KEY' } + @{ Line = '${env:FOD_CLOUD_API_URL}' } + @{ Line = ' url: ${{ env.FOD_CLOUD_API_URL }}' } + @{ Line = '${FOD_CLOUD_API_URL}' } + @{ Line = 'Constants.LEGACY_ENV_VARS' } + @{ Line = ' LICENSE_KEY: ${{ secrets.DEVICE_DETECTION_KEY }}' } + @{ Line = '- RESOURCE_KEY names the resource key' } + @{ Line = 'RESOURCE_KEY=my-key mvn test' } + ) { + Get-Kind $Line | Should -Contain 'fallback' + } + + # The cloud request engine's tests name the query string parameter + # resource_key, which is on the wire and not in the environment, so the + # old names keep their case. + It 'leaves alone' -ForEach @( + @{ Line = 'assertEquals("resource_key=" + key, query);' } + @{ Line = ' DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }}' } + @{ Line = ' PAID_RESOURCE_KEY: ${{ secrets.X }}' } + @{ Line = 'System.getenv("_51DEGREES_RESOURCE_KEY")' } + @{ Line = 'System.getenv("_51DEGREES_CLOUD_ENDPOINT")' } + @{ Line = '$env:_51DEGREES_DD_PATH = "file.hash"' } + @{ Line = '$env:_51degrees_dd_path = "file.hash"' } + @{ Line = '${env:_51DEGREES_LICENSE_KEY} = $Keys.TestResourceKey' } + @{ Line = 'assertEquals("${env:user.home}", substitutor.replace(line));' } + @{ Line = ' PAID_RESOURCE_KEY: ${{ env.PAID_RESOURCE_KEY }}' } + @{ Line = 'System.getenv("SUPER_RESOURCE_KEY")' } + @{ Line = 'Set the key with !!YOUR_RESOURCE_KEY!! in the readme.' } + ) { + Get-Kind $Line | Should -BeNullOrEmpty + } +} + +Describe 'the check over a repository' { + + # git quotes a path holding a non-ASCII character unless told not to, + # and a quoted path is not found on disk. + It 'reads a file whose name is not ASCII' { + $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) + New-Item -ItemType Directory $repo | Out-Null + try { + git -C $repo init -q + $name = 'Donn' + [char]0xE9 + 'es.java' + Set-Content -LiteralPath (Join-Path $repo $name) ` + -Value 'System.getenv("51DEGREES_DD_PATH");' + git -C $repo add -A + { & $script:Check -RepoRoot $repo } | + Should -Throw '1 names start with a digit*' + } finally { + Remove-Item -Recurse -Force $repo + } + } +} diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 new file mode 100644 index 000000000..6fa467c2d --- /dev/null +++ b/ci/check-environment-variable-names.ps1 @@ -0,0 +1,124 @@ +# Every 51Degrees environment variable name in this repository starts with an +# underscore, because a POSIX shell cannot set a name that starts with a digit, +# and each variable has one name with nothing read in its place. This reads +# every file git tracks, which leaves out build output and the submodules, and +# fails on any name that starts with the digit and on any old name that is +# still read. +# +# Dot-sourcing the script defines Find-EnvironmentNameProblem without reading +# the repository, which is how its tests use it. + +[CmdletBinding()] +param( + # The root of the checkout to read. The default is the repository this + # script lives in. + [string]$RepoRoot = (Join-Path $PSScriptRoot '..') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# The start of a name a shell cannot set, split so this file does not match +# itself. +$digitInitial = '5' + '1DEGREES_' + +# A name is not matched when a name character comes before it, so a name with +# a leading underscore is left alone. It is matched in any case, because +# Windows reads an environment variable name in any case, so a lower case +# name that starts with the digit is still one a shell cannot set. +$script:DigitInitialPattern = [regex]::new( + "(? * The configuration file for the pipeline is at src/main/webapp/WEB-INF/51Degrees-OnPrem.xml *

    - * The data file is taken from the 51DEGREES_DD_PATH environment variable, then + * The data file is taken from the _51DEGREES_DD_PATH environment variable, then * from TestDataFile (an environment variable or system property), and otherwise * the free Lite file in cxx/data/device-detection is used, which is downloaded * before the server starts if it is missing. diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index b74ec24f8..5ec7feb45 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -28,11 +28,11 @@ false false - ${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} + ${_51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} LowMemory DeviceDetectionHashEngine diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java index 108fbd615..c77110b95 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java @@ -35,12 +35,12 @@ /** * Checks the order in which the on-premise web example picks its data file, - * being 51DEGREES_DD_PATH first, then TestDataFile, then the Lite file. + * being _51DEGREES_DD_PATH first, then TestDataFile, then the Lite file. * System properties stand in for the environment variables, because the * lookup reads either and a test cannot set an environment variable. */ public class OnPremDataFileOptionTest { - private static final String DD_PATH = "51DEGREES_DD_PATH"; + private static final String DD_PATH = "_51DEGREES_DD_PATH"; private static final String TEST_DATA_FILE = "TestDataFile"; private static final String LITE = "cxx/data/device-detection/51Degrees-LiteV4.1.hash"; diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 0359dbaab..e3775778c 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -28,7 +28,7 @@ gives a link to ask for a copy. The Compare, Metrics and Suspicious examples nee properties, so with a free data file they say which they need and stop. [Contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=enterprise-data-file) for a copy of the Enterprise data file. To use it, place it in `cxx/data/ip-intelligence` and pass its path -as the first argument, or set the `51DEGREES_IPI_PATH` environment variable or system +as the first argument, or set the `_51DEGREES_IPI_PATH` environment variable or system property to its path. The Enterprise data file is over 6 GB, and the MaxPerformance profile loads all of it into @@ -61,7 +61,7 @@ Compare and Suspicious examples with the Enterprise data file are skipped. The examples locate the data file in the following order: -1. The `51DEGREES_IPI_PATH` environment variable or system property, which can +1. The `_51DEGREES_IPI_PATH` environment variable or system property, which can be set to an explicit path to the data file. It is used in place of any file named without a full path. 2. The path given as the first argument, or the free file the example uses @@ -104,7 +104,7 @@ environment variable or system property. | MetadataCloud | How to access the meta-data that relates to the properties available from the cloud IP intelligence service. | Cloud examples need a resource key. The key is read from the -`51DEGREES_RESOURCE_KEY` environment variable or system property first. The +`_51DEGREES_RESOURCE_KEY` environment variable or system property first. The legacy `TestResourceKey` name is still supported and is checked second. The cloud property tiers changed in May 2026. The examples and this diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index c800a3d37..8bb5ea91b 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -184,7 +184,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws // We use the low memory profile, which reads the data file // from disk as it is needed, because this example may be // given the Enterprise data file, which is over 6 GB, through - // its first argument or 51DEGREES_IPI_PATH. MaxPerformance + // its first argument or _51DEGREES_IPI_PATH. MaxPerformance // loads the whole data file into memory, which makes each // lookup faster where there is memory to hold it. .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index 19c44616e..66e118cd2 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -133,7 +133,7 @@ public static String findAvailableDataFile(String... candidates) { * hold an explicit path to the IP Intelligence data file. This name is * checked first, before any search of the project space. */ - public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; + public static final String IPI_PATH_ENV_VAR = "_51DEGREES_IPI_PATH"; public static class DatafileInfo { FiftyOneDataFile fileInfo; diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java index 846c714f6..acb4fcf7c 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java @@ -35,7 +35,7 @@ public class KeyHelper { * hold the resource key. This name is checked first, before the legacy * {@link #TEST_RESOURCE_KEY} name. */ - public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; + public static final String RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"; public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java index 67b2660cb..809b7382f 100644 --- a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java @@ -88,7 +88,7 @@ public void requireResourceKey_NoKey_FailsNamingEveryName() { assertEquals("This test calls the live 51Degrees cloud service, so " + "it needs a resource key and there is nothing to fall back " + "on. None of _51DEGREES_RESOURCE_KEY, " + - "_51DEGREES_RESOURCE_KEY_BESPOKE, 51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, _51DEGREES_RESOURCE_KEY, " + "TestResourceKey is set, as an environment variable or as a " + "system property, tried in that order. The key has to carry " + "the ip and device products, because the test asks the " + diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index e653d488c..e5c19598b 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -35,7 +35,7 @@ data file. The free 'Lite' and Asn data files come from the [data](https://github.com/51Degrees/data) repository, which is the `cxx/data` folder of this repository. Run `cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. Otherwise [contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) to obtain an 'Enterprise' data file. For tests, the data file can be supplied explicitly by setting the -`51DEGREES_IPI_PATH` environment variable or system property to the path of +`_51DEGREES_IPI_PATH` environment variable or system property to the path of the .ipi file. When it is not set, the project space is searched for the expected file name, with the free 'Lite' file expected in `cxx/data/ip-intelligence`. @@ -112,7 +112,7 @@ You will need [resource keys](https://51degrees.com/documentation/_services__clo The tests look for a resource key in the following order: -1. The aligned `51DEGREES_RESOURCE_KEY` name, set as an environment variable +1. The aligned `_51DEGREES_RESOURCE_KEY` name, set as an environment variable or as a system property. 2. The legacy `TestResourceKey` name, which is still supported and is checked after the aligned name. diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java index a39dd6b80..d3db631eb 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java @@ -43,7 +43,7 @@ public class FileUtils { * supply an explicit path to the IPI data file. When set and the file * exists it is used in preference to searching the project space. */ - public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; + public static final String IPI_PATH_ENV_VAR = "_51DEGREES_IPI_PATH"; public static final String ENTERPRISE_IPI_DATA_FILE_NAME = "Enterprise-IpIntelligenceV41.ipi"; diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java index cabb6648c..8c6b1f225 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java @@ -33,7 +33,7 @@ public class KeyUtils { * Name of the aligned environment variable or system property which * supplies the resource key. This name is checked before any legacy name. */ - public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; + public static final String RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"; /** * Obtain a resource key from an environment variable or a system property. diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java index a5eb58df6..028bec978 100644 --- a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java @@ -50,7 +50,7 @@ * complete, localized, ordered country dropdown is produced. *

    * The data file is resolved from the {@code TestDataFile} system property, the - * {@code 51DEGREES_IPI_PATH} environment variable, or a few well known + * {@code _51DEGREES_IPI_PATH} environment variable, or a few well known * locations, and the test is skipped if none is found. *

    * The IP Intelligence element data key has differed between engine versions @@ -66,7 +66,7 @@ public class CountryTranslationIntegrationTest { private static String resolveDataFile() { String[] candidates = { System.getProperty("TestDataFile"), - System.getenv("51DEGREES_IPI_PATH"), + System.getenv("_51DEGREES_IPI_PATH"), "51Degrees-IPIV4EnterpriseIpiV41.ipi", "../51Degrees-IPIV4EnterpriseIpiV41.ipi", "../../51Degrees-IPIV4EnterpriseIpiV41.ipi", diff --git a/pipeline/README.md b/pipeline/README.md index d59dc839d..22e8e8118 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -131,13 +131,13 @@ and on our [website](https://51degrees.com/documentation/_device_detection__exam | pipeline.developer-examples.onpremise-engine | Shows how to modify SimpleFlowElement to make use of the 'engine' functionality and use a custom data file to map dates to star signs rather than relying on hard coded data. | | pipeline.developer-examples.clientside-element | Shows how to modify SimpleFlowElement to request the data of birth from the user using client-side JavaScript. | | pipeline.developer-examples.clientside-element-mvc | An example project showing how to use the code from SimpleClientSideElement in a Java web application using the Model-View-Controller Pattern. | -| pipeline.developer-examples.cloud-engine | Shows how to modify SimpleFlowElement to perform the star sign lookup via a cloud service rather than locally. The endpoint and resource key come from `FOD_CLOUD_API_URL` and `_51DEGREES_RESOURCE_KEY` when set, otherwise the star sign service the example was written for. | +| pipeline.developer-examples.cloud-engine | Shows how to modify SimpleFlowElement to perform the star sign lookup via a cloud service rather than locally. The endpoint and resource key come from `_51DEGREES_CLOUD_ENDPOINT` and `_51DEGREES_RESOURCE_KEY` when set, otherwise the star sign service the example was written for. | | pipeline.developer-examples.usage-sharing | Shows how to share usage with 51Degrees. This helps us to keep our products up to date and accurate. | | pipeline.developer-examples.fodid | Shows how to read a 51Did, and, in a web demo, how a browser creates and verifies one against the 51Degrees cloud and the server redeems the encrypted result, so that the creator context proves the identifier is being presented by the browser it was created on. The module README describes the flow, the environment variables, what a run costs and the redeem call to copy into your own server. | ### Pointing the cloud examples at another host -Every example that calls the 51Degrees cloud takes its endpoint from the `FOD_CLOUD_API_URL` environment variable, which is the cloud API base including the `/api/v4/` segment and defaults to `https://cloud.51degrees.com/api/v4/` (the cloud-engine example defaults to the star sign service it was written for, as its row above says). This is the same variable the cloud request engine in this repository honours, so setting it once points every example, and any pipeline built with the engine, at the same place. The resource key comes from `_51DEGREES_RESOURCE_KEY`, or the older `RESOURCE_KEY`. +Every example that calls the 51Degrees cloud takes its endpoint from the `_51DEGREES_CLOUD_ENDPOINT` environment variable, which is the cloud API base including the `/api/v4/` segment and defaults to `https://cloud.51degrees.com/api/v4/` (the cloud-engine example defaults to the star sign service it was written for, as its row above says). This is the same variable the cloud request engine in this repository honours, so setting it once points every example, and any pipeline built with the engine, at the same place. The resource key comes from `_51DEGREES_RESOURCE_KEY`. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons. This is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index b0d420a0b..29c27b960 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -87,12 +87,12 @@ protected CloudRequestEngine newEngine(List properties) throws Exception /* * Check if endPoint has been explicitly set via setEndpoint. - * If not check for environment variable FOD_CLOUD_API_URL. + * If not check for environment variable _51DEGREES_CLOUD_ENDPOINT. * If nothing else is set, use default value. */ if (endPoint == null || endPoint.isEmpty()) { String envVarEndPoint = getEnvVar.apply( - Constants.FOD_CLOUD_API_URL); + Constants.CLOUD_ENDPOINT_VARIABLE); if (envVarEndPoint == null || envVarEndPoint.isEmpty()) { setEndpoint(Constants.END_POINT_DEFAULT); } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java index b9b26c6ae..1a77c3537 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java @@ -27,7 +27,7 @@ public class Constants { public static final String END_POINT_DEFAULT = "https://cloud.51degrees.com/api/v4"; // Environment variable name to set end point - public static final String FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL"; + public static final String CLOUD_ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; // Default timeout for each request to the cloud service, as the // specification's Configuration options table gives. public static final int DEFAULT_TIMEOUT_MILLIS = 2000; diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java index 1ef246c77..8f8487633 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java @@ -52,8 +52,8 @@ * is built, and the JSON request when it processes, so a change in the * service is seen here. *

    - * {@code FOD_CLOUD_API_URL} points the engine at another host, which the - * builder reads for itself. + * {@code _51DEGREES_CLOUD_ENDPOINT} points the engine at another host, + * which the builder reads for itself. */ public class CloudRequestEngineLiveTests { diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java index 55be783d3..6bb87c5fe 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java @@ -40,7 +40,6 @@ final class LiveCloudKeys { */ static final String[] RESOURCE_KEY_NAMES = { "_51DEGREES_RESOURCE_KEY", - "RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", "TestResourceKey", "SUPER_RESOURCE_KEY", @@ -53,7 +52,6 @@ final class LiveCloudKeys { */ static final String[] LICENCE_KEY_NAMES = { "_51DEGREES_LICENSE_KEY", - "LICENSE_KEY", "_51DEGREES_LICENSE_KEY_BESPOKE", "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", "TestLicenseKey", diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index a1011c612..4b039d94c 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -54,9 +54,9 @@ public class Main { "https://cloud.51degrees.com/api/v4/"; /** - * The cloud endpoint, taken from FOD_CLOUD_API_URL when that is set, - * which is the variable every 51Degrees cloud example honours and the - * one the cloud request engine builder reads by itself when no + * The cloud endpoint, taken from _51DEGREES_CLOUD_ENDPOINT when that is + * set, which is the variable every 51Degrees cloud example honours and + * the one the cloud request engine builder reads by itself when no * endpoint is given. A host other than cloud.51degrees.com would be * used to (a) use an on premise web server, or (b) use a privately * hosted version of the 51Degrees cloud for performance reasons, @@ -66,7 +66,7 @@ public class Main { * directly. */ static String endpoint() { - String value = env("FOD_CLOUD_API_URL"); + String value = env("_51DEGREES_CLOUD_ENDPOINT"); if (value == null) { // A resource key and an endpoint travel together, because a // key is only known to the service that issued it. So the @@ -91,7 +91,6 @@ static String endpoint() { */ static final String[] RESOURCE_KEY_NAMES = { "_51DEGREES_RESOURCE_KEY", - "RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", "TestResourceKey", "SUPER_RESOURCE_KEY", @@ -132,10 +131,10 @@ static String env(String name) { public static class Example { public void run() throws Exception { //! [usage] - // The endpoint comes from FOD_CLOUD_API_URL when it is set, so - // the example can be pointed at an on premise web server or a - // privately hosted version of the 51Degrees cloud, and - // otherwise at the star sign service it was written for. + // The endpoint comes from _51DEGREES_CLOUD_ENDPOINT when it is + // set, so the example can be pointed at an on premise web + // server or a privately hosted version of the 51Degrees cloud, + // and otherwise at the star sign service it was written for. CloudRequestEngine cloudRequestEngine = new CloudRequestEngineBuilder(loggerFactory, httpClient) .setEndpoint(endpoint()) diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md index 586556167..73ab80af9 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md @@ -51,7 +51,7 @@ The only server-side part of the flow is the redeem call, which adds the licence key the browser never sees. The `redeem` method in `CreatorContextDemoServer.java` is that call, and these are its essential lines. `DidClient` is built once at start-up from the -resource key and the licence key (it reads `FOD_CLOUD_API_URL` itself +resource key and the licence key (it reads `_51DEGREES_CLOUD_ENDPOINT` itself and falls back to the public cloud), and `did`, `result` and `challenge` are what the page passed on from `verify-full`: @@ -90,9 +90,9 @@ scope. | Variable | Meaning | | --- | --- | -| `_51DEGREES_RESOURCE_KEY` | Required. The resource key of the page, public by nature. The older name `RESOURCE_KEY` is read when this one is not set. | -| `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, server side only. The older name `LICENSE_KEY` is read when this one is not set. Only an account that holds licence keys needs one to redeem, because the licence key is what keeps redemption to the acting party's own servers, so an account holding none redeems without one. | -| `FOD_CLOUD_API_URL` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine in this repository honours, so setting it once points every 51Degrees example at the same place. A trailing slash is added where missing. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. | +| `_51DEGREES_RESOURCE_KEY` | Required. The resource key of the page, public by nature. | +| `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, server side only. Only an account that holds licence keys needs one to redeem, because the licence key is what keeps redemption to the acting party's own servers, so an account holding none redeems without one. | +| `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine in this repository honours, so setting it once points every 51Degrees example at the same place. A trailing slash is added where missing. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. | | `PORT` | The port to listen on, defaulting to `5100`. | ### How to run diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java index 02c7e8326..1057eb480 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java @@ -101,11 +101,11 @@ * {@code fodid/creator-context/} in this module's resources. Environment * variables: *

      - *
    • {@code _51DEGREES_RESOURCE_KEY}, or the older {@code RESOURCE_KEY}, - * required. The resource key of the page, public by nature.
      • - *
    • {@code _51DEGREES_LICENSE_KEY}, or the older {@code LICENSE_KEY}, - * optional. A licence key of the same account, server side only.
      • - *
    • {@code FOD_CLOUD_API_URL}, optional. The cloud API base including + *
    • {@code _51DEGREES_RESOURCE_KEY}, required. The resource key of the + * page, public by nature.
      • + *
    • {@code _51DEGREES_LICENSE_KEY}, optional. A licence key of the same + * account, server side only.
      • + *
    • {@code _51DEGREES_CLOUD_ENDPOINT}, optional. The cloud API base including * the {@code /api/v4/} segment, defaulting to * {@code https://cloud.51degrees.com/api/v4/}. This is the same variable * the cloud request engine of this package honours.
      • @@ -119,9 +119,9 @@ public class CreatorContextDemoServer { static final String RESOURCES = "/fodid/creator-context/"; static final String RESOURCE = - env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); + env("_51DEGREES_RESOURCE_KEY"); static final String LICENCE = - env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); + env("_51DEGREES_LICENSE_KEY"); /** * The one client, built at start-up and shared by every request, @@ -131,8 +131,8 @@ public class CreatorContextDemoServer { public static void main(String[] args) throws Exception { if (RESOURCE == null) { - System.err.println("Set _51DEGREES_RESOURCE_KEY (or the older " - + "RESOURCE_KEY) to the resource key of the page."); + System.err.println( + "Set _51DEGREES_RESOURCE_KEY to the resource key of the page."); System.exit(1); } if (LICENCE == null) { @@ -149,7 +149,7 @@ public static void main(String[] args) throws Exception { + "will report the context unreadable where it holds " + "some."); } - // The client reads FOD_CLOUD_API_URL itself when no endpoint is + // The client reads _51DEGREES_CLOUD_ENDPOINT itself when no endpoint is // given, and falls back to the public cloud. A host other than // cloud.51degrees.com would be used to (a) use an on premise web // server, or (b) use a privately hosted version of the 51Degrees diff --git a/pipeline/pipeline.did/README.md b/pipeline/pipeline.did/README.md index 8784643ad..2b5fdea00 100644 --- a/pipeline/pipeline.did/README.md +++ b/pipeline/pipeline.did/README.md @@ -357,7 +357,7 @@ import fiftyone.pipeline.did.RedeemResult; // The resource key is the page's and public by nature. The licence key is // server side only and is needed to redeem where the account holds licence // keys. The endpoint defaults to https://cloud.51degrees.com/api/v4/, or -// the FOD_CLOUD_API_URL environment variable where that is set. +// the _51DEGREES_CLOUD_ENDPOINT environment variable where that is set. DidClient client = new DidClient(resourceKey, licenceKey); ``` diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index 9afb29af1..cc4e28b92 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -114,7 +114,7 @@ public final class DidClient { * The environment variable read for the API base when the constructor * is given none, the same one the cloud request engine honours. */ - public static final String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; + public static final String ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; /** * How old the whole held key list may be before a request for a key diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java index a91f840a1..ea6055154 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java @@ -48,8 +48,8 @@ * Tests against the live cloud service. They need a resource key, read * from the names {@link LiveCloudKeys} lists, and they fail rather than * skip when there is none, because a run that called nothing must not - * report success. {@code FOD_CLOUD_API_URL} points them at another host, - * and a licence key is taken from the names {@link LiveCloudKeys} lists + * report success. {@code _51DEGREES_CLOUD_ENDPOINT} points them at another + * host, and a licence key is taken from the names {@link LiveCloudKeys} lists * where the account holds one. Each test creates a 51Did through the * cloud {@code json} endpoint, which is one use against the resource key, * plus one for each cloud call it then makes. @@ -103,9 +103,9 @@ public void redeem_GarbageResult_IsUnreadable() throws Exception { "The cloud service at " + client.getEndpoint() + " answered that it does not offer the creator " + "context, so the redeem endpoint could not be " - + "called. Point FOD_CLOUD_API_URL at a service that " - + "offers it, or leave it unset for the 51Degrees " - + "cloud.", reported.getCause()); + + "called. Point _51DEGREES_CLOUD_ENDPOINT at a " + + "service that offers it, or leave it unset for the " + + "51Degrees cloud.", reported.getCause()); } throw reported; } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java index ba5efdc30..26061200e 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java @@ -45,7 +45,6 @@ final class LiveCloudKeys { */ static final String[] RESOURCE_KEY_NAMES = { "_51DEGREES_RESOURCE_KEY", - "RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", "TestResourceKey", "SUPER_RESOURCE_KEY", @@ -58,7 +57,6 @@ final class LiveCloudKeys { */ static final String[] LICENCE_KEY_NAMES = { "_51DEGREES_LICENSE_KEY", - "LICENSE_KEY", "_51DEGREES_LICENSE_KEY_BESPOKE", "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", "TestLicenseKey", From 03c3dd9eaca0b90b8fab1ddf6791fad05ee78450 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 20:26:39 +0000 Subject: [PATCH 84/95] Send one Evidence name once, whatever its letter case, by precedence (#107) * Test that one Evidence name in two letter cases is sent once by precedence Evidence keys ignore letter case, so header.User-Agent and query.user-agent name one field, which is sent once with the name and value of the key that wins by the precedence in evidence.md: query, header, cookie, server, fiftyone and location, then any other prefix alphabetically. * Send one Evidence name once, whatever its letter case, by precedence Evidence keys ignore letter case, so the form fields are now keyed without regard to case. One name under two prefixes, such as header.User-Agent and query.user-agent, is sent once, with the name and value of the key that wins, where before both were sent. The other prefixes now follow evidence.md's order, being server, fiftyone and location, then any other prefix alphabetically, where before server, fiftyone and location were ranked alphabetically among the rest. * Test that two other prefixes are compared as prefixes ab.value ranks before ab-c.Value, because the prefix ab sorts before ab-c, where comparing the whole key put ab-c first, as a hyphen sorts before a dot. _x ranks before a, as the prefixes compare in lower case. * Compare two other prefixes as prefixes, in lower case Two keys under other prefixes were ordered by the whole key, so ab-c.n ranked before ab.n because a hyphen sorts before a dot. evidence.md ranks prefixes, so the prefixes are compared, in lower case as Go, Node, PHP and Python compare them, and the whole key only breaks a tie. --- .../CloudRequestEngineDefault.java | 58 ++++++++++++++++--- .../flowelements/CloudRequestEngineTests.java | 57 ++++++++++++++++++ 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 2e2571081..9ba26c7ee 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -131,6 +131,13 @@ public class CloudRequestEngineDefault private final CloudEvidenceKeyFilter evidenceKeyFilter = new CloudEvidenceKeyFilter(); + /** + * The prefixes after query, header and cookie, highest precedence + * first, as evidence.md lists them. + */ + private static final String[] OTHER_PREFIX_PRECEDENCE = + { "server", "fiftyone", "location" }; + /** * Held while the discovery requests are made, so that concurrent first * uses make them once. @@ -469,9 +476,11 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { /** * The Evidence to send, keyed by the name the cloud service expects. - * Only Evidence that {@link #isSent(String)} allows is included, and - * where two prefixes give the same name the precedence order decides, - * query first, then header, then cookie, then any other prefix. + * Only Evidence that {@link #isSent(String)} allows is included. Keys + * ignore letter case, so one name under two prefixes is one field, + * sent with the name and value of the key that wins by the precedence + * in evidence.md, being query, header, cookie, server, fiftyone and + * location, then any other prefix alphabetically. * Evidence with no prefix ranks with the other prefixes, unless the * whole key is query, header or cookie, as {@link #keyPrefix(String)} * gives the whole key as its prefix. @@ -486,9 +495,12 @@ Map getFormData(FlowData flowData) { evidence.put(entry.getKey(), entry.getValue()); } } - Map formData = new HashMap<>(); + Map formData = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - // Add evidence in reverse alphabetical order, excluding special keys. + // Add the other evidence lowest precedence first, being other + // prefixes in reverse alphabetical order, then location, fiftyone + // and server. addFormData(formData, evidence, getSelectedEvidence(evidence, "other")); // Add cookie evidence. addFormData(formData, evidence, getSelectedEvidence(evidence, "cookie")); @@ -552,8 +564,16 @@ private void addFormData( Map allEvidence, Map evidence) { - List evidenceKeys = Arrays.asList(evidence.keySet().toArray(new String[0])); - Collections.sort(evidenceKeys, Collections.reverseOrder()); + // Lowest precedence first, so a later key replaces an earlier one. + // Other prefixes are compared as prefixes, in lower case, then the + // whole key, so that the order is the same whatever the map gives. + List evidenceKeys = new ArrayList<>(evidence.keySet()); + evidenceKeys.sort(Comparator + .comparingInt((String key) -> otherPrefixRank(keyPrefix(key))) + .thenComparing((String key) -> + keyPrefix(key).toLowerCase(Locale.ROOT)) + .thenComparing((String key) -> key.toLowerCase(Locale.ROOT)) + .reversed()); for (String evidenceKey : evidenceKeys) { String prefix = keyPrefix(evidenceKey); @@ -572,7 +592,9 @@ private void addFormData( if (prefix.equalsIgnoreCase("query") == false) { Map conflicts = new HashMap<>(); for (String key : allEvidence.keySet()) { - if (key.equals(evidenceKey) == false && key.contains(suffix)) { + if (key.equals(evidenceKey) == false && + key.toLowerCase(Locale.ROOT).contains( + suffix.toLowerCase(Locale.ROOT))) { conflicts.put(key, allEvidence.get(key)); } } @@ -592,12 +614,30 @@ private void addFormData( conflictStr.toString()); logger.warn(warningMessage); } - // Overwrite the existing queryParameter value. + // Replace the existing field, name and value, so the name + // sent is the winning key's. + formData.remove(suffix); formData.put(suffix, evidence.get(evidenceKey)); } } } + /** + * The precedence of a prefix that is not query, header or cookie, + * lower first, being server, fiftyone and location as evidence.md + * lists them, then any other prefix. + * @param prefix the prefix + * @return the rank, where a lower rank wins + */ + private static int otherPrefixRank(String prefix) { + for (int i = 0; i < OTHER_PREFIX_PRECEDENCE.length; i++) { + if (OTHER_PREFIX_PRECEDENCE[i].equalsIgnoreCase(prefix)) { + return i; + } + } + return OTHER_PREFIX_PRECEDENCE.length; + } + /** * Get evidence with specified prefix. * @param evidence the Evidence being sent diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 994408d1e..3a64e5f5a 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -51,6 +51,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -539,6 +540,62 @@ public void getFormData(String name, Map evidence, String expect } } + private static Stream oneNameInTwoCasesArgs() { + return Stream.of( + Arguments.of("header.User-Agent", "query.user-agent", "user-agent", "B"), + Arguments.of("query.User-Agent", "header.user-agent", "User-Agent", "A"), + Arguments.of("cookie.X-Thing", "header.x-thing", "x-thing", "B"), + Arguments.of("location.Value", "server.value", "value", "B"), + Arguments.of("location.Value", "fiftyone.value", "value", "B"), + Arguments.of("fiftyone.Value", "server.value", "value", "B"), + Arguments.of("server.Value", "a.value", "Value", "A"), + Arguments.of("b.Value", "a.value", "value", "B"), + Arguments.of("ab-c.Value", "ab.value", "value", "B"), + Arguments.of("a.Value", "_x.value", "value", "B")); + } + + /** + * Evidence keys ignore letter case, so one name under two prefixes in + * two letter cases is one field. It is sent once, with the name and + * value of the key that wins by the precedence in evidence.md, being + * query, header, cookie, server, fiftyone and location, then any other + * prefix alphabetically. Two other prefixes are compared as prefixes, + * in lower case, so ab ranks before ab-c and _x before a. + */ + @ParameterizedTest(name = "{0} and {1}") + @MethodSource("oneNameInTwoCasesArgs") + public void getFormData_OneNameInTwoCases_SentOnceByPrecedence( + String first, + String second, + String expectedName, + String expectedValue) throws Exception { + acceptEvidence(Arrays.asList(first, second)); + configureMockedClient(); + CloudRequestEngineDefault engine = (CloudRequestEngineDefault) + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence(first, "A"); + data.addEvidence(second, "B"); + + Map result = engine.getFormData(data); + List names = new ArrayList<>(); + for (String key : result.keySet()) { + if (key.equalsIgnoreCase(expectedName)) { + names.add(key); + } + } + assertEquals(Collections.singletonList(expectedName), names, + "The name must be sent once: " + result); + assertEquals(expectedValue, result.get(expectedName)); + } + } + /** * The web integration adds the whole web request as server.http-request * for usage sharing to read. It holds every header and cookie, so it is From 4bec158f2823b4065b1f6cbe28c7b6912bad9302 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 22:53:11 +0000 Subject: [PATCH 85/95] Authenticate a Cloud Request Engine on a license key alone, and send the properties named (#97) * Test the credentials a Cloud Request Engine accepts and sends Cover the rows of the specification's table of credentials, the data request for a license key alone, a resource key alone, both keys, a resource key with requested properties and all three, the accessible properties request for each, the report of requested properties the answer left out, and the license key staying out of every URL, log entry and error message when a request fails. * Authenticate a Cloud Request Engine on a license key alone A license key without a resource key is accepted when the properties wanted are named. The properties set with setProperty or setProperties, which reached newEngine and were never used, now go to the cloud service as 'values' whenever the list is not empty, beside a resource key, a license key or both, so the service adds them to its answer. The builder refuses at build time only the two combinations the cloud service refuses on every request, being no credential at all and a license key with no requested properties, each with a message that says why the combination cannot work. When a license key is set, the accessible properties request is a POST that carries the license key, and the resource key if there is one, in its body, so discovery finds what the license key grants and the key never appears in a URL. A resource key alone is sent in the query string as before. The engine warns once about requested properties the answer left out. A blank resource key counts as absent, so the data endpoint for a license key alone is the plain json endpoint. * Send only the properties named with setRequestedProperties The previous commit sent the properties set with setProperty and setProperties as 'values'. On rc that list reached newEngine and was never sent, so a configuration that set it, as the .NET cloud web examples do, would change the cloud service's answer. The .NET full build showed that answer losing device.deviceid. The engine builder now keeps its own list, set with setRequestedProperties as a list or as one comma separated string, and only that list is sent. setProperty and setProperties behave exactly as they did on rc, and the refusal for a license key with no requested properties names setRequestedProperties. * Count only the requests a refused build sends in the license key tests * Test that requested property names are sent as given, with two report texts As in .NET, the names sent as values keep the letter case they were given in, and a name given twice in different cases is sent once as first given. With a license key, names the answer leaves out are reported as not covered by the keys, and with a resource key, requested properties and no license key, as not properties, as cloud-request-engine.md says. A null list is refused with the parameter's name. * Send requested property names as given, and say when one is not a property The cloud service reads property names without regard to letter case, so the names sent as values keep the case they were given in, and a repeat in another case is dropped, as .NET now does. A null list of requested properties is refused with an IllegalArgumentException naming the parameter, rather than a NullPointerException from inside the list. With a resource key, requested properties and no license key, a property the resource key does not carry is answered as null with its reason, so a name the answer leaves out is not a property. The warning for that case says so and names no license key, as cloud-request-engine.md asks. * Word the warning for a name that is not a property without the license key The warning for a resource key with requested properties and no license key names no license key, because none was given. * Record the license key tests' fake cloud calls in a synchronized list Discovery on rc fetches the evidence keys on another thread while it fetches the accessible properties, so the fake cloud records calls from two threads. A plain ArrayList could lose a call or hold a null, which failed the resource key discovery and data request tests depending on the order the tests ran in. rc's own discovery tests use a synchronized list for the same reason. * Answer the test cloud's POSTs by URL, as its GETs are rc's Cloud Request Engine tests build engines with a license key, and a license key makes the discovery requests POSTs, which the shared mock answered with the data response. The mock now answers a POST to the accessible properties or evidence keys endpoint as it answers a GET, and the three tests that count the data request count only that request. * Record the license key tests' fake connections in a concurrent map The fake cloud looked each request's URL up in a HashMap that both discovery threads wrote to, so a lost entry left a call with no URL and failed Process_ResourceKeyAndLicenseKey_SendsBoth in one run. --- .../cloudrequestengine/Constants.java | 29 + .../CloudRequestEngineBuilder.java | 126 +++- .../CloudRequestEngineDefault.java | 146 +++- .../CloudRequestEngineLicenseKeyTests.java | 640 ++++++++++++++++++ .../flowelements/CloudRequestEngineTests.java | 15 +- .../CloudRequestEngineTestsBase.java | 16 +- 6 files changed, 946 insertions(+), 26 deletions(-) create mode 100644 pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index e957273ee..eebb82eb4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -27,6 +27,35 @@ public class Constants { public static final String OriginHeaderName = "Origin"; public class Messages { + public static final String NoCredential = + "The Cloud Request Engine needs a resource key or a license key " + + "to authenticate with, as the cloud service answers 401 to a " + + "request that carries neither. Use setResourceKey(String) with " + + "a resource key created for free at " + + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=no-credential"; + + public static final String LicenseKeyNeedsProperties = + "The Cloud Request Engine has a license key and no requested " + + "properties. A license key names no properties of its own, so " + + "the cloud service answers 400 to every request made with one " + + "that does not list the properties it wants. Use " + + "setRequestedProperties to name the fully qualified " + + "properties wanted, for example \"device.ismobile\" and " + + "\"device.iscrawler\"."; + + public static final String PropertiesNotCovered = + "The cloud service did not return the requested properties %s " + + "in its answer. They are not covered by the keys in use, so " + + "they will be missing from every result. The other requested " + + "properties came back and are usable."; + + public static final String RequestedNamesNotProperties = + "The cloud service did not return the requested properties %s " + + "in its answer. Every property is answered, with a reason " + + "when it has no value, so these names are not properties. " + + "Check the spelling of each one. The other requested " + + "properties came back and are usable."; + public static final String EvidenceConflict = "'%s:%s' evidence conflicts with %s"; public static final String MessageNoDataInResponse = diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index 29c27b960..576df6df1 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -25,6 +25,7 @@ import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.cloudrequestengine.Constants.Messages; import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.data.FlowData; @@ -37,7 +38,11 @@ import fiftyone.pipeline.engines.services.HttpClientDefault; import org.slf4j.ILoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.TreeSet; import java.util.function.Function; /** @@ -55,6 +60,7 @@ public class CloudRequestEngineBuilder extends private String evidenceKeysEndpoint = null; private String resourceKey = null; private String licenseKey = null; + private final List requestedProperties = new ArrayList<>(); private String cloudRequestOrigin = null; private int timeoutMillis = Constants.DEFAULT_TIMEOUT_MILLIS; private CloudDiscoveryState discoveryState = null; @@ -78,12 +84,10 @@ public CloudRequestEngineBuilder( @Override protected CloudRequestEngine newEngine(List properties) throws Exception { - if(resourceKey == null || resourceKey.isEmpty()){ - throw new PipelineConfigurationException("A resource key is " + - "required to access the cloud server. Please use the " + - "'setResourceKey(String) method to supply your resource " + - "key obtained from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-flowelements-cloudrequestenginebuilder.java&utm_term=resource-key-required"); - } + String trimmedResourceKey = trimmed(resourceKey); + String trimmedLicenseKey = trimmed(licenseKey); + List requested = normalized(requestedProperties); + checkCredentials(trimmedResourceKey, trimmedLicenseKey, requested); /* * Check if endPoint has been explicitly set via setEndpoint. @@ -106,12 +110,13 @@ protected CloudRequestEngine newEngine(List properties) throws Exception new CloudRequestDataFactory(loggerFactory), httpClient, dataEndpoint, - resourceKey, - licenseKey, + trimmedResourceKey, + trimmedLicenseKey, propertiesEndpoint, evidenceKeysEndpoint, - timeoutMillis, - cloudRequestOrigin); + timeoutMillis, + cloudRequestOrigin, + requested); if (discoveryState != null) { engine.useDiscoveryState(discoveryState); } @@ -150,6 +155,51 @@ public CloudDiscoveryState getDiscoveryState() { CloudRequestEngineDefault engine = built; return engine == null ? null : engine.getDiscoveryState(); } + + private static String trimmed(String value) { + return value == null ? "" : value.trim(); + } + + /** + * The property names to send as values, trimmed and sent as given, + * with blank names and repeats left out. A repeat differs only in + * letter case, as the cloud service reads names without regard to + * case, and the first spelling given is kept. + */ + private static List normalized(List names) { + List result = new ArrayList<>(); + Set seen = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String name : names) { + String normal = trimmed(name); + if (normal.isEmpty() == false && seen.add(normal)) { + result.add(normal); + } + } + return result; + } + + /** + * Refuse the two combinations of keys and requested properties that the + * cloud service refuses on every request, so that a deployment sees one + * configuration error when the engine is built. A blank value counts as + * absent. + */ + private static void checkCredentials( + String resourceKey, + String licenseKey, + List requestedProperties) { + boolean hasResourceKey = resourceKey.isEmpty() == false; + boolean hasLicenseKey = licenseKey.isEmpty() == false; + + if (hasResourceKey == false && hasLicenseKey == false) { + throw new PipelineConfigurationException( + Messages.NoCredential); + } + if (hasResourceKey == false && requestedProperties.isEmpty()) { + throw new PipelineConfigurationException( + Messages.LicenseKeyNeedsProperties); + } + } private String getEnvironmentVariable(String name) { return System.getenv(name); @@ -173,7 +223,8 @@ public CloudRequestEngineBuilder setEndpoint(String uri) { uri += '/'; } endPoint = uri; - return setDataEndpoint(uri + (resourceKey != null ? resourceKey + "." : "") + "json") + String key = trimmed(resourceKey); + return setDataEndpoint(uri + (key.isEmpty() ? "" : key + ".") + "json") .setPropertiesEndpoint(uri + "accessibleproperties") .setEvidenceKeysEndpoint(uri + "evidencekeys"); @@ -182,7 +233,8 @@ public CloudRequestEngineBuilder setEndpoint(String uri) { /** * The endpoint the CloudRequestEngine will query to get a processing result. *

      - * By default, this is the endpoint value suffixed with the resourcekey and ".json" + * By default, this is the endpoint value suffixed with the resourcekey and + * ".json", or with "json" alone when no resource key is set * @param uri data endpoint * @return this builder */ @@ -223,11 +275,11 @@ public CloudRequestEngineBuilder setEvidenceKeysEndpoint(String uri) { /** * The resource key to query the endpoint with. *

      - * No default, a value must be supplied + * No default. Either a resource key or a license key must be supplied. * @param resourceKey resource key * @return this builder */ - @DefaultValue("No default - a resource key must be supplied") + @DefaultValue("No default - a resource key or a license key must be supplied") public CloudRequestEngineBuilder setResourceKey(String resourceKey) { this.resourceKey = resourceKey; return this; @@ -236,6 +288,11 @@ public CloudRequestEngineBuilder setResourceKey(String resourceKey) { /** * The license key to query the endpoint with. *

      + * A license key used without a resource key needs the properties + * wanted, named with {@link #setRequestedProperties(List)}. Beside a + * resource key it adds the requested properties the license grants to + * what the resource key carries. + *

      * There is no default * @param licenseKey license key * @return this builder @@ -246,6 +303,47 @@ public CloudRequestEngineBuilder setLicenseKey(String licenseKey) { return this; } + /** + * The properties each request asks the cloud service for, as fully + * qualified names such as {@code device.ismobile}, sent as values. + * Required with a license key alone. Beside a resource key the cloud + * service adds them to the properties the resource key carries. Names + * are trimmed and sent as given, and a name given again in another + * letter case is sent once. The properties set with + * {@link #setProperty(String)} and {@link #setProperties(List)} are not + * sent. + *

      + * By default no properties are requested + * @param properties the fully qualified property names + * @return this builder + */ + @DefaultValue("None - required with a license key alone") + public CloudRequestEngineBuilder setRequestedProperties( + List properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + requestedProperties.addAll(properties); + return this; + } + + /** + * As {@link #setRequestedProperties(List)}, taking the names as one + * comma separated string, as a configuration file gives them. + * @param properties comma separated fully qualified property names + * @return this builder + */ + @DefaultValue("None - required with a license key alone") + public CloudRequestEngineBuilder setRequestedProperties( + String properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + return setRequestedProperties(Arrays.asList(properties.split(","))); + } + /** * Timeout in seconds for each request to the cloud service, the two * discovery requests when the engine is built and each data request. diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 9ba26c7ee..b95d7ec5d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -53,6 +53,7 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.regex.Pattern; @@ -97,6 +98,9 @@ public class CloudRequestEngineDefault private String evidenceKeysEndpoint; private String cloudRequestOrigin; private Integer timeoutMillis; + private final List requestedProperties; + private final AtomicBoolean requestedPropertiesChecked = + new AtomicBoolean(false); /** * The lowest HTTP status of a client error. A discovery request answered @@ -188,7 +192,6 @@ public CloudRequestEngineDefault( cloudRequestOrigin); } - @SuppressWarnings("this-escape") public CloudRequestEngineDefault( Logger logger, ElementDataFactory aspectDataFactory, @@ -200,7 +203,42 @@ public CloudRequestEngineDefault( String evidenceKeysEndpoint, int timeoutMillis, String cloudRequestOrigin) { + this( + logger, + aspectDataFactory, + httpClient, + endPoint, + resourceKey, + licenseKey, + propertiesEndpoint, + evidenceKeysEndpoint, + timeoutMillis, + cloudRequestOrigin, + Collections.emptyList()); + } + + /** + * Constructor. + * @param requestedProperties the fully qualified names of the + * properties to ask for, sent as {@code values} + * when the list is not empty + */ + @SuppressWarnings("this-escape") + public CloudRequestEngineDefault( + Logger logger, + ElementDataFactory aspectDataFactory, + HttpClient httpClient, + String endPoint, + String resourceKey, + String licenseKey, + String propertiesEndpoint, + String evidenceKeysEndpoint, + int timeoutMillis, + String cloudRequestOrigin, + List requestedProperties) { super(logger, aspectDataFactory); + this.requestedProperties = Collections.unmodifiableList( + new ArrayList<>(requestedProperties)); this.endPoint = endPoint; this.resourceKey = resourceKey; @@ -438,6 +476,61 @@ protected void processEngine(FlowData data, CloudRequestData aspectData) throws ((CloudRequestDataInternal)aspectData).setJsonResponse(response); validateResponse(response, connection); + reportPropertiesNotCovered(response); + } + + /** + * Compare the properties asked for with the ones the answer carries, and + * warn once about any left out. The service leaves out a requested + * property it does not answer without naming it, and lower-cases the + * names it answers with. A property that is present but null counts as + * answered, as it carries a null reason beside it. The request is never + * failed, because the properties that did come back are usable. With no + * license key, a property the resource key does not carry is answered + * as null, so a name left out is not a property, and the message says + * so. + */ + private void reportPropertiesNotCovered(String response) { + if (requestedProperties.isEmpty() || + requestedPropertiesChecked.compareAndSet(false, true) == false) { + return; + } + JSONObject answer; + try { + answer = new JSONObject(response); + } catch (JSONException e) { + return; + } + List notCovered = new ArrayList<>(); + for (String name : requestedProperties) { + String[] parts = name.split( + Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); + if (parts.length < 2 || hasKeyIgnoringCase( + answer.optJSONObject(keyIgnoringCase(answer, parts[0])), + parts[1]) == false) { + notCovered.add(name); + } + } + if (notCovered.isEmpty() == false) { + logger.warn(String.format( + hasLicenseKey() ? PropertiesNotCovered : + RequestedNamesNotProperties, + stringJoin(notCovered, ", "))); + } + } + + private static String keyIgnoringCase(JSONObject object, String name) { + for (String key : object.keySet()) { + if (key.equalsIgnoreCase(name)) { + return key; + } + } + return name; + } + + private static boolean hasKeyIgnoringCase(JSONObject object, String name) { + return object != null && + object.has(keyIgnoringCase(object, name)); } /** @@ -457,9 +550,10 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { Map formData = getFormData(data); List formItems = new ArrayList<>(); - formItems.add("resource=" + resourceKey); - if(licenseKey != null && licenseKey.isEmpty() == false){ - formItems.add("license=" + licenseKey); + formItems.addAll(credentialItems()); + if (requestedProperties.isEmpty() == false) { + formItems.add("values=" + URLEncoder.encode( + stringJoin(requestedProperties, ","), "UTF-8")); } List formKeys = Arrays.asList(formData.keySet().toArray(new String[0])); @@ -474,6 +568,26 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { return string.getBytes(StandardCharsets.UTF_8); } + private boolean hasResourceKey() { + return resourceKey != null && resourceKey.isEmpty() == false; + } + + private boolean hasLicenseKey() { + return licenseKey != null && licenseKey.isEmpty() == false; + } + + /** The resource and license keys that are set, as form items. */ + private List credentialItems() throws UnsupportedEncodingException { + List items = new ArrayList<>(); + if (hasResourceKey()) { + items.add("resource=" + URLEncoder.encode(resourceKey, "UTF-8")); + } + if (hasLicenseKey()) { + items.add("license=" + URLEncoder.encode(licenseKey, "UTF-8")); + } + return items; + } + /** * The Evidence to send, keyed by the name the cloud service expects. * Only Evidence that {@link #isSent(String)} allows is included. Keys @@ -740,10 +854,26 @@ private void fetchPublicProperties() throws CloudRequestException, AggregateExce Map headers = new HashMap<>(); setCommonHeaders(headers); - HttpURLConnection connection = httpClient.connect(toUrl(propertiesEndpoint.trim() + (resourceKey != null ? "?Resource=" + resourceKey : ""))); - setTimeouts(connection); - jsonResult = httpClient.getResponseString(connection, headers); - validateResponse(jsonResult, connection); + if (hasLicenseKey()) { + // The credentials go in the body, not the URL, so a license + // key stays out of request logs. + byte[] content = stringJoin(credentialItems(), "&") + .getBytes(StandardCharsets.UTF_8); + headers.put( + "Content-Type", "application/x-www-form-urlencoded"); + headers.put("Content-Length", Integer.toString(content.length)); + HttpURLConnection connection = + httpClient.connect(toUrl(propertiesEndpoint.trim())); + setTimeouts(connection); + jsonResult = httpClient.postData(connection, headers, content); + validateResponse(jsonResult, connection); + } else { + HttpURLConnection connection = httpClient.connect(toUrl( + propertiesEndpoint.trim() + "?Resource=" + resourceKey)); + setTimeouts(connection); + jsonResult = httpClient.getResponseString(connection, headers); + validateResponse(jsonResult, connection); + } if (jsonResult.isEmpty() == false) { setPublicProperties(jsonResult); diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java new file mode 100644 index 000000000..e05e9bc09 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java @@ -0,0 +1,640 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +/** + * The credentials a Cloud Request Engine accepts, what it sends for each + * and that a license key stays out of URLs, logs and messages. + */ +public class CloudRequestEngineLicenseKeyTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final List PROPERTY_LIST = + Arrays.asList("device.ismobile", "device.iscrawler"); + + /** A request the engine made. */ + private static class Call { + final String method; + final URL url; + final String body; + + Call(String method, URL url, String body) { + this.method = method; + this.url = url; + this.body = body; + } + } + + /** Records every request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + // Discovery makes its two requests at once, so connections and + // calls are recorded from two threads. + final List calls = + Collections.synchronizedList(new ArrayList<>()); + String dataResponse = "{\"device\":{\"ismobile\":true," + + "\"iscrawler\":false}}"; + int dataStatus = 200; + String propertiesResponse = + "{\"Products\":{\"device\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"IsMobile\",\"Type\":\"Boolean\"," + + "\"Category\":\"Device\"}]}}}"; + String evidenceKeysResponse = "[\"query.User-Agent\"]"; + private final Map urls = + new ConcurrentHashMap<>(); + + @Override + public HttpURLConnection connect(URL url) { + HttpURLConnection connection = mock(HttpURLConnection.class); + try { + when(connection.getURL()).thenReturn(url); + when(connection.getResponseCode()).thenReturn( + url.getPath().endsWith("json") ? dataStatus : 200); + } catch (IOException e) { + throw new IllegalStateException(e); + } + when(connection.getHeaderFields()).thenReturn( + Collections.emptyMap()); + urls.put(connection, url); + return connection; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + URL url = urls.get(connection); + calls.add(new Call( + "POST", + url, + data == null ? "" : + new String(data, StandardCharsets.UTF_8))); + return answer(url); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return getResponseString(connection, null); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + URL url = urls.get(connection); + calls.add(new Call("GET", url, "")); + return answer(url); + } + + private String answer(URL url) { + if (url.getPath().endsWith("accessibleproperties")) { + return propertiesResponse; + } + if (url.getPath().endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return dataResponse; + } + + List callsTo(String pathEnd) { + List found = new ArrayList<>(); + for (Call call : calls) { + if (call.url.getPath().endsWith(pathEnd)) { + found.add(call); + } + } + return found; + } + } + + private FakeCloud cloud; + private Logger logger; + private ILoggerFactory loggerFactory; + + @BeforeEach + public void setUp() { + cloud = new FakeCloud(); + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + private CloudRequestEngineBuilder builder() { + return new CloudRequestEngineBuilder(loggerFactory, cloud); + } + + /** + * Build with the test endpoint set last, as the default endpoint is when + * no endpoint is set, so the data endpoint's route reflects the keys. + */ + private CloudRequestEngine build(CloudRequestEngineBuilder builder) + throws Exception { + return builder.setEndpoint(BASE).build(); + } + + private String process(CloudRequestEngine engine) throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + StringBuilder errors = new StringBuilder(); + for (FlowError error : data.getErrors()) { + for (Throwable t = error.getThrowable(); t != null; + t = t.getCause()) { + errors.append(t).append('\n'); + } + } + return errors.toString(); + } + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + private Map dataRequest(String pathEnd) + throws UnsupportedEncodingException { + List posts = cloud.callsTo(pathEnd); + assertEquals(1, posts.size()); + assertEquals("POST", posts.get(0).method); + return form(posts.get(0).body); + } + + private void assertRefused( + CloudRequestEngineBuilder builder, + String... expectedInMessage) { + int sent = cloud.calls.size(); + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> build(builder)); + for (String expected : expectedInMessage) { + assertTrue( + e.getMessage().contains(expected), + "Message should contain '" + expected + "' but was: " + + e.getMessage()); + } + assertEquals(sent, cloud.calls.size(), "A refused build sends nothing."); + } + + // The rows of the specification's table of accepted combinations. + + @Test + public void Build_ResourceKeyOnly_Accepted() throws Exception { + assertNotNull(build(builder().setResourceKey(RESOURCE_KEY))); + } + + @Test + public void Build_ResourceKeyAndLicenseKey_Accepted() throws Exception { + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY))); + } + + @Test + public void Build_AllThree_Accepted() throws Exception { + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST))); + } + + @Test + public void Build_LicenseKeyAndProperties_Accepted() throws Exception { + assertNotNull(build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST))); + } + + @Test + public void Build_LicenseKeyWithoutProperties_Refused() { + assertRefused( + builder().setLicenseKey(LICENSE_KEY), + "answers 400", + "setRequestedProperties"); + } + + @Test + public void Build_ResourceKeyWithProperties_Accepted() throws Exception { + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties("device.hardwarevendor"))); + } + + @Test + public void Build_NoCredential_Refused() { + assertRefused(builder(), "401", "resource key"); + assertRefused(builder().setRequestedProperties(PROPERTY_LIST), "401"); + } + + @Test + public void Build_BlankValuesCountAsAbsent() throws Exception { + assertRefused( + builder().setResourceKey(" ").setLicenseKey("\n"), + "401"); + assertRefused( + builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(" ", "")), + "answers 400"); + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(" ") + .setRequestedProperties(" "))); + } + + @Test + public void Build_RefusalsDoNotContainTheLicenseKey() { + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> build(builder().setLicenseKey(LICENSE_KEY))); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + + // Requests. + + @Test + public void Process_LicenseKeyAlone_SendsLicenseAndValues() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY + "\n") + .setRequestedProperties( + Arrays.asList(" Device.IsMobile", "device.iscrawler "))); + + assertEquals("", process(engine)); + + Map fields = dataRequest("/api/v4/json"); + assertEquals(LICENSE_KEY, fields.get("license")); + assertEquals("Device.IsMobile,device.iscrawler", fields.get("values")); + assertFalse(fields.containsKey("resource")); + assertFalse(fields.containsKey("licence")); + assertEquals("iPhone", fields.get("User-Agent")); + } + + @Test + public void Process_ResourceKey_SendsResourceOnly() throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY)); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertFalse(fields.containsKey("license")); + assertFalse(fields.containsKey("values")); + } + + @Test + public void Process_ResourceKeyAndLicenseKey_SendsBoth() + throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY)); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals(LICENSE_KEY, fields.get("license")); + assertFalse(fields.containsKey("values")); + } + + @Test + public void Process_ResourceKeyAndProperties_SendsResourceAndValues() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true," + + "\"hardwarevendor\":null," + + "\"hardwarevendornullreason\":\"Paid feature\"}}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties("Device.HardwareVendor")); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals("Device.HardwareVendor", fields.get("values")); + assertFalse(fields.containsKey("license")); + verify(logger, never()).warn(anyString()); + } + + @Test + public void Process_AllThree_SendsResourceLicenseAndValues() + throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties("IP.RegisteredName")); + cloud.dataResponse = "{\"device\":{\"ismobile\":true}," + + "\"ip\":{\"registeredname\":\"Example\"}}"; + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals(LICENSE_KEY, fields.get("license")); + assertEquals("IP.RegisteredName", fields.get("values")); + for (Call call : cloud.calls) { + assertFalse(call.url.toString().contains(LICENSE_KEY)); + } + } + + @Test + public void Process_RequestedPropertiesAsText_Sent() throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties("Device.IsMobile, device.iscrawler ,")); + + assertEquals("", process(engine)); + + Map fields = dataRequest("/api/v4/json"); + assertEquals("Device.IsMobile,device.iscrawler", fields.get("values")); + } + + /** + * Properties set with setProperty or setProperties are not sent, as + * before, so a configuration that lists them keeps the answer it had. A + * license key with only those properties is refused. + */ + @Test + public void Process_SetProperties_NotSent() throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setProperties(Arrays.asList("IsMobile", "DeviceId"))); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertFalse(fields.containsKey("values")); + verify(logger, never()).warn(anyString()); + + assertRefused( + builder() + .setLicenseKey(LICENSE_KEY) + .setProperty("device.ismobile"), + "answers 400"); + } + + // Discovery. + + @Test + public void Discovery_LicenseKeyAlone_PostsCredentialsInTheBody() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST)); + + Map products = + engine.getPublicProperties(); + assertTrue(products.containsKey("device")); + assertNotNull(engine.getEvidenceKeyFilter()); + + List properties = cloud.callsTo("accessibleproperties"); + assertEquals(1, properties.size()); + assertEquals("POST", properties.get(0).method); + assertNull(properties.get(0).url.getQuery()); + assertFalse(properties.get(0).url.toString().contains(LICENSE_KEY)); + Map fields = form(properties.get(0).body); + assertEquals(LICENSE_KEY, fields.get("license")); + assertFalse(fields.containsKey("resource")); + + List evidence = cloud.callsTo("evidencekeys"); + assertEquals(1, evidence.size()); + assertEquals("GET", evidence.get(0).method); + assertNull(evidence.get(0).url.getQuery()); + } + + @Test + public void Discovery_ResourceKeyAndProperties_GetsWithTheKeyInTheQuery() + throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties("device.hardwarevendor")); + + engine.getPublicProperties(); + + List properties = cloud.callsTo("accessibleproperties"); + assertEquals(1, properties.size()); + assertEquals("GET", properties.get(0).method); + assertEquals( + "Resource=" + RESOURCE_KEY, + properties.get(0).url.getQuery()); + } + + @Test + public void Discovery_BothKeys_PostsBothInTheBody() throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY)); + + engine.getPublicProperties(); + + Call call = cloud.callsTo("accessibleproperties").get(0); + assertEquals("POST", call.method); + assertNull(call.url.getQuery()); + Map fields = form(call.body); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals(LICENSE_KEY, fields.get("license")); + } + + // Keys in URLs, logs and messages. + + @Test + public void Process_FailedRequest_LicenseKeyInNoUrlLogOrMessage() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList( + "device.ismobile", "device.other"))); + assertEquals("", process(engine)); + + cloud.dataStatus = 500; + cloud.dataResponse = "{\"status\":\"failure\"}"; + String errors = process(engine); + + assertTrue(errors.contains("500"), errors); + assertFalse(errors.contains(LICENSE_KEY), errors); + for (Call call : cloud.calls) { + assertFalse(call.url.toString().contains(LICENSE_KEY)); + } + assertFalse(mockingDetails(logger).getInvocations().isEmpty()); + mockingDetails(logger).getInvocations().forEach(i -> + assertFalse( + Arrays.deepToString(i.getArguments()).contains(LICENSE_KEY), + i.toString())); + } + + // Properties the service did not return. + + @Test + public void Process_PropertyDropped_ReportedOnceAndRequestSucceeds() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties( + Arrays.asList("Device.IsMobile", "Device.IsCrawler"))); + + assertEquals("", process(engine)); + assertEquals("", process(engine)); + + verify(logger, times(1)).warn(argThat( + (String m) -> m.contains("Device.IsCrawler") && + m.contains("not covered by the keys") && + !m.contains("Device.IsMobile"))); + verify(logger, times(1)).warn(anyString()); + } + + /** + * With a resource key, requested properties and no license key, a + * property no licence grants is answered as null, so a name the + * answer leaves out is not a property. The report says so, and names + * no license key, because none was given. + */ + @Test + public void Process_ResourceKeyNameNotAnswered_ReportedAsNotAProperty() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties( + Arrays.asList("Device.IsMobile", "device.notaproperty"))); + + assertEquals("", process(engine)); + assertEquals("", process(engine)); + + verify(logger, times(1)).warn(argThat( + (String m) -> m.contains("device.notaproperty") && + m.contains("not properties") && + !m.contains("license") && + !m.contains("not covered"))); + verify(logger, times(1)).warn(anyString()); + } + + /** + * A name given twice in different letter cases is sent once, as it + * was first given, because the cloud service reads property names + * without regard to case. + */ + @Test + public void Process_NameGivenTwice_SentOnceAsFirstGiven() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList( + "Device.IsMobile", "device.ismobile", "device.iscrawler"))); + + assertEquals("", process(engine)); + + assertEquals("Device.IsMobile,device.iscrawler", + dataRequest("/api/v4/json").get("values")); + } + + /** + * A null list of requested properties is refused with the name of + * the parameter, as .NET refuses it, rather than failing inside the + * list. + */ + @Test + public void SetRequestedProperties_Null_Refused() { + IllegalArgumentException list = assertThrows( + IllegalArgumentException.class, + () -> builder().setRequestedProperties((List) null)); + assertTrue(list.getMessage().contains("properties")); + IllegalArgumentException text = assertThrows( + IllegalArgumentException.class, + () -> builder().setRequestedProperties((String) null)); + assertTrue(text.getMessage().contains("properties")); + } + + @Test + public void Process_PropertyPresentButNull_CountsAsAnswered() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true," + + "\"iscrawler\":null,\"iscrawlernullreason\":\"No data\"}}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST)); + + assertEquals("", process(engine)); + + verify(logger, never()).warn(anyString()); + } + + @Test + public void Process_ResourceKey_NothingToCompare() throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY)); + + assertEquals("", process(engine)); + + verify(logger, never()).warn(anyString()); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 3a64e5f5a..f7b274895 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -680,7 +680,10 @@ public void process_PostsWholeEvidenceName() throws Exception { } ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); - verify(httpClient, times(1)).postData(any(), any(), body.capture()); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + body.capture()); List items = Arrays.asList( new String(body.getValue(), StandardCharsets.UTF_8).split("&")); assertTrue(items.contains("id.usage=non-marketing"), @@ -906,7 +909,10 @@ public void process_RepeatedRequestHitsTheCache() throws Exception { engine.getEvidenceKeyFilter().include("header.sec-ch-ua-platform"), "The engine advertises a key the cloud service does not list."); ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); - verify(httpClient, times(1)).postData(any(), any(), body.capture()); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + body.capture()); List items = Arrays.asList( new String(body.getValue(), StandardCharsets.UTF_8).split("&")); assertTrue(items.contains("sec-ch-ua-platform=Windows"), @@ -948,7 +954,10 @@ private List postedItems( } ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); - verify(httpClient, times(1)).postData(any(), any(), body.capture()); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + body.capture()); return Arrays.asList( new String(body.getValue(), StandardCharsets.UTF_8).split("&")); } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java index 98d854117..dfbd106b4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java @@ -122,7 +122,21 @@ else if (url.getPath().endsWith("evidencekeys")) { .when(httpClient) .getResponseString(any(HttpURLConnection.class), anyMap()); - doReturn(jsonResponse) + // A license key makes the discovery requests POSTs, so a POST is + // answered by its URL in the same way. + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + URL url = ((HttpURLConnection)invocationOnMock.getArgument(0)).getURL(); + if (url.getPath().endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (url.getPath().endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return jsonResponse; + } + }) .when(httpClient) .postData( any(HttpURLConnection.class), From 6034fa4f2588a77b303c552d9eb6d47585dc793f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 23:50:19 +0000 Subject: [PATCH 86/95] Take the session id and the sequence from the Sequence Element's data (#102) * Test the web integration switches, the Sequence Element data and usage sharing characters The tests cover the init parameters that set the JavaScript and JSON paths and switch the Sequence Element and the Set Headers Element, the Element Data the Sequence Element writes, how the JSON Builder Element shows that data, and the characters XML 1.0 allows in a shared Evidence value. * Add switches and paths to the web integration, and Element Data to the Sequence Element The filter takes init parameters for the JavaScript path, the JSON path, and whether the Sequence Element and the Set Headers Element are added. The defaults keep today's paths and today's behaviour. The Sequence Element writes session-id and sequence as Element Data as well as Evidence. Usage sharing keeps every character XML 1.0 allows instead of replacing everything above U+00FF. * Test the MVC integration's paths and element switches * Give the MVC integration the filter's paths and element switches, and say what the Lite file lacks The MVC configuration takes the JavaScript path, the JSON path and the two element switches the filter takes, with defaults that keep today's behaviour. The example comments about the Lite data file say it has a small number of Properties, as the specification does. * Test the Sequence Element data under the key sequence * Key the Sequence Element data as sequence, as the other languages do * Do not call an overridable method from the constructor Java 21 warns about it, and the build treats warnings as errors. * Test that the JSON is served from /51dpipeline/json and from the old path * Serve the JSON from /51dpipeline/json by default, and from the old path as well The specification's JSON path becomes the default in the filter and the MVC integration, and the JavaScript they serve calls it. The path 51Degrees.core.json is still answered, and json-path still sets the path. * Test the Sequence Element with session ids and sequences of every kind * Treat the session id and the sequence independently, as the specification says A session id that is not a non-empty string, and a sequence that is not a whole number or text of decimal digits that fits in an int with room to add one, count as absent. The Evidence and the Element Data are written from the same values. * Test that the usage sharing filter never includes the web request * Keep the web request out of the usage sharing filter, as .NET does * Expect a new session id when the Sequence Element is given an empty one * Test that a build with the JSON Builder Element and no Sequence Element is refused The json-path test uses a path that is not the default. * Refuse a build with the JSON Builder Element and no Sequence Element The JSON Builder Element fails on every request without the sequence number, so the build says which option to change. * Use a configuration without the JSON Builder Element when client-side properties are off * Test the exact path match and the rule that a sequence is not negative - FiftyOneJSServiceCoreTests checks that a configured path, with or without its leading slash, answers only that path inside the application, so /anything/json, /json/more and /notjson are not answered, and that the context path is not part of the path. - SequenceElementTests checks that a negative sequence of any whole number type, a sign and white space each count as no sequence, and that 0 gives 1. * Match the JSON and JavaScript paths exactly, and count a negative sequence as absent FiftyOneJSServiceCore.Default answered any request whose whole URL ended with the configured path, so a configured path of json also answered /anything/json. The path is now compared with the whole path of the request inside the application, after the context path, ignoring case and with or without a leading slash, and a request with no URI is not answered. The old 51Degrees.core.json path is matched the same way. The Sequence Element took a negative whole number as a sequence. The Sequence Element specification counts a sequence only from 0 up to 2,147,483,646, so a negative one now counts as absent. * Give a caller with no JSON path the default path * Take the session id and the sequence from the Sequence Element's data * Check the Flow Data's keys before reading the Sequence Element's data * Name the Sequence Element's data key DEFAULT_ELEMENT_DATA_KEY The constant is the key the element uses unless a subclass gives another, so its name says it is the default, as .NET's does. * Give the JavaScript builder tests' mocked Flow Data the Sequence Element's data The builder reads the session id and the sequence from the Sequence Element's data, so the shared mock holds that data as a web integration's pipeline does. JavaScriptBuilderElement_VerifySession failed in the full build without it. --- .../flowelements/SequenceElement.java | 9 +- .../JavaScriptBuilderElement.java | 92 ++++++++----- .../JavaScriptBuilderTests.java | 16 +++ .../SessionIdAndSequenceTests.java | 121 ++++++++++++++++-- 4 files changed, 197 insertions(+), 41 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java index 79a70b91e..588f1947c 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java @@ -146,9 +146,16 @@ private static Integer toSequence(Object sequence) { Integer.valueOf((int) value) : null; } + /** + * The key the Element Data is stored under in the Flow Data, unless a + * subclass gives another from {@link #getElementDataKey()}. + * @see Specification + */ + public static final String DEFAULT_ELEMENT_DATA_KEY = "sequence"; + @Override public String getElementDataKey() { - return "sequence"; + return DEFAULT_ELEMENT_DATA_KEY; } @Override diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 782c9fa7a..509fc1b0b 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -32,6 +32,8 @@ import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; import fiftyone.pipeline.javascriptbuilder.Constants; import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.templates.JavaScriptResource; @@ -243,8 +245,9 @@ protected void processInternal(FlowData data) throws Exception { String url = getUrl(reqProtocol, reqHost); - String sessionId = getSessionId(data); - int sequence = getSequence(data); + ElementData sequenceData = getSequenceData(data); + String sessionId = getSessionId(sequenceData); + int sequence = getSequence(sequenceData); String serializedParameters = serializeParameters(parameters); // With the gathered resources, build a new JavaScriptResource. @@ -473,52 +476,77 @@ private String getPath() { /** * Gets the session id to write into the script. This is the session id - * evidence, set by the Sequence Element or by the request, when it - * matches SESSION_ID_PATTERN, and an empty string otherwise. - * @param data the flow data + * the Sequence Element wrote, when it matches SESSION_ID_PATTERN, and an + * empty string otherwise, including when the pipeline has no Sequence + * Element. + * @param sequenceData the Sequence Element's data, or null * @return the session id, or an empty string */ - private String getSessionId(FlowData data) { - TryGetResult trySessionId = - data.tryGetEvidence(EVIDENCE_SESSIONID, Object.class); - if (trySessionId.hasValue() && - trySessionId.getValue() instanceof String) { - String sessionId = (String) trySessionId.getValue(); - if (SESSION_ID_PATTERN.matcher(sessionId).matches()) { - return sessionId; - } + private String getSessionId(ElementData sequenceData) { + Object value = getSequenceValue( + sequenceData, SequenceData.SESSION_ID_PROPERTY_NAME); + if (value instanceof String && + SESSION_ID_PATTERN.matcher((String) value).matches()) { + return (String) value; } return ""; } /** - * Gets the sequence to write into the script. This is the sequence - * evidence, as a number or as text, when it is a positive 32 bit - * integer, and 1 otherwise, including when there is no Sequence Element. - * The value is written into the script as code, so text that is not a - * number could break the script, and a number below 1 is not a sequence - * the Sequence Element would ever give. - * @param data the flow data + * Gets the sequence to write into the script. This is the sequence the + * Sequence Element wrote, as a number or as text, when it is a positive + * 32 bit integer, and 1 otherwise, including when the pipeline has no + * Sequence Element. The value is written into the script as code, so + * text that is not a number could break the script, and a number below 1 + * is not a sequence the Sequence Element would ever give. + * @param sequenceData the Sequence Element's data, or null * @return the sequence, which is always at least 1 */ - private int getSequence(FlowData data) { - TryGetResult trySequence = - data.tryGetEvidence(EVIDENCE_SEQUENCE, Object.class); + private int getSequence(ElementData sequenceData) { + Object value = getSequenceValue( + sequenceData, SequenceData.SEQUENCE_PROPERTY_NAME); long sequence = 0; - if (trySequence.hasValue()) { - Object value = trySequence.getValue(); - if (value instanceof Integer) { - sequence = (Integer) value; - } else if (value instanceof String && - SEQUENCE_PATTERN.matcher((String) value).matches()) { - sequence = Long.parseLong((String) value); - } + if (value instanceof Integer) { + sequence = (Integer) value; + } else if (value instanceof String && + SEQUENCE_PATTERN.matcher((String) value).matches()) { + sequence = Long.parseLong((String) value); } return sequence >= 1 && sequence <= Integer.MAX_VALUE ? (int) sequence : 1; } + /** + * The Sequence Element's data, or null when the pipeline has no Sequence + * Element. Step 5 of the specification takes the session id and the + * sequence from that data and not from Evidence, so a pipeline with no + * Sequence Element renders the defaults even when the request carried a + * session id or a sequence of its own. + * @param data the flow data + * @return the Sequence Element's data, or null + * @see Specification + */ + private static ElementData getSequenceData(FlowData data) { + // get(String) throws when the Flow Data holds nothing under the key, + // so the keys it does hold are checked first. + return data.getDataKeys().contains(SequenceElement.DEFAULT_ELEMENT_DATA_KEY) + ? data.get(SequenceElement.DEFAULT_ELEMENT_DATA_KEY) + : null; + } + + /** + * One value from the Sequence Element's data. + * @param sequenceData the Sequence Element's data, or null + * @param name the name of the value within that data + * @return the value, or null + */ + private static Object getSequenceValue( + ElementData sequenceData, + String name) { + return sequenceData == null ? null : sequenceData.get(name); + } + private void buildJavaScript( FlowData data, String jsonObject, diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java index 5cfde9467..7ec0c95e1 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java @@ -30,6 +30,8 @@ import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; @@ -57,6 +59,7 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -527,6 +530,19 @@ private void configureMocks() { when(flowData.get(anyString())).thenReturn(elementDataMock); + // The Sequence Element's data, which is where the builder reads the + // session id and the sequence from, as a web integration's + // pipeline holds it. + ElementData sequenceData = mock(ElementData.class); + when(sequenceData.get(SequenceData.SESSION_ID_PROPERTY_NAME)) + .thenReturn(sessionId); + when(sequenceData.get(SequenceData.SEQUENCE_PROPERTY_NAME)) + .thenReturn(sequence); + when(flowData.getDataKeys()).thenReturn(Collections.singletonList( + SequenceElement.DEFAULT_ELEMENT_DATA_KEY)); + when(flowData.get(SequenceElement.DEFAULT_ELEMENT_DATA_KEY)) + .thenReturn(sequenceData); + when(flowData.getAs(anyString(), any(Class.class))).thenReturn(new AspectPropertyValueDefault<>("None")); doAnswer((Answer) invocation -> { diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java index 8c74887c4..06bc9b6b6 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java @@ -30,6 +30,8 @@ import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; @@ -66,6 +68,11 @@ * the builder's own handling can be tested, the tests "without the Sequence * Element" use a stand in for the JSON builder that supplies the payload and * leaves the evidence as it was given. + *

      + * The builder takes both values from the Sequence Element's data, so the + * tests of the rule above put the value straight into that data with a + * stand in for the Sequence Element, which lets a value the real element + * would never write be tried as well. */ public class SessionIdAndSequenceTests { @@ -122,9 +129,65 @@ protected void unmanagedResourcesCleanup() { } } + /** + * Writes the two values given straight into the Sequence Element's data, + * in place of that element, so that a value the real element would never + * write can be tried as well. A null value is not written. + */ + private static class SequenceStandIn extends + FlowElementBase { + + private final Object sessionId; + private final Object sequence; + + SequenceStandIn(Logger logger, Object sessionId, Object sequence) { + super(logger, null); + this.sessionId = sessionId; + this.sequence = sequence; + } + + @Override + protected void processInternal(FlowData data) { + SequenceData sequenceData = new SequenceData(logger, data); + if (sessionId != null) { + sequenceData.put( + SequenceData.SESSION_ID_PROPERTY_NAME, sessionId); + } + if (sequence != null) { + sequenceData.put( + SequenceData.SEQUENCE_PROPERTY_NAME, sequence); + } + data.getOrAdd(getElementDataKey(), f -> sequenceData); + } + + @Override + public String getElementDataKey() { + return SequenceElement.DEFAULT_ELEMENT_DATA_KEY; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + @Test public void SessionId_Valid_Rendered() throws Exception { - String script = render(false, "abc-123", null); + String script = renderWithSequenceData("abc-123", null); assertEquals("abc-123", sessionId(script)); assertEquals("1", sequence(script)); } @@ -132,7 +195,7 @@ public void SessionId_Valid_Rendered() throws Exception { @Test public void SessionId_LongestValid_Rendered() throws Exception { String id = repeat('a', 64); - String script = render(false, id, null); + String script = renderWithSequenceData(id, null); assertEquals(id, sessionId(script)); } @@ -149,7 +212,7 @@ static Stream invalidSessionIds() { @ParameterizedTest @MethodSource("invalidSessionIds") public void SessionId_Invalid_RenderedEmpty(String id) throws Exception { - String script = render(false, id, null); + String script = renderWithSequenceData(id, null); assertEquals("", sessionId(script)); } @@ -190,14 +253,31 @@ public void NoSequenceElement_NoEvidence_DefaultsRendered() } /** - * A sequence from a web request is text. With no Sequence Element it - * reaches the builder as it is. + * With no Sequence Element the defaults are rendered even when the + * request carried a session id and a sequence of its own, because step 5 + * of the + * specification + * takes both from the Sequence Element's data and says "Where there is + * no Sequence Element, the sequence MUST be rendered as 1 and the + * session id as an empty string, so the script always parses." + */ + @Test + public void NoSequenceElement_WithEvidence_DefaultsRendered() + throws Exception { + String script = render(false, "abc-123", 7); + assertEquals("", sessionId(script)); + assertEquals("1", sequence(script)); + } + + /** + * A sequence another Sequence Element wrote as text is read as a number + * when it is one, because the Element Data holds values of any type. */ @ParameterizedTest @ValueSource(strings = {"abc", "-1", "0", "99999999999", "", "2147483648"}) public void Sequence_InvalidText_RenderedOne(String value) throws Exception { - String script = render(false, "abc-123", value); + String script = renderWithSequenceData("abc-123", value); assertEquals("1", sequence(script)); } @@ -209,7 +289,7 @@ static Stream invalidSequenceNumbers() { @MethodSource("invalidSequenceNumbers") public void Sequence_InvalidNumber_RenderedOne(Object value) throws Exception { - String script = render(false, "abc-123", value); + String script = renderWithSequenceData("abc-123", value); assertEquals("1", sequence(script)); } @@ -225,7 +305,7 @@ static Stream validSequences() { @MethodSource("validSequences") public void Sequence_Valid_Rendered(Object value, String expected) throws Exception { - String script = render(false, "abc-123", value); + String script = renderWithSequenceData("abc-123", value); assertEquals(expected, sequence(script)); } @@ -245,6 +325,31 @@ public void Sequence_InvalidWithSequenceElement_RenderedOne(int value) assertEquals("1", sequence(script)); } + /** + * Builds a pipeline whose Sequence Element data holds the two values + * given, processes it, checks the script parses and returns it. A null + * value is not written into that data. + */ + private String renderWithSequenceData(Object sessionId, Object sequence) + throws Exception { + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(loggerFactory).build(); + PipelineBuilder builder = new PipelineBuilder(loggerFactory); + builder.addFlowElement(new SequenceStandIn( + loggerFactory.getLogger("sequence"), sessionId, sequence)); + builder.addFlowElement( + new JsonStandIn(loggerFactory.getLogger("json"))); + builder.addFlowElement(jsElement); + String script; + try (Pipeline pipeline = builder.build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + script = flowData.getFromElement(jsElement).getJavaScript(); + } + assertParses(script); + return script; + } + /** * Builds a pipeline with the Sequence Element and the JSON builder, or * with the stand in for them, processes it From 2f6ab51c5b3304d00227ae6fdb178ec8daf323ab Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 00:10:05 +0000 Subject: [PATCH 87/95] Use the standard licence text in one LICENSE, and take the schema from configuration-validation (#109) * Use the standard licence text once, with its additional terms in NOTICE The root LICENSE and five copies in device-detection, examples and pipeline held the EUPL v1.2 text with its title in capitals, and location/LICENSE held the 51Degrees header form. The root LICENSE now holds the text byte for byte as 51degrees-cxx has it, NOTICE holds the two additional terms as 51degrees-cxx has them, and the six other copies are removed. Maven Central reads the licence from each pom, so no package needs a file of its own. pipelineOptions.xsd was a copy of the file in configuration-validation. pipeline.core now takes it from that repository as a submodule, as the .NET pipeline does, and adds it to the jar at the same resource path. * Keep the additional terms in the file headers rather than in NOTICE The two additional terms apply where each file's header states them, so the repository has LICENSE alone, without a NOTICE file. The README's licence line goes back to naming LICENSE only. META-INF/NOTICE.txt in pipeline.did is the OWID attribution and stays. --- .gitmodules | 3 + LICENSE | 313 +++++++++--------- device-detection/LICENSE | 287 ---------------- examples/device-detection/LICENSE | 287 ---------------- examples/ip-intelligence/LICENSE | 287 ---------------- ip-intelligence/LICENSE | 287 ---------------- location/LICENSE | 298 ----------------- pipeline/LICENSE | 287 ---------------- .../pipeline.core/configuration-validation | 1 + pipeline/pipeline.core/pom.xml | 13 + .../src/main/resources/pipelineOptions.xsd | 42 --- 11 files changed, 167 insertions(+), 1938 deletions(-) delete mode 100644 device-detection/LICENSE delete mode 100644 examples/device-detection/LICENSE delete mode 100644 examples/ip-intelligence/LICENSE delete mode 100644 ip-intelligence/LICENSE delete mode 100644 location/LICENSE delete mode 100644 pipeline/LICENSE create mode 160000 pipeline/pipeline.core/configuration-validation delete mode 100644 pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd diff --git a/.gitmodules b/.gitmodules index ed93e738d..f3fd5d50e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ path = pipeline/owid-java url = https://github.com/SWAN-community/owid-java.git branch = main +[submodule "pipeline/pipeline.core/configuration-validation"] + path = pipeline/pipeline.core/configuration-validation + url = ../configuration-validation diff --git a/LICENSE b/LICENSE index 4153cd377..a74f04438 100644 --- a/LICENSE +++ b/LICENSE @@ -1,53 +1,44 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 +European Union Public Licence +V. 1.2 -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). +EUPL © the European Union 2007, 2016 + +This European Union Public Licence (the ‘EUPL’) applies to the Work (as +defined below) which is provided under the terms of this Licence. Any use of +the Work, other than as authorised under this Licence is prohibited (to the +extent such use is covered by a right of the copyright holder of the Work). The Work is provided under the terms of this Licence when the Licensor (as defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. +copyright notice for the Work: “Licensed under the EUPL”, or has expressed by +any other means his willingness to license under the EUPL. 1. Definitions In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most +— ‘The Licence’: this Licence. +— ‘The Original Work’: the work or software distributed or communicated by the + ‘Licensor under this Licence, available as Source Code and also as + ‘Executable Code as the case may be. +— ‘Derivative Works’: the works or software that could be created by the + ‘Licensee, based upon the Original Work or modifications thereof. This + ‘Licence does not define the extent of modification or dependence on the + ‘Original Work required in order to classify a work as a Derivative Work; + ‘this extent is determined by copyright law applicable in the country + ‘mentioned in Article 15. +— ‘The Work’: the Original Work or its Derivative Works. +— ‘The Source Code’: the human-readable form of the Work which is the most convenient for people to study and modify. -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates +— ‘The Executable Code’: any code which has generally been compiled and which + is meant to be interpreted by a computer as a program. +— ‘The Licensor’: the natural or legal person that distributes or communicates the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of +— ‘Contributor(s)’: any natural or legal person who modifies the Work under + the Licence, or otherwise contributes to the creation of a Derivative Work. +— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, +— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, renting, distributing, communicating, transmitting, or otherwise making available, online or offline, copies of the Work or providing access to its essential functionalities at the disposal of any other natural or legal @@ -56,46 +47,46 @@ In this Licence, the following terms have the following meaning: 2. Scope of the rights granted by the Licence The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. +sublicensable licence to do the following, for the duration of copyright +vested in the Original Work: + +— use the Work in any circumstance and for all usage, +— reproduce the Work, +— modify the Work, and make Derivative Works based upon the Work, +— communicate to the public, including the right to make available or display + the Work or copies thereof to the public and perform publicly, as the case + may be, the Work, +— distribute the Work or copies thereof, +— lend and rent the Work or copies thereof, +— sublicense rights in the Work or copies thereof. Those rights can be exercised on any media, supports and formats, whether now known or later invented, as far as the applicable law permits so. In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. +exercise his moral right to the extent allowed by law in order to make +effective the licence of the economic rights here above listed. -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. +The Licensor grants to the Licensee royalty-free, non-exclusive usage rights +to any patents held by the Licensor, to the extent necessary to make use of +the rights granted on the Work under this Licence. 3. Communication of the Source Code The Licensor may provide the Work either in its Source Code form, or as Executable Code. If the Work is provided as Executable Code, the Licensor provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. +along with each copy of the Work that the Licensor distributes or indicates, +in a notice following the copyright notice attached to the Work, a repository +where the Source Code is easily and freely accessible for as long as the +Licensor continues to distribute or communicate the Work. 4. Limitations on copyright -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. +Nothing in this Licence is intended to deprive the Licensee of the benefits +from any exception or limitation to the exclusive rights of the rights owners +in the Work, of the exhaustion of those rights or of other applicable +limitations thereto. 5. Obligations of the Licensee @@ -104,36 +95,37 @@ obligations imposed on the Licensee. Those obligations are the following: Attribution right: The Licensee shall keep intact all copyright, patent or trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or +disclaimer of warranties. The Licensee must include a copy of such notices and +a copy of the Licence with every copy of the Work he/she distributes or communicates. The Licensee must cause any Derivative Work to carry prominent notices stating that the Work has been modified and the date of modification. Copyleft clause: If the Licensee distributes or communicates copies of the Original Works or Derivative Works, this Distribution or Communication will be done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. +unless the Original Work is expressly distributed only under this version of +the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee +(becoming Licensor) cannot offer or impose any additional terms or conditions +on the Work or Derivative Work that alter or restrict the terms of the +Licence. Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and +Works or copies thereof based upon both the Work and another work licensed +under a Compatible Licence, this Distribution or Communication can be done +under the terms of this Compatible Licence. For the sake of this clause, +‘Compatible Licence’ refers to the licences listed in the appendix attached to +this Licence. Should the Licensee's obligations under the Compatible Licence +conflict with his/her obligations under this Licence, the obligations of the +Compatible Licence shall prevail. + +Provision of Source Code: When distributing or communicating copies of the +Work, the Licensee will provide a machine-readable copy of the Source Code or +indicate a repository where this Source will be easily and freely available +for as long as the Licensee continues to distribute or communicate the Work. + +Legal Protection: This Licence does not grant permission to use the trade +names, trademarks, service marks, or names of the Licensor, except as required +for reasonable and customary use in describing the origin of the Work and reproducing the content of the copyright notice. 6. Chain of Authorship @@ -142,9 +134,9 @@ The original Licensor warrants that the copyright in the Original Work granted hereunder is owned by him/her or licensed to him/her and that he/she has the power and authority to grant the Licence. -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. +Each Contributor warrants that the copyright in the modifications he/she +brings to the Work are owned by him/her or licensed to him/her and that he/she +has the power and authority to grant the Licence. Each time You accept the Licence, the original Licensor and subsequent Contributors grant You a licence to their contributions to the Work, under the @@ -156,66 +148,66 @@ The Work is a work in progress, which is continuously improved by numerous Contributors. It is not a finished work and may therefore contain defects or ‘bugs’ inherent to this type of development. -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. +For the above reason, the Work is provided under the Licence on an ‘as is’ +basis and without warranties of any kind concerning the Work, including +without limitation merchantability, fitness for a particular purpose, absence +of defects or errors, accuracy, non-infringement of intellectual property +rights other than copyright as stated in Article 6 of this Licence. -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. +This disclaimer of warranty is an essential part of the Licence and a +condition for the grant of any rights to the Work. 8. Disclaimer of Liability Except in the cases of wilful misconduct or damages directly caused to natural persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. +material or moral, damages of any kind, arising out of the Licence or of the +use of the Work, including without limitation, damages for loss of goodwill, +work stoppage, computer failure or malfunction, loss of data or any commercial +damage, even if the Licensor has been advised of the possibility of such +damage. However, the Licensor will be liable under statutory product liability +laws as far such laws apply to the Work. 9. Additional agreements -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. +While distributing the Work, You may choose to conclude an additional +agreement, defining obligations or services consistent with this Licence. +However, if accepting obligations, You may act only on your own behalf and on +your sole responsibility, not on behalf of the original Licensor or any other +Contributor, and only if You agree to indemnify, defend, and hold each +Contributor harmless for any liability incurred by, or claims asserted against +such Contributor by the fact You have accepted any warranty or additional +liability. 10. Acceptance of the Licence -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable +The provisions of this Licence can be accepted by clicking on an icon ‘I +agree’ placed under the bottom of a window displaying the text of this Licence +or by affirming consent in any other similar way, in accordance with the rules +of applicable law. Clicking on that icon indicates your clear and irrevocable acceptance of this Licence and all of its terms and conditions. Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. +conditions by exercising any rights granted to You by Article 2 of this +Licence, such as the use of the Work, the creation by You of a Derivative Work +or the Distribution or Communication by You of the Work or copies thereof. 11. Information to the public -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. +In case of any Distribution or Communication of the Work by means of +electronic communication by You (for example, by offering to download the Work +from a remote location) the distribution channel or media (for example, a +website) must at least provide to the public the information requested by the +applicable law regarding the Licensor, the Licence and the way it may be +accessible, concluded, stored and reproduced by the Licensee. 12. Termination of the Licence The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. +any breach by the Licensee of the terms of the Licence. Such a termination +will not terminate the licences of any person who has received the Work from +the Licensee under the Licence, provided such persons remain in full +compliance with the Licence. 13. Miscellaneous @@ -227,10 +219,11 @@ law, this will not affect the validity or enforceability of the Licence as a whole. Such provision will be construed or reformed so as necessary to make it valid and enforceable. -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. +The European Commission may publish other linguistic versions or new versions +of this Licence or updated versions of the Appendix, so far this is required +and reasonable, without reducing the scope of the rights granted by the +Licence. New versions of the Licence will be published with a unique version +number. All linguistic versions of this Licence, approved by the European Commission, have identical value. Parties can take advantage of the linguistic version of @@ -239,49 +232,43 @@ their choice. 14. Jurisdiction Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising +— any litigation resulting from the interpretation of this License, arising between the European Union institutions, bodies, offices or agencies, as a Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. + of Justice of the European Union, as laid down in article 272 of the Treaty + on the Functioning of the European Union, +— any litigation arising between other parties and resulting from the + interpretation of this License, will be subject to the exclusive + jurisdiction of the competent court where the Licensor resides or conducts + its primary business. 15. Applicable Law Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State +— this Licence shall be governed by the law of the European Union Member State where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, +— this licence shall be governed by Belgian law if the Licensor has no seat, residence or registered office inside a European Union Member State. Appendix ‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for +— GNU General Public License (GPL) v. 2, v. 3 +— GNU Affero General Public License (AGPL) v. 3 +— Open Software License (OSL) v. 2.1, v. 3.0 +— Eclipse Public License (EPL) v. 1.0 +— CeCILL v. 2.0, v. 2.1 +— Mozilla Public Licence (MPL) v. 2 +— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 +— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. +— European Union Public Licence (EUPL) v. 1.1, v. 1.2 +— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or + Strong Reciprocity (LiLiQ-R+) + +— The European Commission may update this Appendix to later versions of the + above licences without producing a new version of the EUPL, as long as they + provide the rights granted in Article 2 of this Licence and protect the + covered Source Code from exclusive appropriation. +— All other changes or additions to this Appendix require the production of a + new EUPL version. diff --git a/device-detection/LICENSE b/device-detection/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/device-detection/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/examples/device-detection/LICENSE b/examples/device-detection/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/examples/device-detection/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/examples/ip-intelligence/LICENSE b/examples/ip-intelligence/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/examples/ip-intelligence/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/ip-intelligence/LICENSE b/ip-intelligence/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/ip-intelligence/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/location/LICENSE b/location/LICENSE deleted file mode 100644 index 43159c60b..000000000 --- a/location/LICENSE +++ /dev/null @@ -1,298 +0,0 @@ -/* ********************************************************************* -* This Original Work is copyright of 51 Degrees Mobile Experts Limited. -* Copyright 2019 51 Degrees Mobile Experts Limited, 5 Charlotte Close, -* Caversham, Reading, Berkshire, United Kingdom RG4 7BY. -* -* This Original Work is licensed under the European Union Public Licence (EUPL) -* v.1.2 and is subject to its terms as set out below. -* -* If a copy of the EUPL was not distributed with this file, You can obtain -* one at https://opensource.org/licenses/EUPL-1.2. -* ********************************************************************* */ - -European Union Public Licence -V. 1.2 - -EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as -defined below) which is provided under the terms of this Licence. Any use of -the Work, other than as authorised under this Licence is prohibited (to the -extent such use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: “Licensed under the EUPL”, or has expressed by -any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: -— ‘The Licence’: this Licence. -— ‘The Original Work’: the work or software distributed or communicated by the - ‘Licensor under this Licence, available as Source Code and also as - ‘Executable Code as the case may be. -— ‘Derivative Works’: the works or software that could be created by the - ‘Licensee, based upon the Original Work or modifications thereof. This - ‘Licence does not define the extent of modification or dependence on the - ‘Original Work required in order to classify a work as a Derivative Work; - ‘this extent is determined by copyright law applicable in the country - ‘mentioned in Article 15. -— ‘The Work’: the Original Work or its Derivative Works. -— ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -— ‘The Executable Code’: any code which has generally been compiled and which - is meant to be interpreted by a computer as a program. -— ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. -— ‘Contributor(s)’: any natural or legal person who modifies the Work under - the Licence, or otherwise contributes to the creation of a Derivative Work. -— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. -— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright -vested in the Original Work: - -— use the Work in any circumstance and for all usage, -— reproduce the Work, -— modify the Work, and make Derivative Works based upon the Work, -— communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case - may be, the Work, -— distribute the Work or copies thereof, -— lend and rent the Work or copies thereof, -— sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make -effective the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights -to any patents held by the Licensor, to the extent necessary to make use of -the rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, -in a notice following the copyright notice attached to the Work, a repository -where the Source Code is easily and freely accessible for as long as the -Licensor continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits -from any exception or limitation to the exclusive rights of the rights owners -in the Work, of the exhaustion of those rights or of other applicable -limitations thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and -a copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of -the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions -on the Work or Derivative Work that alter or restrict the terms of the -Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed -under a Compatible Licence, this Distribution or Communication can be done -under the terms of this Compatible Licence. For the sake of this clause, -‘Compatible Licence’ refers to the licences listed in the appendix attached to -this Licence. Should the Licensee's obligations under the Compatible Licence -conflict with his/her obligations under this Licence, the obligations of the -Compatible Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the -Work, the Licensee will provide a machine-readable copy of the Source Code or -indicate a repository where this Source will be easily and freely available -for as long as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade -names, trademarks, service marks, or names of the Licensor, except as required -for reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she -brings to the Work are owned by him/her or licensed to him/her and that he/she -has the power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ -basis and without warranties of any kind concerning the Work, including -without limitation merchantability, fitness for a particular purpose, absence -of defects or errors, accuracy, non-infringement of intellectual property -rights other than copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a -condition for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the -use of the Work, including without limitation, damages for loss of goodwill, -work stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such -damage. However, the Licensor will be liable under statutory product liability -laws as far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional -agreement, defining obligations or services consistent with this Licence. -However, if accepting obligations, You may act only on your own behalf and on -your sole responsibility, not on behalf of the original Licensor or any other -Contributor, and only if You agree to indemnify, defend, and hold each -Contributor harmless for any liability incurred by, or claims asserted against -such Contributor by the fact You have accepted any warranty or additional -liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I -agree’ placed under the bottom of a window displaying the text of this Licence -or by affirming consent in any other similar way, in accordance with the rules -of applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this -Licence, such as the use of the Work, the creation by You of a Derivative Work -or the Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of -electronic communication by You (for example, by offering to download the Work -from a remote location) the distribution channel or media (for example, a -website) must at least provide to the public the information requested by the -applicable law regarding the Licensor, the Licence and the way it may be -accessible, concluded, stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. Such a termination -will not terminate the licences of any person who has received the Work from -the Licensee under the Licence, provided such persons remain in full -compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions -of this Licence or updated versions of the Appendix, so far this is required -and reasonable, without reducing the scope of the rights granted by the -Licence. New versions of the Licence will be published with a unique version -number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, -— any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty - on the Functioning of the European Union, -— any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive - jurisdiction of the competent court where the Licensor resides or conducts - its primary business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, -— this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, -— this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: -— GNU General Public License (GPL) v. 2, v. 3 -— GNU Affero General Public License (AGPL) v. 3 -— Open Software License (OSL) v. 2.1, v. 3.0 -— Eclipse Public License (EPL) v. 1.0 -— CeCILL v. 2.0, v. 2.1 -— Mozilla Public Licence (MPL) v. 2 -— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -— European Union Public Licence (EUPL) v. 1.1, v. 1.2 -— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or - Strong Reciprocity (LiLiQ-R+) - -— The European Commission may update this Appendix to later versions of the - above licences without producing a new version of the EUPL, as long as they - provide the rights granted in Article 2 of this Licence and protect the - covered Source Code from exclusive appropriation. -— All other changes or additions to this Appendix require the production of a - new EUPL version. - -/* ********************************************************************* -* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -* amended by the European Commission) shall be deemed incompatible for -* the purposes of the Work and the provisions of the compatibility -* clause in Article 5 of the EUPL shall not apply. -* -* If using the Work as, or as part of, a network application, by -* including the attribution notice(s) required under Article 5 of the EUPL -* in the end user terms of the application under an appropriate heading, -* such notice(s) shall fulfil the requirements of that article. -* ********************************************************************* */ diff --git a/pipeline/LICENSE b/pipeline/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/pipeline/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/pipeline/pipeline.core/configuration-validation b/pipeline/pipeline.core/configuration-validation new file mode 160000 index 000000000..b2740eddf --- /dev/null +++ b/pipeline/pipeline.core/configuration-validation @@ -0,0 +1 @@ +Subproject commit b2740eddf2824290157cca5f1f7d00f150162f87 diff --git a/pipeline/pipeline.core/pom.xml b/pipeline/pipeline.core/pom.xml index 3e30f3f64..39a8335ff 100644 --- a/pipeline/pipeline.core/pom.xml +++ b/pipeline/pipeline.core/pom.xml @@ -88,6 +88,19 @@ + + + src/main/resources + + + + configuration-validation + + pipelineOptions.xsd + + + org.apache.maven.plugins diff --git a/pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd b/pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd deleted file mode 100644 index e04f6dc41..000000000 --- a/pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd +++ /dev/null @@ -1,42 +0,0 @@ - - - - This is an XML schema for 51Degrees pipeline options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 9a668883e0c65ef22dbd8f48bce02bb6eede8fe8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 00:53:35 +0000 Subject: [PATCH 88/95] Name the data update logging test's key so it is not an old variable name (#111) The environment variable names check fails on rc at line 76 of DataUpdateServiceLoggingTests.java, where the constant LICENSE_KEY starts a wrapped line and so matches the check's start of line form for an old name. The constant holds a made up licence key for the test rather than naming an environment variable, so it becomes FULL_KEY, beside MASKED_KEY, and the test reads the same. --- .../engines/services/DataUpdateServiceLoggingTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java index 4ec40a1b0..15a2f9a1b 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -70,10 +70,10 @@ public class DataUpdateServiceLoggingTests { // 13 September 2020, 12:26:40 UTC. private static final long FILE_MODIFIED_MILLIS = 1600000000000L; // A license key of the length and characters the Distributor issues. - private static final String LICENSE_KEY = "QF8ZR2KD7NMB4XWT9HJC3LPA"; + private static final String FULL_KEY = "QF8ZR2KD7NMB4XWT9HJC3LPA"; private static final String MASKED_KEY = "********************3LPA"; private static final String KEYED_URL = UPDATE_URL + "?LicenseKeys=" + - LICENSE_KEY + "&Download=True&Type=HashV41"; + FULL_KEY + "&Download=True&Type=HashV41"; private static final String MASKED_URL = UPDATE_URL + "?LicenseKeys=" + MASKED_KEY + "&Download=True&Type=HashV41"; @@ -430,7 +430,7 @@ private AspectEngineDataFileDefault newKeyedDataFile(boolean inMemory) { AspectEngineDataFileDefault file = newDataFile(inMemory, false, false); DataFileConfiguration config = file.getConfiguration(); config.setDataUpdateLicenseKeys( - Collections.singletonList(LICENSE_KEY)); + Collections.singletonList(FULL_KEY)); config.setUrlFormatter(dataFile -> KEYED_URL); return file; } @@ -449,7 +449,7 @@ private void assertKeyOnlyMasked() { } assertFalse( "The license key is in full in: " + text, - text.contains(LICENSE_KEY)); + text.contains(FULL_KEY)); } } From 6644140a928f6a2739626d6db7e826c352333b59 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 03:48:23 +0000 Subject: [PATCH 89/95] Set the licence key field the live 51Did test reads (#113) DidClientLiveTests declares a licenceKey field, never assigns it, and reads it in identifiersCreatedFor to build the "&license=" part of the form body, so DidClient.encode is handed null and anIdentifierForAnotherAddress_ReportsEveryFactor fails with a NullPointerException. That is the whole of the Cloud 51Did failure on rc. init() now takes the licence key once, keeps it in the field and gives that field to the client, which is the shape the file had before the lookup moved from env() to LiveCloudKeys.find() and was inlined into the constructor call. No fail() is added for a missing licence key, because LiveCloudKeys says the licence key is optional and both tests that need one already skip on client.hasLicenceKey(). One lookup now feeds both that guard and the form body, so the field cannot be null where it is read. Test code only, so nothing a user sees changes. --- .../test/java/fiftyone/pipeline/did/DidClientLiveTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java index ea6055154..6794f4ee8 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java @@ -70,8 +70,10 @@ public void init() { if (resourceKey == null) { fail(LiveCloudKeys.noResourceKey()); } - client = new DidClient( - resourceKey, LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES)); + // One lookup serves both, so the key the form body sends and the + // one hasLicenceKey() answers for can never disagree. + licenceKey = LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); + client = new DidClient(resourceKey, licenceKey); } @Test From 0b9257f4fac665a7084b830a603ca42109adeee1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 11:46:40 +0000 Subject: [PATCH 90/95] Take the free data files from the run's asset cache (#114) Each test job downloaded the Asn data file again with the data repository's script, outside the asset cache that the run fills once and every test job restores. They are now fetched as assets with the others, restored from the cache in each test job, and copied or linked to where the tests and examples look for them, so no test job downloads a data file. The notes say CI takes every data file from the assets, and that a new data file needs an entry in common-ci's steps/fetch-assets.ps1. --- AGENTS.md | 7 +++++-- ci/README.md | 9 +++++---- ci/fetch-assets.ps1 | 12 ++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7a0e7b924..b5125dfc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -246,8 +246,11 @@ from wherever Maven is run. Do not add a `.mvn` folder below the root. Those folders are the data repository, a submodule of cxx, where each free data file is a `.link` until `get-data-files.ps1` or `get-data-files.sh` -downloads it. `ci/fetch-assets.ps1` runs the script, and the examples' -`DataFileHelper.findDataFile` runs it for a file that is missing. Look for a +downloads it. In CI, `ci/fetch-assets.ps1` puts every file there from +common-ci's assets, which each run caches once, so no test job downloads a +data file. A file added to the data repository needs an entry in +common-ci's `steps/fetch-assets.ps1` before CI can use it. The examples' +`DataFileHelper.findDataFile` runs the script for a file that is missing. Look for a free file by its folder and name, such as `device-detection/51Degrees-LiteV4.1.hash`, so the search does not stop at a file of the same name elsewhere, such as a Git LFS pointer in an old diff --git a/ci/README.md b/ci/README.md index b80c3659e..b00a5034a 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,13 +20,14 @@ repositories used to do for one package tree each. request tests the examples against the libraries it changes. The publish workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). -- [fetch-assets.ps1](fetch-assets.ps1) links every data file it downloads into +- [fetch-assets.ps1](fetch-assets.ps1) fetches every data file as one of + common-ci's assets, which each run caches once, and links or copies it into `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests and examples find them. Those are the folders of the [data](https://github.com/51Degrees/data) repository, which is a submodule - of cxx. The script then runs the data repository's `get-data-files.ps1`, - which keeps the linked files and downloads the free files that are not - assets, which is the Asn file. + of cxx. No test job downloads a data file, so a file added to the data + repository needs an entry in common-ci's `steps/fetch-assets.ps1` before CI + can use it. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 4062061b1..600d3f6fc 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -19,20 +19,16 @@ foreach ($asset in $deviceDetectionAssets) { New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$asset" -Path "$deviceDetectionData/$asset" } -./steps/fetch-assets.ps1 -IpIntelligence:$IpIntelligence -IpIntelligenceUrl:$IpIntelligenceUrl -Assets '51Degrees-EnterpriseIpiV41.ipi', '51Degrees-LiteIpiV41.ipi' +./steps/fetch-assets.ps1 -IpIntelligence:$IpIntelligence -IpIntelligenceUrl:$IpIntelligenceUrl -Assets '51Degrees-EnterpriseIpiV41.ipi', '51Degrees-LiteIpiV41.ipi', '51Degrees-IPIV4AsnIpiV41.ipi' New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-EnterpriseIpiV41.ipi" -Path "$ipIntelligenceData/51Degrees-EnterpriseIpiV41.ipi" # The asset is fetched under the blob name, but the tests, DataFileHelper and # the data repository scripts use the unpacked name 51Degrees-LiteV41.ipi, so # link it as that. New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$ipIntelligenceData/51Degrees-LiteV41.ipi" -# The data repository's script downloads every linked file that is not -# already there. The files linked above are kept, so this gets the free files -# that are not assets, which is the Asn file. -& "$data/get-data-files.ps1" -if (-not $?) { - throw "The data repository's get-data-files.ps1 could not download every file" -} +# The Asn file the IP Intelligence tests use. Every data file is an asset, so +# a test job restores it from the run's cache rather than downloading it. +Copy-Item "assets/51Degrees-IPIV4AsnIpiV41.ipi" $ipIntelligenceData Write-Host "Assets hashes:" Get-FileHash -Algorithm MD5 -Path assets/* From 36c153d5cc824b83f461a556735ecde8793ceaf4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 21:01:06 +0000 Subject: [PATCH 91/95] Keep reading the old cloud endpoint name and bring back its constants (#116) * Test that the old cloud endpoint names still work A patch release keeps every public name, and the name of an environment variable a package reads is a public name. These tests ask for FOD_CLOUD_API_URL to be read when _51DEGREES_CLOUD_ENDPOINT is not set, for _51DEGREES_CLOUD_ENDPOINT to win when both are set, for the use of the old name to be logged as deprecated once, and for the two constants that held the old name to hold it again. They cover the cloud request engine builder, the 51Did client, which is run in a Java process of its own so that it reads a real environment, and the cloud engine example. The cases of the environment variable name check now expect the old endpoint name to be allowed. The tests fail until the fix that follows. * Keep reading the old cloud endpoint name and bring back its constants The cloud endpoint variable was renamed from FOD_CLOUD_API_URL to _51DEGREES_CLOUD_ENDPOINT, and the old name was no longer read. A patch release keeps every public name, and that covers the name of an environment variable a package reads. The cloud request engine builder, the 51Did client and the cloud engine example now read _51DEGREES_CLOUD_ENDPOINT first and FOD_CLOUD_API_URL when that is not set or is empty. The first use of the old name in a process is logged as a warning that names its replacement. The engine builder logs through its logger factory, the example through its own, and the 51Did client through java.util.logging, because that module has no logging dependency and this adds none. Constants.FOD_CLOUD_API_URL comes back holding "FOD_CLOUD_API_URL", deprecated and pointing at Constants.CLOUD_ENDPOINT_VARIABLE. DidClient.ENDPOINT_VARIABLE holds "FOD_CLOUD_API_URL" again and is deprecated, and the new DidClient.CLOUD_ENDPOINT_VARIABLE holds "_51DEGREES_CLOUD_ENDPOINT". The environment variable name check no longer fails on FOD_CLOUD_API_URL, because the packages read it again. * Test that every old environment variable name still works Every environment variable name that a package, a test, an example or a CI script reads is a public name, so a patch release keeps reading the old one when the new one is not set. These tests ask for that wherever main read a name that rc had stopped reading. 51DEGREES_IPI_PATH and 51DEGREES_RESOURCE_KEY in the IP Intelligence test helpers and example helpers, 51DEGREES_DD_PATH in the on-premise web example, and RESOURCE_KEY and LICENSE_KEY in the cloud engine example, the 51Did example and the 51Did live test keys. They also ask for the four helper constants that held an old name to hold it again. The cases of the environment variable name check now expect an old name to be allowed only in the files that read it as a deprecated fallback, and refused everywhere else. The tests fail until the fix that follows. * Keep reading every old environment variable name as a deprecated fallback Every environment variable name that a package, a test, an example or a CI script reads is a public name. Each reader that main had and rc changed now reads the new name first and the old name when the new one is not set or is empty, and says once that the old name is deprecated and which name replaces it. 51DEGREES_IPI_PATH is read again by the IP Intelligence test helper FileUtils, the example helper DataFileHelper and the country translation integration test. 51DEGREES_RESOURCE_KEY is read again by the test helper KeyUtils and the example helper KeyHelper. 51DEGREES_DD_PATH is read again by the on-premise web example's configuration. RESOURCE_KEY is read again by the cloud engine example, the 51Did example and the 51Did live test keys, and LICENSE_KEY by the last two. FileUtils.IPI_PATH_ENV_VAR, KeyUtils.RESOURCE_KEY_ENV_VAR, DataFileHelper.IPI_PATH_ENV_VAR and KeyHelper.RESOURCE_KEY_ENV_VAR hold their old values again and are deprecated. IPI_PATH_VARIABLE and RESOURCE_KEY_VARIABLE beside them hold the new names, and the code that named the old constants now names those. The environment variable name check has FOD_CLOUD_API_URL back in its pattern, and allows each old name only in the files that read it as a deprecated fallback or test that read. * Guard two tests against the old names they now meet Two tests run a helper with nothing set, and each checked and cleared only the new name of a variable. The helpers now read the old name when the new one is not set, so each test failed wherever the old name was set. KeyHelperTest.getOrSetTestResourceKey_NoKey_StoresNothing now also checks and clears 51DEGREES_RESOURCE_KEY, and ExamplesWithoutSettingsTest also checks and clears 51DEGREES_IPI_PATH, through the deprecated constant that holds it. --- ci/check-environment-variable-names.Tests.ps1 | 59 ++++++ ci/check-environment-variable-names.ps1 | 91 +++++++- .../examples/web/GettingStartedWebOnPrem.java | 28 +++ .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 7 +- .../web/OnPremDataFileOptionTest.java | 20 ++ .../examples/console/GettingStartedMixed.java | 2 +- .../console/GettingStartedOnPrem.java | 2 +- .../examples/console/MetadataOnPrem.java | 2 +- .../examples/console/OfflineProcessing.java | 2 +- .../console/ExamplesWithoutSettingsTest.java | 18 +- .../examples/shared/DataFileHelper.java | 56 ++++- .../examples/shared/KeyHelper.java | 60 +++++- .../examples/shared/DataFileHelperTest.java | 71 +++++++ .../examples/shared/KeyHelperTest.java | 53 ++++- .../examples/web/GettingStartedWebMixed.java | 4 +- .../examples/web/GettingStartedWebOnPrem.java | 4 +- .../shared/testhelpers/FileUtils.java | 48 ++++- .../shared/testhelpers/KeyUtils.java | 40 +++- .../shared/testhelpers/FileUtilsTest.java | 67 +++++- .../shared/testhelpers/KeyUtilsTest.java | 64 +++++- .../CountryTranslationIntegrationTest.java | 33 ++- .../OnPremisePerformanceProfileTests.java | 2 +- pipeline/README.md | 2 + .../CloudRequestEngineBuilder.java | 39 +++- .../flowelements/Constants.java | 8 + .../CloudRequestEngineBuilderTests.java | 144 +++++++++++++ .../developerexamples/cloudengine/Main.java | 57 ++++- .../cloudengine/EndpointTests.java | 119 +++++++++++ .../fodid/CreatorContextDemoServer.java | 31 ++- .../fodid/DeprecatedKeyNameTests.java | 196 ++++++++++++++++++ .../developerexamples/fodid/KeyProbe.java | 48 +++++ .../java/fiftyone/pipeline/did/DidClient.java | 57 ++++- .../did/DidClientEnvironmentTests.java | 190 +++++++++++++++++ .../fiftyone/pipeline/did/DidClientTests.java | 7 +- .../fiftyone/pipeline/did/EndpointProbe.java | 71 +++++++ .../fiftyone/pipeline/did/LiveCloudKeys.java | 63 +++++- .../pipeline/did/LiveCloudKeysTests.java | 105 ++++++++++ 37 files changed, 1784 insertions(+), 86 deletions(-) create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java create mode 100644 pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java create mode 100644 pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java create mode 100644 pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java create mode 100644 pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 index 7b9273d80..84460b9a8 100644 --- a/ci/check-environment-variable-names.Tests.ps1 +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -89,4 +89,63 @@ Describe 'the check over a repository' { Remove-Item -Recurse -Force $repo } } + + It 'passes an old name in a file that reads it as a deprecated fallback' { + $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) + New-Item -ItemType Directory $repo | Out-Null + try { + git -C $repo init -q + $file = Join-Path $repo 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java' + New-Item -ItemType Directory (Split-Path $file) -Force | Out-Null + Set-Content -LiteralPath $file ` + -Value 'String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL";' + git -C $repo add -A + { & $script:Check -RepoRoot $repo } | Should -Not -Throw + } finally { + Remove-Item -Recurse -Force $repo + } + } + + It 'fails on the same old name in a file that is not named for it' { + $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) + New-Item -ItemType Directory $repo | Out-Null + try { + git -C $repo init -q + Set-Content -LiteralPath (Join-Path $repo 'Other.java') ` + -Value 'String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL";' + git -C $repo add -A + { & $script:Check -RepoRoot $repo } | + Should -Throw '*and 1 old names are read*' + } finally { + Remove-Item -Recurse -Force $repo + } + } +} + +Describe 'Test-DeprecatedFallback' { + + # An old name is allowed only in a file that reads it as a deprecated + # fallback or tests that read, and only the names listed for that file. + It 'allows in ' -ForEach @( + @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = '"FOD_CLOUD_API_URL' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"RESOURCE_KEY' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"LICENSE_KEY' } + @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51DEGREES_IPI_PATH' } + @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_DD_PATH' } + ) { + Test-DeprecatedFallback $Path $Value | Should -BeTrue + } + + It 'refuses in ' -ForEach @( + @{ Path = 'pipeline/pipeline.core/src/main/java/Any.java'; Value = '"FOD_CLOUD_API_URL' } + @{ Path = 'ci/run-unit-tests.ps1'; Value = 'env:RESOURCE_KEY' } + @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = '"RESOURCE_KEY' } + @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = 'LEGACY_ENV_VARS' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"FOD_CLOUD_API_URL' } + @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51DEGREES_DD_PATH' } + @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51degrees_ipi_path' } + @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_IPI_PATH' } + ) { + Test-DeprecatedFallback $Path $Value | Should -BeFalse + } } diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 index 6fa467c2d..11d75e85c 100644 --- a/ci/check-environment-variable-names.ps1 +++ b/ci/check-environment-variable-names.ps1 @@ -1,12 +1,15 @@ # Every 51Degrees environment variable name in this repository starts with an # underscore, because a POSIX shell cannot set a name that starts with a digit, -# and each variable has one name with nothing read in its place. This reads -# every file git tracks, which leaves out build output and the submodules, and -# fails on any name that starts with the digit and on any old name that is -# still read. +# and each variable has one name. An old name is still read, as a deprecated +# fallback for when its new name is not set, by the files listed in +# DeprecatedFallbacks below and by nothing else. This reads every file git +# tracks, which leaves out build output and the submodules, and fails on any +# name that starts with the digit and on any old name, outside the files +# listed for that name. # -# Dot-sourcing the script defines Find-EnvironmentNameProblem without reading -# the repository, which is how its tests use it. +# Dot-sourcing the script defines Find-EnvironmentNameProblem and +# Test-DeprecatedFallback without reading the repository, which is how its +# tests use them. [CmdletBinding()] param( @@ -53,6 +56,79 @@ $script:FallbackPattern = [regex]::new( '(?:RESOURCE_KEY|LICENSE_KEY|FOD_CLOUD_API_URL)(?![A-Za-z0-9_])|' + '\bLEGACY_[A-Z_]*ENV_VARS?\b') +# The files that read an old name as a deprecated fallback, or that test +# that read, each with the old names it may hold. The names are put together +# from parts so that this file does not match itself. +$oldEndpoint = 'FOD_CLOUD' + '_API_URL' +$oldResourceKey = 'RESOURCE' + '_KEY' +$oldLicenseKey = 'LICENSE' + '_KEY' +$oldDigitKey = $digitInitial + $oldResourceKey +$oldIpiPath = $digitInitial + 'IPI_PATH' +$oldDdPath = $digitInitial + 'DD_PATH' +$cloudEngine = 'pipeline/pipeline.cloudrequestengine/src' +$did = 'pipeline/pipeline.did/src' +$examples = 'pipeline/pipeline.developer-examples/pipeline.developer-examples' +$ipiShared = 'ip-intelligence/ip-intelligence.shared/src' +$ipiHelpers = 'fiftyone/ipintelligence/shared/testhelpers' +$ipiExamples = 'examples/ip-intelligence/shared/src' +$ipiExampleHelpers = 'fiftyone/ipintelligence/examples/shared' +$onPrem = 'examples/device-detection/web/getting-started.onprem/src' +$onPremPackage = 'fiftyone/devicedetection/examples/web' +$script:DeprecatedFallbacks = @{ + "$cloudEngine/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java" = + @($oldEndpoint) + "$cloudEngine/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java" = + @($oldEndpoint) + "$did/main/java/fiftyone/pipeline/did/DidClient.java" = + @($oldEndpoint) + "$did/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java" = + @($oldEndpoint) + "$did/test/java/fiftyone/pipeline/did/LiveCloudKeys.java" = + @($oldResourceKey, $oldLicenseKey) + "$did/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java" = + @($oldResourceKey, $oldLicenseKey) + "$examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java" = + @($oldEndpoint, $oldResourceKey) + "$examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java" = + @($oldEndpoint, $oldResourceKey) + "$examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java" = + @($oldResourceKey, $oldLicenseKey) + "$examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java" = + @($oldResourceKey, $oldLicenseKey) + "$ipiShared/main/java/$ipiHelpers/FileUtils.java" = @($oldIpiPath) + "$ipiShared/test/java/$ipiHelpers/FileUtilsTest.java" = @($oldIpiPath) + "$ipiShared/main/java/$ipiHelpers/KeyUtils.java" = @($oldDigitKey) + "$ipiShared/test/java/$ipiHelpers/KeyUtilsTest.java" = @($oldDigitKey) + "$ipiExamples/main/java/$ipiExampleHelpers/DataFileHelper.java" = + @($oldIpiPath) + "$ipiExamples/test/java/$ipiExampleHelpers/DataFileHelperTest.java" = + @($oldIpiPath) + "$ipiExamples/main/java/$ipiExampleHelpers/KeyHelper.java" = + @($oldDigitKey) + "$ipiExamples/test/java/$ipiExampleHelpers/KeyHelperTest.java" = + @($oldDigitKey) + 'ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java' = + @($oldIpiPath) + "$onPrem/main/webapp/WEB-INF/51Degrees-OnPrem.xml" = @($oldDdPath) + "$onPrem/main/java/$onPremPackage/GettingStartedWebOnPrem.java" = + @($oldDdPath) + "$onPrem/test/java/$onPremPackage/OnPremDataFileOptionTest.java" = + @($oldDdPath) +} + +# Whether the text matched in the file at this path, relative to the root +# with forward slashes, is an old name that file is listed for. The name is +# the run of name characters the match ends with, compared in its exact +# case. +function Test-DeprecatedFallback([string]$Path, [string]$Value) { + $allowed = $script:DeprecatedFallbacks[$Path] + if ($null -eq $allowed) { + return $false + } + $name = [regex]::Match($Value, '[A-Za-z0-9_]+$').Value + return $allowed -ccontains $name +} + # The problems on one line, each as its kind, 'digit' or 'fallback', and the # text matched. function Find-EnvironmentNameProblem([string]$Line) { @@ -99,6 +175,9 @@ foreach ($relative in $tracked) { foreach ($line in [IO.File]::ReadLines($path)) { $lineNumber++ foreach ($problem in Find-EnvironmentNameProblem $line) { + if (Test-DeprecatedFallback $relative $problem.Value) { + continue + } $text = "${relative}:${lineNumber}: $($problem.Value)" if ($problem.Kind -eq 'digit') { $found += $text diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index 1ed151902..e74dec2b1 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -62,9 +62,37 @@ public class GettingStartedWebOnPrem extends HttpServlet { public static String resourceBase = "examples/device-detection/web/getting-started.onprem/src/main/webapp"; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebOnPrem.class); + private static final String DD_PATH_VARIABLE = "_51DEGREES_DD_PATH"; + private static final String OLD_DD_PATH_VARIABLE = "51DEGREES_DD_PATH"; + + /** + * Logs a warning when the configuration takes the data file from the + * deprecated name, which it does when that name is set and the name + * that replaces it is not. + */ + private static void warnOfOldDataFileName() { + if (isSet(OLD_DD_PATH_VARIABLE) && isSet(DD_PATH_VARIABLE) == false) { + logger.warn( + "The environment variable " + OLD_DD_PATH_VARIABLE + + " is deprecated and " + DD_PATH_VARIABLE + + " replaces it. Set " + DD_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + OLD_DD_PATH_VARIABLE + "."); + } + } + + private static boolean isSet(String name) { + String value = System.getenv(name); + if (value == null || value.isEmpty()) { + value = System.getProperty(name); + } + return value != null && value.isEmpty() == false; + } + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); logger.info("Running Example {}", GettingStartedWebOnPrem.class); + warnOfOldDataFileName(); // If the data file the configuration names is not at that path, it // is searched for, and the free Lite file is downloaded if missing. diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index 5ec7feb45..f6bc4c25d 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -29,10 +29,11 @@ false - ${_51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} + ${_51DEGREES_DD_PATH:-${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}}} LowMemory DeviceDetectionHashEngine diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java index c77110b95..7f9bc4df7 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java @@ -41,11 +41,13 @@ */ public class OnPremDataFileOptionTest { private static final String DD_PATH = "_51DEGREES_DD_PATH"; + private static final String OLD_DD_PATH = "51DEGREES_DD_PATH"; private static final String TEST_DATA_FILE = "TestDataFile"; private static final String LITE = "cxx/data/device-detection/51Degrees-LiteV4.1.hash"; private String savedDdPath; + private String savedOldDdPath; private String savedTestDataFile; @Before @@ -53,17 +55,21 @@ public void saveProperties() { // An environment variable of either name would take precedence over // the system properties set here, so the test cannot run. assumeTrue(System.getenv(DD_PATH) == null); + assumeTrue(System.getenv(OLD_DD_PATH) == null); assumeTrue(System.getenv(TEST_DATA_FILE) == null); assumeTrue(System.getenv(TEST_DATA_FILE.toUpperCase()) == null); savedDdPath = System.getProperty(DD_PATH); + savedOldDdPath = System.getProperty(OLD_DD_PATH); savedTestDataFile = System.getProperty(TEST_DATA_FILE); System.clearProperty(DD_PATH); + System.clearProperty(OLD_DD_PATH); System.clearProperty(TEST_DATA_FILE); } @After public void restoreProperties() { restore(DD_PATH, savedDdPath); + restore(OLD_DD_PATH, savedOldDdPath); restore(TEST_DATA_FILE, savedTestDataFile); } @@ -85,6 +91,20 @@ public void prefersDdPathOverTestDataFile() throws Exception { assertEquals("from-dd-path.hash", dataFile()); } + @Test + public void usesOldDdPathWhenNewIsNotSet() throws Exception { + System.setProperty(TEST_DATA_FILE, "from-test-data-file.hash"); + System.setProperty(OLD_DD_PATH, "from-old-dd-path.hash"); + assertEquals("from-old-dd-path.hash", dataFile()); + } + + @Test + public void prefersDdPathOverOldDdPath() throws Exception { + System.setProperty(OLD_DD_PATH, "from-old-dd-path.hash"); + System.setProperty(DD_PATH, "from-dd-path.hash"); + assertEquals("from-dd-path.hash", dataFile()); + } + private static String dataFile() throws Exception { PipelineOptions options = PipelineOptionsFactory.getOptionsFromFile( getFilePath(resourceBase) + "/WEB-INF/51Degrees-OnPrem.xml"); diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 7c0940afe..f007e2fb3 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -138,7 +138,7 @@ public static void run(String ddDataFile, logger.error("Failed to find IP Intelligence data file at '{}'. " + "Please provide a valid path to an IP Intelligence data file (.ipi). " + "An explicit path can be supplied via the {} environment variable.", - ipiDataFile, DataFileHelper.IPI_PATH_ENV_VAR); + ipiDataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 268d00aa3..e25f3e109 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -129,7 +129,7 @@ public static void run(String dataFile, "An explicit path can be supplied via the {} environment variable. " + "For testing, you can obtain an enterprise data file by contacting us at " + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=data-file-not-found", - dataFile, DataFileHelper.IPI_PATH_ENV_VAR); + dataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index be168e638..223121f1d 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -91,7 +91,7 @@ public static void run(String dataFile, OutputStream output) throws Exception { logger.error("Failed to find IP Intelligence data file at '{}'. " + "Please provide a valid path to an IP Intelligence data file (.ipi). " + "An explicit path can be supplied via the {} environment variable.", - dataFile, DataFileHelper.IPI_PATH_ENV_VAR); + dataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } // Build an on-premise IP Intelligence engine with the low memory diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 8bb5ea91b..0357b64d8 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -143,7 +143,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws logger.error("Failed to find IP Intelligence data file at '{}'. " + "Please provide a valid path to an IP Intelligence data file (.ipi). " + "An explicit path can be supplied via the {} environment variable.", - dataFile, DataFileHelper.IPI_PATH_ENV_VAR); + dataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java index 45a906637..d14a522f6 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java @@ -57,14 +57,20 @@ private interface Main { /** * Runs the example's main with no arguments and the data file and * licence key system properties removed, and returns what it wrote to - * standard output. + * standard output. The deprecated name of the data file variable is + * read when the current one is not set, so both are checked and + * removed. */ + @SuppressWarnings("deprecation") private static String runWithoutSettings(Main main) throws Exception { assumeTrue("Skipping, a data file or licence key is set in the environment", - System.getenv(DataFileHelper.IPI_PATH_ENV_VAR) == null && + System.getenv(DataFileHelper.IPI_PATH_VARIABLE) == null && + System.getenv(DataFileHelper.IPI_PATH_ENV_VAR) == null && System.getenv(LICENSE_KEY) == null && System.getenv(LICENSE_KEY.toUpperCase()) == null); - String dataFile = System.clearProperty(DataFileHelper.IPI_PATH_ENV_VAR); + String dataFile = System.clearProperty(DataFileHelper.IPI_PATH_VARIABLE); + String oldNameDataFile = + System.clearProperty(DataFileHelper.IPI_PATH_ENV_VAR); String licenseKey = System.clearProperty(LICENSE_KEY); PrintStream out = System.out; ByteArrayOutputStream buffer = new ByteArrayOutputStream(); @@ -74,7 +80,11 @@ private static String runWithoutSettings(Main main) throws Exception { } finally { System.setOut(out); if (dataFile != null) { - System.setProperty(DataFileHelper.IPI_PATH_ENV_VAR, dataFile); + System.setProperty(DataFileHelper.IPI_PATH_VARIABLE, dataFile); + } + if (oldNameDataFile != null) { + System.setProperty(DataFileHelper.IPI_PATH_ENV_VAR, + oldNameDataFile); } if (licenseKey != null) { System.setProperty(LICENSE_KEY, licenseKey); diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index 66e118cd2..e617da672 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -47,6 +47,7 @@ import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -133,7 +134,33 @@ public static String findAvailableDataFile(String... candidates) { * hold an explicit path to the IP Intelligence data file. This name is * checked first, before any search of the project space. */ - public static final String IPI_PATH_ENV_VAR = "_51DEGREES_IPI_PATH"; + public static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; + + /** + * The old name of {@link #IPI_PATH_VARIABLE}, read only when that one + * is not set. + * @deprecated use {@link #IPI_PATH_VARIABLE}, and set the variable it + * names in place of this one. + */ + @Deprecated + public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; + + // Set once the use of the deprecated name has been logged. + private static final AtomicBoolean OLD_NAME_LOGGED = + new AtomicBoolean(false); + + /** + * The value of the environment variable of this name, or of the system + * property where that is not set or is empty, or null when neither + * holds a value. + */ + private static String environmentOrProperty(String name) { + String value = System.getenv(name); + if (Objects.isNull(value) || value.isEmpty()) { + value = System.getProperty(name); + } + return Objects.isNull(value) || value.isEmpty() ? null : value; + } public static class DatafileInfo { FiftyOneDataFile fileInfo; @@ -362,7 +389,7 @@ public static void cantFindDataFile(String dataFile) { "somewhere in the project space to be found. An explicit " + "path to the data file can be supplied via the {} " + "environment variable or system property.", - dataFile, IPI_PATH_ENV_VAR); + dataFile, IPI_PATH_VARIABLE); } /** @@ -391,9 +418,11 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { /** * Tries to find the passed file, or if null a default file. Handles both absolute and relative paths. * An absolute path supplied by the caller is used as is. Otherwise an - * explicit path supplied via the {@link #IPI_PATH_ENV_VAR} environment + * explicit path supplied via the {@link #IPI_PATH_VARIABLE} environment * variable or system property is checked before the folder hierarchy is - * searched for the passed filename. + * searched for the passed filename. When neither is set the deprecated + * {@link #IPI_PATH_ENV_VAR} is read, and its first use is logged as a + * warning. * @param dataFilename a filename to find (can be absolute or relative path) * @return a full pathname * @throws Exception if the file was not found @@ -404,16 +433,27 @@ public static String getDataFileLocation(String dataFilename) throws Exception { // system property is checked before any search for the filename if (Objects.isNull(dataFilename) || Paths.get(dataFilename).isAbsolute() == false) { - String envDataFile = System.getenv(IPI_PATH_ENV_VAR); + String envName = IPI_PATH_VARIABLE; + String envDataFile = environmentOrProperty(envName); if (Objects.isNull(envDataFile)) { - envDataFile = System.getProperty(IPI_PATH_ENV_VAR); + envName = IPI_PATH_ENV_VAR; + envDataFile = environmentOrProperty(envName); + if (Objects.nonNull(envDataFile) + && OLD_NAME_LOGGED.compareAndSet(false, true)) { + logger.warn( + "The environment variable " + IPI_PATH_ENV_VAR + + " is deprecated and " + IPI_PATH_VARIABLE + + " replaces it. Set " + IPI_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + IPI_PATH_ENV_VAR + "."); + } } if (Objects.nonNull(envDataFile)) { if (Files.exists(Paths.get(envDataFile))) { return envDataFile; } logger.warn("Ignoring {} value '{}' as no file exists there", - IPI_PATH_ENV_VAR, envDataFile); + envName, envDataFile); } } @@ -451,7 +491,7 @@ public static String getDataFileLocation(String dataFilename) throws Exception { DataFileHelper.cantFindDataFile(dataFilename); throw new IOException("Data file not found at relative path: " + dataFilename + ". Searched in project directories and current working directory." + - " An explicit path can be supplied via the " + IPI_PATH_ENV_VAR + + " An explicit path can be supplied via the " + IPI_PATH_VARIABLE + " environment variable.", notFound); } } diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java index acb4fcf7c..2342f4047 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; public class KeyHelper { @@ -35,7 +36,20 @@ public class KeyHelper { * hold the resource key. This name is checked first, before the legacy * {@link #TEST_RESOURCE_KEY} name. */ - public static final String RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"; + public static final String RESOURCE_KEY_VARIABLE = "_51DEGREES_RESOURCE_KEY"; + + /** + * The old name of {@link #RESOURCE_KEY_VARIABLE}, read only when that + * one is not set. + * @deprecated use {@link #RESOURCE_KEY_VARIABLE}, and set the variable + * it names in place of this one. + */ + @Deprecated + public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; + + // Set once the use of the deprecated name has been logged. + private static final AtomicBoolean OLD_NAME_LOGGED = + new AtomicBoolean(false); public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); @@ -64,7 +78,7 @@ public class KeyHelper { public static String requireTestResourceKey(String asksFor, String... products) { return requireResourceKey(KeyHelper::readKey, Arrays.asList( - RESOURCE_KEY_ENV_VAR, TEST_RESOURCE_KEY), asksFor, products); + RESOURCE_KEY_VARIABLE, TEST_RESOURCE_KEY), asksFor, products); } static String requireResourceKey(Function read, @@ -75,6 +89,10 @@ static String requireResourceKey(Function read, names.addAll(exampleNames); for (String name : names) { String value = read.apply(name); + if (KeyUtils.isInvalidKey(value) + && RESOURCE_KEY_VARIABLE.equals(name)) { + value = oldNameResourceKey(read); + } if (KeyUtils.isInvalidKey(value) == false) { return value.trim(); } @@ -82,6 +100,28 @@ static String requireResourceKey(Function read, throw new AssertionError(noResourceKey(names, asksFor, products)); } + /** + * The key held under the deprecated {@link #RESOURCE_KEY_ENV_VAR}, or + * null when there is none. It is read only when + * {@link #RESOURCE_KEY_VARIABLE} holds no key, and its first use is + * logged as a warning. + */ + private static String oldNameResourceKey(Function read) { + String value = read.apply(RESOURCE_KEY_ENV_VAR); + if (KeyUtils.isInvalidKey(value)) { + return null; + } + if (OLD_NAME_LOGGED.compareAndSet(false, true)) { + logger.warn( + "The environment variable " + RESOURCE_KEY_ENV_VAR + + " is deprecated and " + RESOURCE_KEY_VARIABLE + + " replaces it. Set " + RESOURCE_KEY_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + RESOURCE_KEY_ENV_VAR + "."); + } + return value; + } + static String noResourceKey(List names, String asksFor, String... products) { return "This test calls the live 51Degrees cloud service, so it " + @@ -149,16 +189,22 @@ public static String getOrSetTestResourceKey(String value) { /** * Obtain a resource key from the passed argument, * from environment variable or from a property. The aligned - * {@link #RESOURCE_KEY_ENV_VAR} name is checked first, then the - * passed variable name. Store as System Property under the - * passed variable name. + * {@link #RESOURCE_KEY_VARIABLE} name is checked first, then the + * deprecated {@link #RESOURCE_KEY_ENV_VAR}, then the passed variable + * name. Store as System Property under the passed variable name. */ public static String getOrSetResourceKey(String value, String variableName, String errorMessage, boolean shouldThrow) { if (Objects.isNull(value)) { // check the aligned name first, then the legacy name - value = KeyUtils.getNamedKey(RESOURCE_KEY_ENV_VAR); + value = KeyUtils.getNamedKey(RESOURCE_KEY_VARIABLE); + if (Objects.isNull(value) || value.isEmpty()) { + String oldNameKey = oldNameResourceKey(KeyUtils::getNamedKey); + if (Objects.nonNull(oldNameKey)) { + value = oldNameKey; + } + } if (Objects.isNull(value)) { value = KeyUtils.getNamedKey(variableName); } @@ -170,7 +216,7 @@ public static String getOrSetResourceKey(String value, String variableName, "runnable example,\n - as an Environment Variable named \"\u001B[36m{}\u001B[0m\"," + "\n - as a System Property named \"\u001B[36m{}\u001B[0m\" " + "(the legacy name \"\u001B[36m{}\u001B[0m\" is also still accepted).", - RESOURCE_KEY_ENV_VAR, RESOURCE_KEY_ENV_VAR, variableName); + RESOURCE_KEY_VARIABLE, RESOURCE_KEY_VARIABLE, variableName); logger.error(errorMessage); if (shouldThrow) { throw new IllegalStateException("\"" + value + "\" is not a valid resource key"); diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java index 6d578521f..e1a3418b3 100644 --- a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java @@ -25,12 +25,14 @@ import fiftyone.pipeline.engines.Constants.PerformanceProfiles; import org.junit.Test; +import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; public class DataFileHelperTest { @@ -56,4 +58,73 @@ public void findPerformanceProfile_Unknown_WritesNamesAccepted() { written.toString().contains(profile.name())); } } + + private static final String NEW_PATH_NAME = "_51DEGREES_IPI_PATH"; + private static final String OLD_PATH_NAME = "51DEGREES_IPI_PATH"; + + /** + * System properties stand in for the environment variables, because + * the lookup reads either and a test cannot set an environment + * variable. + */ + @Test + public void getDataFileLocation_OldNameWhenNewIsNotSet() + throws Exception { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File file = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(OLD_PATH_NAME, file.getPath()); + assertEquals(file.getPath(), + DataFileHelper.getDataFileLocation(null)); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + file.delete(); + } + } + + @Test + public void getDataFileLocation_NewNameWinsOverOld() throws Exception { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File newFile = File.createTempFile("new-path-name", ".ipi"); + File oldFile = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(NEW_PATH_NAME, newFile.getPath()); + System.setProperty(OLD_PATH_NAME, oldFile.getPath()); + assertEquals(newFile.getPath(), + DataFileHelper.getDataFileLocation(null)); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + newFile.delete(); + oldFile.delete(); + } + } + + @Test + @SuppressWarnings("deprecation") + public void oldPathConstantKeepsItsValue() throws Exception { + assertEquals(OLD_PATH_NAME, DataFileHelper.IPI_PATH_ENV_VAR); + assertTrue(DataFileHelper.class.getField("IPI_PATH_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_PATH_NAME, + DataFileHelper.class.getField("IPI_PATH_VARIABLE").get(null)); + } + + private static void assumePathNamesNotInEnvironment() { + assumeTrue(System.getenv(NEW_PATH_NAME) == null); + assumeTrue(System.getenv(OLD_PATH_NAME) == null); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } } diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java index 809b7382f..df25ee4b2 100644 --- a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java @@ -24,6 +24,7 @@ import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -42,10 +43,11 @@ public class KeyHelperTest { private static final String KEY_B = "AgAAAAAAAAAAAAAAAAAAAAAA"; /** - * The names getOrSetTestResourceKey reads, in its order. + * The names getOrSetTestResourceKey reads, in its order, apart from + * the deprecated name it reads when the first is not set. */ private static final List EXAMPLE_NAMES = Arrays.asList( - KeyHelper.RESOURCE_KEY_ENV_VAR, KeyHelper.TEST_RESOURCE_KEY); + KeyHelper.RESOURCE_KEY_VARIABLE, KeyHelper.TEST_RESOURCE_KEY); /** * With no key anywhere, the lookup logs that there is none and stores @@ -53,20 +55,24 @@ public class KeyHelperTest { */ @Test public void getOrSetTestResourceKey_NoKey_StoresNothing() { + // The deprecated name is read as well, so it must not be set + // either. + List namesRead = new ArrayList<>(EXAMPLE_NAMES); + namesRead.add(OLD_KEY_NAME); // A test can clear a system property but not an environment // variable. - for (String name : EXAMPLE_NAMES) { + for (String name : namesRead) { assumeTrue(name + " is set in the environment", System.getenv(name) == null && System.getenv(name.toUpperCase()) == null); } Map saved = new HashMap<>(); - for (String name : EXAMPLE_NAMES) { + for (String name : namesRead) { saved.put(name, System.clearProperty(name)); } try { assertNull(KeyHelper.getOrSetTestResourceKey(false)); - for (String name : EXAMPLE_NAMES) { + for (String name : namesRead) { assertNull(name, System.getProperty(name)); } } finally { @@ -118,12 +124,45 @@ public void requireResourceKey_FirstValidNameWins() { @Test public void requireResourceKey_ExampleNamesInTheirOrder() { Map values = new HashMap<>(); - values.put(KeyHelper.RESOURCE_KEY_ENV_VAR, KEY_A); + values.put(KeyHelper.RESOURCE_KEY_VARIABLE, KEY_A); values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, EXAMPLE_NAMES, "every IP Intelligence property", "ip")); - values.remove(KeyHelper.RESOURCE_KEY_ENV_VAR); + values.remove(KeyHelper.RESOURCE_KEY_VARIABLE); assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, EXAMPLE_NAMES, "every IP Intelligence property", "ip")); } + + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "51DEGREES_RESOURCE_KEY"; + + @Test + public void requireResourceKey_OldNameWhenNewIsNotSet() { + Map values = new HashMap<>(); + values.put(OLD_KEY_NAME, KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + Arrays.asList(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY), + "every IP Intelligence property", "ip")); + } + + @Test + public void requireResourceKey_NewNameWinsOverOld() { + Map values = new HashMap<>(); + values.put(NEW_KEY_NAME, KEY_B); + values.put(OLD_KEY_NAME, KEY_A); + assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, + Arrays.asList(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY), + "every IP Intelligence property", "ip")); + } + + @Test + @SuppressWarnings("deprecation") + public void oldKeyConstantKeepsItsValue() throws Exception { + assertEquals(OLD_KEY_NAME, KeyHelper.RESOURCE_KEY_ENV_VAR); + assertEquals(true, KeyHelper.class.getField("RESOURCE_KEY_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_KEY_NAME, + KeyHelper.class.getField("RESOURCE_KEY_VARIABLE").get(null)); + } } diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 1accfb336..18f5bc05b 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -74,7 +74,7 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_ENV_VAR; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_VARIABLE; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.*; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -120,7 +120,7 @@ public static void main(String[] args) throws Exception { } catch (Exception e) { logger.warn("IP Intelligence data file not found at: {}. " + "An explicit path can be supplied via the {} environment variable.", - ipiDataFile, IPI_PATH_ENV_VAR); + ipiDataFile, IPI_PATH_VARIABLE); logger.warn("Will attempt to use default path from XML configuration"); logger.debug("Error finding IP Intelligence data file", e); } diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index 0520012b8..d87b20d5d 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -84,7 +84,7 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_ENV_VAR; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_VARIABLE; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -125,7 +125,7 @@ public static void main(String[] args) throws Exception { } catch (Exception e) { logger.warn("Data file not found at expected location: {}. " + "An explicit path can be supplied via the {} environment variable.", - dataFile, IPI_PATH_ENV_VAR); + dataFile, IPI_PATH_VARIABLE); logger.warn("Will attempt to use default path from XML configuration"); logger.debug("Error finding data file", e); } diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java index d3db631eb..ded023de3 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -43,7 +44,20 @@ public class FileUtils { * supply an explicit path to the IPI data file. When set and the file * exists it is used in preference to searching the project space. */ - public static final String IPI_PATH_ENV_VAR = "_51DEGREES_IPI_PATH"; + public static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; + + /** + * The old name of {@link #IPI_PATH_VARIABLE}, read only when that one + * is not set. + * @deprecated use {@link #IPI_PATH_VARIABLE}, and set the variable it + * names in place of this one. + */ + @Deprecated + public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; + + // Set once the use of the deprecated name has been reported. + private static final AtomicBoolean OLD_NAME_REPORTED = + new AtomicBoolean(false); public static final String ENTERPRISE_IPI_DATA_FILE_NAME = "Enterprise-IpIntelligenceV41.ipi"; @@ -99,15 +113,26 @@ public static File getHashFile() { } /** - * Check the aligned {@link #IPI_PATH_ENV_VAR} environment variable and - * system property for an explicit path to the IPI data file. + * Check the aligned {@link #IPI_PATH_VARIABLE} environment variable and + * system property for an explicit path to the IPI data file, and when + * neither is set the deprecated {@link #IPI_PATH_ENV_VAR}, whose first + * use is reported on standard error. * * @return a file if the path is set and the file exists, otherwise null */ private static File getExplicitHashFile() { - String explicitPath = System.getenv(IPI_PATH_ENV_VAR); + String explicitPath = environmentOrProperty(IPI_PATH_VARIABLE); if (Objects.isNull(explicitPath)) { - explicitPath = System.getProperty(IPI_PATH_ENV_VAR); + explicitPath = environmentOrProperty(IPI_PATH_ENV_VAR); + if (Objects.nonNull(explicitPath) + && OLD_NAME_REPORTED.compareAndSet(false, true)) { + System.err.println( + "The environment variable " + IPI_PATH_ENV_VAR + + " is deprecated and " + IPI_PATH_VARIABLE + + " replaces it. Set " + IPI_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + IPI_PATH_ENV_VAR + "."); + } } if (Objects.nonNull(explicitPath)) { File explicitFile = new File(explicitPath); @@ -118,6 +143,19 @@ private static File getExplicitHashFile() { return null; } + /** + * The value of the environment variable of this name, or of the system + * property where that is not set or is empty, or null when neither + * holds a value. + */ + private static String environmentOrProperty(String name) { + String value = System.getenv(name); + if (Objects.isNull(value) || value.isEmpty()) { + value = System.getProperty(name); + } + return Objects.isNull(value) || value.isEmpty() ? null : value; + } + public static File getEvidenceFile() { if (Objects.nonNull(EVIDENCE_FILE)) { return EVIDENCE_FILE.orElse(null); diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java index 8c6b1f225..75533a5ef 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java @@ -24,6 +24,7 @@ import java.util.Base64; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; /** * Helpers to obtain keys from the environment @@ -33,19 +34,48 @@ public class KeyUtils { * Name of the aligned environment variable or system property which * supplies the resource key. This name is checked before any legacy name. */ - public static final String RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"; + public static final String RESOURCE_KEY_VARIABLE = "_51DEGREES_RESOURCE_KEY"; + + /** + * The old name of {@link #RESOURCE_KEY_VARIABLE}, read only when that + * one is not set. + * @deprecated use {@link #RESOURCE_KEY_VARIABLE}, and set the variable + * it names in place of this one. + */ + @Deprecated + public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; + + // Set once the use of the deprecated name has been reported. + private static final AtomicBoolean OLD_NAME_REPORTED = + new AtomicBoolean(false); /** * Obtain a resource key from an environment variable or a system property. *

      - * The aligned name {@link #RESOURCE_KEY_ENV_VAR} is checked first and the - * supplied legacy name is checked second. Each name is looked up using + * The aligned name {@link #RESOURCE_KEY_VARIABLE} is checked first. When + * it is not set or is empty the deprecated {@link #RESOURCE_KEY_ENV_VAR} + * is checked, and its first use is reported on standard error. The + * supplied legacy name is checked last. Each name is looked up using * {@link #getNamedKey(String)}. * @param legacyKeyName legacy name to try when the aligned name is not set - * @return the resource key, or null if neither name is set + * @return the resource key, or null if no name is set */ public static String getResourceKey(String legacyKeyName) { - String resourceKey = getNamedKey(RESOURCE_KEY_ENV_VAR); + String resourceKey = getNamedKey(RESOURCE_KEY_VARIABLE); + if (Objects.isNull(resourceKey) || resourceKey.isEmpty()) { + String oldNameKey = getNamedKey(RESOURCE_KEY_ENV_VAR); + if (Objects.nonNull(oldNameKey) && oldNameKey.isEmpty() == false) { + if (OLD_NAME_REPORTED.compareAndSet(false, true)) { + System.err.println( + "The environment variable " + RESOURCE_KEY_ENV_VAR + + " is deprecated and " + RESOURCE_KEY_VARIABLE + + " replaces it. Set " + RESOURCE_KEY_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + RESOURCE_KEY_ENV_VAR + "."); + } + return oldNameKey; + } + } if (Objects.isNull(resourceKey)) { resourceKey = getNamedKey(legacyKeyName); } diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java index bb1b302b7..b1478996b 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java @@ -30,6 +30,7 @@ import java.io.IOException; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; public class FileUtilsTest { @@ -47,5 +48,69 @@ public void testUserDir() throws IOException { assertEquals(". and user dir should be the same", f.getCanonicalPath(), g.getCanonicalPath()); } + private static final String NEW_PATH_NAME = "_51DEGREES_IPI_PATH"; + private static final String OLD_PATH_NAME = "51DEGREES_IPI_PATH"; -} \ No newline at end of file + /** + * System properties stand in for the environment variables, because + * the lookup reads either and a test cannot set an environment + * variable. + */ + @Test + public void oldPathNameIsReadWhenNewIsNotSet() throws IOException { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File file = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(OLD_PATH_NAME, file.getPath()); + assertEquals(file, FileUtils.getHashFile()); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + file.delete(); + } + } + + @Test + public void newPathNameWinsOverOld() throws IOException { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File newFile = File.createTempFile("new-path-name", ".ipi"); + File oldFile = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(NEW_PATH_NAME, newFile.getPath()); + System.setProperty(OLD_PATH_NAME, oldFile.getPath()); + assertEquals(newFile, FileUtils.getHashFile()); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + newFile.delete(); + oldFile.delete(); + } + } + + @Test + @SuppressWarnings("deprecation") + public void oldPathConstantKeepsItsValue() throws Exception { + assertEquals(OLD_PATH_NAME, FileUtils.IPI_PATH_ENV_VAR); + assertTrue(FileUtils.class.getField("IPI_PATH_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_PATH_NAME, + FileUtils.class.getField("IPI_PATH_VARIABLE").get(null)); + } + + private static void assumePathNamesNotInEnvironment() { + assumeTrue(System.getenv(NEW_PATH_NAME) == null); + assumeTrue(System.getenv(OLD_PATH_NAME) == null); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } +} diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java index f102146b9..03a7d5341 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; public class KeyUtilsTest { @Test @@ -33,4 +34,65 @@ public void testGetEnv(){ System.out.println(KeyUtils.getNamedKey("path")); } -} \ No newline at end of file + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "51DEGREES_RESOURCE_KEY"; + + /** + * System properties stand in for the environment variables, because + * the lookup reads either and a test cannot set an environment + * variable. + */ + @Test + public void oldKeyNameIsReadWhenNewIsNotSet() { + assumeKeyNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_KEY_NAME); + String savedOld = System.clearProperty(OLD_KEY_NAME); + try { + System.setProperty(OLD_KEY_NAME, "old-name-key"); + assertEquals("old-name-key", + KeyUtils.getResourceKey("NoSuchLegacyName")); + } finally { + restore(NEW_KEY_NAME, savedNew); + restore(OLD_KEY_NAME, savedOld); + } + } + + @Test + public void newKeyNameWinsOverOld() { + assumeKeyNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_KEY_NAME); + String savedOld = System.clearProperty(OLD_KEY_NAME); + try { + System.setProperty(NEW_KEY_NAME, "new-name-key"); + System.setProperty(OLD_KEY_NAME, "old-name-key"); + assertEquals("new-name-key", + KeyUtils.getResourceKey("NoSuchLegacyName")); + } finally { + restore(NEW_KEY_NAME, savedNew); + restore(OLD_KEY_NAME, savedOld); + } + } + + @Test + @SuppressWarnings("deprecation") + public void oldKeyConstantKeepsItsValue() throws Exception { + assertEquals(OLD_KEY_NAME, KeyUtils.RESOURCE_KEY_ENV_VAR); + assertTrue(KeyUtils.class.getField("RESOURCE_KEY_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_KEY_NAME, + KeyUtils.class.getField("RESOURCE_KEY_VARIABLE").get(null)); + } + + private static void assumeKeyNamesNotInEnvironment() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + assumeTrue(System.getenv(OLD_KEY_NAME) == null); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } +} diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java index 028bec978..3f5ab406b 100644 --- a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java @@ -50,7 +50,8 @@ * complete, localized, ordered country dropdown is produced. *

      * The data file is resolved from the {@code TestDataFile} system property, the - * {@code _51DEGREES_IPI_PATH} environment variable, or a few well known + * {@code _51DEGREES_IPI_PATH} environment variable (or its deprecated + * name {@code 51DEGREES_IPI_PATH} when that is not set), or a few well known * locations, and the test is skipped if none is found. *

      * The IP Intelligence element data key has differed between engine versions @@ -63,10 +64,38 @@ public class CountryTranslationIntegrationTest { private static final String SAMPLE_IP = "8.8.8.8"; + private static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; + private static final String OLD_IPI_PATH_VARIABLE = "51DEGREES_IPI_PATH"; + + private static boolean oldNameReported = false; + + /** + * The path the environment gives, from the deprecated name when the + * name that replaces it is not set, saying so on standard error the + * first time. + */ + private static String pathFromEnvironment() { + String path = System.getenv(IPI_PATH_VARIABLE); + if (path == null || path.isEmpty()) { + path = System.getenv(OLD_IPI_PATH_VARIABLE); + if (path != null && path.isEmpty() == false + && oldNameReported == false) { + oldNameReported = true; + System.err.println( + "The environment variable " + OLD_IPI_PATH_VARIABLE + + " is deprecated and " + IPI_PATH_VARIABLE + + " replaces it. Set " + IPI_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + OLD_IPI_PATH_VARIABLE + "."); + } + } + return path; + } + private static String resolveDataFile() { String[] candidates = { System.getProperty("TestDataFile"), - System.getenv("_51DEGREES_IPI_PATH"), + pathFromEnvironment(), "51Degrees-IPIV4EnterpriseIpiV41.ipi", "../51Degrees-IPIV4EnterpriseIpiV41.ipi", "../../51Degrees-IPIV4EnterpriseIpiV41.ipi", diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java index 01664c283..c7fb46d09 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java @@ -61,7 +61,7 @@ public static void findDataFile() { public void requireDataFile() { Assume.assumeTrue( "No IP Intelligence data file was found, so this test is " - + "skipped. Set " + FileUtils.IPI_PATH_ENV_VAR + " to the path " + + "skipped. Set " + FileUtils.IPI_PATH_VARIABLE + " to the path " + "of an .ipi file, or download the Lite file with " + "cxx/data/get-data-files.", dataFile != null); diff --git a/pipeline/README.md b/pipeline/README.md index 22e8e8118..66b7d319d 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -139,6 +139,8 @@ and on our [website](https://51degrees.com/documentation/_device_detection__exam Every example that calls the 51Degrees cloud takes its endpoint from the `_51DEGREES_CLOUD_ENDPOINT` environment variable, which is the cloud API base including the `/api/v4/` segment and defaults to `https://cloud.51degrees.com/api/v4/` (the cloud-engine example defaults to the star sign service it was written for, as its row above says). This is the same variable the cloud request engine in this repository honours, so setting it once points every example, and any pipeline built with the engine, at the same place. The resource key comes from `_51DEGREES_RESOURCE_KEY`. +The variable was called `FOD_CLOUD_API_URL` before, and that name is deprecated. The cloud request engine, the 51Did client and the cloud-engine example still read `FOD_CLOUD_API_URL` when `_51DEGREES_CLOUD_ENDPOINT` is not set, and each logs a warning the first time it does. When both are set, `_51DEGREES_CLOUD_ENDPOINT` is used. + A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons. This is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index 576df6df1..db9f39443 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; /** @@ -71,6 +72,11 @@ public class CloudRequestEngineBuilder extends return System.getenv(name); }; + // Set once the use of the deprecated end point variable has been + // logged, so that a process building many engines logs it once. + private static final AtomicBoolean oldEndPointVariableLogged = + new AtomicBoolean(false); + public CloudRequestEngineBuilder(ILoggerFactory loggerFactory) { this(loggerFactory, new HttpClientDefault()); } @@ -91,12 +97,12 @@ protected CloudRequestEngine newEngine(List properties) throws Exception /* * Check if endPoint has been explicitly set via setEndpoint. - * If not check for environment variable _51DEGREES_CLOUD_ENDPOINT. + * If not check for environment variable _51DEGREES_CLOUD_ENDPOINT, + * and then for its deprecated name FOD_CLOUD_API_URL. * If nothing else is set, use default value. */ if (endPoint == null || endPoint.isEmpty()) { - String envVarEndPoint = getEnvVar.apply( - Constants.CLOUD_ENDPOINT_VARIABLE); + String envVarEndPoint = endPointFromEnvironment(); if (envVarEndPoint == null || envVarEndPoint.isEmpty()) { setEndpoint(Constants.END_POINT_DEFAULT); } @@ -156,6 +162,33 @@ public CloudDiscoveryState getDiscoveryState() { return engine == null ? null : engine.getDiscoveryState(); } + /** + * The end point the environment gives, or null or empty when it gives + * none. {@link Constants#CLOUD_ENDPOINT_VARIABLE} is read first. Only + * when that is not set or is empty is the deprecated + * {@link Constants#FOD_CLOUD_API_URL} read, and the first use of it in + * the process is logged as a warning. + */ + @SuppressWarnings("deprecation") + private String endPointFromEnvironment() { + String value = getEnvVar.apply(Constants.CLOUD_ENDPOINT_VARIABLE); + if (value != null && value.isEmpty() == false) { + return value; + } + value = getEnvVar.apply(Constants.FOD_CLOUD_API_URL); + if (value != null && value.isEmpty() == false + && oldEndPointVariableLogged.compareAndSet(false, true)) { + loggerFactory.getLogger(CloudRequestEngineBuilder.class.getName()) + .warn("The environment variable " + + Constants.FOD_CLOUD_API_URL + " is deprecated and " + + Constants.CLOUD_ENDPOINT_VARIABLE + " replaces it. " + + "Set " + Constants.CLOUD_ENDPOINT_VARIABLE + + " to the same value, as a later release may no " + + "longer read " + Constants.FOD_CLOUD_API_URL + "."); + } + return value; + } + private static String trimmed(String value) { return value == null ? "" : value.trim(); } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java index 1a77c3537..84b3e024e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java @@ -28,6 +28,14 @@ public class Constants { "https://cloud.51degrees.com/api/v4"; // Environment variable name to set end point public static final String CLOUD_ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; + /** + * The old name of the environment variable that sets the end point, + * read only when {@link #CLOUD_ENDPOINT_VARIABLE} is not set. + * @deprecated use {@link #CLOUD_ENDPOINT_VARIABLE}, and set the + * environment variable it names in place of this one. + */ + @Deprecated + public static final String FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL"; // Default timeout for each request to the cloud service, as the // specification's Configuration options table gives. public static final int DEFAULT_TIMEOUT_MILLIS = 2000; diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java index 87851ceb0..8159049a0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java @@ -22,9 +22,11 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; +import fiftyone.common.testhelpers.TestLogger; import fiftyone.pipeline.cloudrequestengine.CloudRequestException; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.LoggerFactory; @@ -32,6 +34,10 @@ import java.io.IOException; import java.lang.reflect.Field; import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.Function; import static org.junit.jupiter.api.Assertions.*; @@ -40,6 +46,17 @@ public class CloudRequestEngineBuilderTests extends CloudRequestEngineTestsBase private final String testResourceKey = "resource_key"; private final String testEndpoint = "https://testEndpoint/"; private final String testEnvVarEndpoint = "https://testEnvVarEndpoint/"; + private final String testOldEnvVarEndpoint = + "https://testOldEnvVarEndpoint/"; + + // The deprecated name of the end point variable, which is also the + // name of the constant that holds it. + private static final String OLD_ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; + + // The old name is logged as deprecated once in a process, so what was + // logged is carried from one test of this class to the next. + private static final List deprecationsInEarlierTests = + new ArrayList<>(); public CloudRequestEngineBuilderTests() throws MalformedURLException { super(); @@ -49,6 +66,11 @@ public CloudRequestEngineBuilderTests() throws MalformedURLException { public void setUp() throws IOException { configureMockedClient(); } + + @AfterEach + public void countDeprecations() { + deprecationsInEarlierTests.addAll(deprecationsLogged()); + } @Test @SuppressWarnings("unused") @@ -122,4 +144,126 @@ public void CloudEndPoint_Default_Setting() throws Exception { assertEquals( Constants.END_POINT_DEFAULT + "/", getEndpointValue(builder)); } + + /** + * Gives the builder an environment holding only the variables in the + * map, so that a test chooses which of the two names is set. + */ + private void setEnvironment( + CloudRequestEngineBuilder builder, + Map environment) throws Exception { + Field getEnvVarField = builder.getClass().getDeclaredField("getEnvVar"); + getEnvVarField.setAccessible(true); + getEnvVarField.set( + builder, + (Function)(name) -> environment.get(name)); + } + + /** + * A builder that finds the old name in the environment, and the new + * name only where a value is given for it. + */ + private CloudRequestEngineBuilder builderWithOldName( + String newNameValue) throws Exception { + Map environment = new HashMap<>(); + environment.put(OLD_ENDPOINT_VARIABLE, testOldEnvVarEndpoint); + if (newNameValue != null) { + environment.put(Constants.CLOUD_ENDPOINT_VARIABLE, newNameValue); + } + CloudRequestEngineBuilder builder = + new CloudRequestEngineBuilder(loggerFactory, httpClient); + setEnvironment(builder, environment); + return builder.setResourceKey(testResourceKey); + } + + /** + * The warnings this test's loggers hold that name the old variable. + */ + private List deprecationsLogged() { + List result = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + for (String warning : logger.warningsLogged) { + if (warning.contains(OLD_ENDPOINT_VARIABLE)) { + result.add(warning); + } + } + } + return result; + } + + @Test + public void CloudEndPoint_Old_Environment_Name() throws Exception { + CloudRequestEngineBuilder builder = builderWithOldName(null); + + builder.build(); + + assertEquals(testOldEnvVarEndpoint, getEndpointValue(builder)); + assertEquals( + testOldEnvVarEndpoint + "accessibleproperties", + getPropertiesEndpointValue(builder)); + } + + @Test + public void CloudEndPoint_Old_Environment_Name_When_New_Is_Empty() + throws Exception { + CloudRequestEngineBuilder builder = builderWithOldName(""); + + builder.build(); + + assertEquals(testOldEnvVarEndpoint, getEndpointValue(builder)); + } + + @Test + public void CloudEndPoint_New_Environment_Name_Wins() throws Exception { + CloudRequestEngineBuilder builder = + builderWithOldName(testEnvVarEndpoint); + + builder.build(); + + assertEquals(testEnvVarEndpoint, getEndpointValue(builder)); + assertEquals( + testEnvVarEndpoint + "accessibleproperties", + getPropertiesEndpointValue(builder)); + assertEquals(0, deprecationsLogged().size()); + } + + @Test + public void CloudEndPoint_Explicit_Setting_Beats_Old_Name() + throws Exception { + CloudRequestEngineBuilder builder = builderWithOldName(null); + + builder.setEndpoint(testEndpoint).build(); + + assertEquals(testEndpoint, getEndpointValue(builder)); + assertEquals(0, deprecationsLogged().size()); + } + + /** + * Two engines built from the old name log one warning between them, + * however many of this class's tests read the old name before this one. + */ + @Test + public void CloudEndPoint_Old_Environment_Name_Logged_Once() + throws Exception { + builderWithOldName(null).build(); + builderWithOldName(null).build(); + + List logged = new ArrayList<>(deprecationsInEarlierTests); + logged.addAll(deprecationsLogged()); + assertEquals(1, logged.size(), logged.toString()); + String warning = logged.get(0); + assertTrue( + warning.contains(Constants.CLOUD_ENDPOINT_VARIABLE), warning); + assertTrue(warning.contains("deprecated"), warning); + } + + @Test + public void CloudEndPoint_Old_Constant_Is_Kept() throws Exception { + Field old = Constants.class.getField(OLD_ENDPOINT_VARIABLE); + + assertEquals(OLD_ENDPOINT_VARIABLE, old.get(null)); + assertTrue(old.isAnnotationPresent(Deprecated.class)); + assertEquals( + "_51DEGREES_CLOUD_ENDPOINT", Constants.CLOUD_ENDPOINT_VARIABLE); + } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 4b039d94c..963827bdb 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -34,6 +34,8 @@ import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngine; import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngineBuilder; +import java.util.concurrent.atomic.AtomicBoolean; + public class Main { private static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); private static final HttpClient httpClient = new HttpClientDefault(); @@ -53,6 +55,26 @@ public class Main { private static final String FIFTYONE_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/"; + /** + * The name the cloud endpoint is read from, and the deprecated name + * read when that one is not set. + */ + private static final String ENDPOINT_VARIABLE = + "_51DEGREES_CLOUD_ENDPOINT"; + private static final String OLD_ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; + + private static final AtomicBoolean oldEndpointVariableLogged = + new AtomicBoolean(false); + + /** + * The deprecated name of the first of the resource key names, read + * when that one is not set. + */ + private static final String OLD_RESOURCE_KEY_VARIABLE = "RESOURCE_KEY"; + + private static final AtomicBoolean oldResourceKeyVariableLogged = + new AtomicBoolean(false); + /** * The cloud endpoint, taken from _51DEGREES_CLOUD_ENDPOINT when that is * set, which is the variable every 51Degrees cloud example honours and @@ -63,10 +85,24 @@ public class Main { * which is the private hosting option of the cloud service. Both run * the same service, so the example works unchanged. Normalised to * end in one slash so the builder appends its three paths to it - * directly. + * directly. The deprecated name FOD_CLOUD_API_URL is read when + * _51DEGREES_CLOUD_ENDPOINT is not set, and the first use of it is + * logged as a warning. */ static String endpoint() { - String value = env("_51DEGREES_CLOUD_ENDPOINT"); + String value = env(ENDPOINT_VARIABLE); + if (value == null) { + value = env(OLD_ENDPOINT_VARIABLE); + if (value != null + && oldEndpointVariableLogged.compareAndSet(false, true)) { + loggerFactory.getLogger(Main.class.getName()).warn( + "The environment variable " + OLD_ENDPOINT_VARIABLE + + " is deprecated and " + ENDPOINT_VARIABLE + + " replaces it. Set " + ENDPOINT_VARIABLE + + " to the same value, as a later release may no " + + "longer read " + OLD_ENDPOINT_VARIABLE + "."); + } + } if (value == null) { // A resource key and an endpoint travel together, because a // key is only known to the service that issued it. So the @@ -98,11 +134,26 @@ static String endpoint() { /** * The 51Degrees resource key that was found, or null when none of the - * names above is set anywhere. + * names above is set anywhere. When the first name is not set its + * deprecated name RESOURCE_KEY is read before the others, and the + * first use of it is logged as a warning. */ static String configuredResourceKey() { for (String name : RESOURCE_KEY_NAMES) { String value = env(name); + if (value == null && name.equals(RESOURCE_KEY_NAMES[0])) { + value = env(OLD_RESOURCE_KEY_VARIABLE); + if (value != null + && oldResourceKeyVariableLogged.compareAndSet( + false, true)) { + loggerFactory.getLogger(Main.class.getName()).warn( + "The environment variable " + OLD_RESOURCE_KEY_VARIABLE + + " is deprecated and " + name + + " replaces it. Set " + name + + " to the same value, as a later release may no longer " + + "read " + OLD_RESOURCE_KEY_VARIABLE + "."); + } + } if (value != null) { return value; } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java new file mode 100644 index 000000000..c6969a90e --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java @@ -0,0 +1,119 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.cloudengine; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Tests which names the example takes its endpoint and its resource key + * from. The example reads each name as an environment variable and then as + * a system property, so these tests set system properties, and are skipped + * where the environment already sets a name they use, because the + * environment would answer first. + */ +public class EndpointTests { + + private static final String NEW_NAME = "_51DEGREES_CLOUD_ENDPOINT"; + private static final String OLD_NAME = "FOD_CLOUD_API_URL"; + + @BeforeEach + public void setUp() { + assumeTrue(System.getenv(NEW_NAME) == null); + assumeTrue(System.getenv(OLD_NAME) == null); + clear(); + } + + @AfterEach + public void clear() { + System.clearProperty(NEW_NAME); + System.clearProperty(OLD_NAME); + } + + @Test + public void Endpoint_Old_Name_Alone() { + System.setProperty(OLD_NAME, "https://old.example/api/v4"); + + assertEquals("https://old.example/api/v4/", Main.endpoint()); + } + + @Test + public void Endpoint_New_Name_Wins() { + System.setProperty(NEW_NAME, "https://new.example/api/v4/"); + System.setProperty(OLD_NAME, "https://old.example/api/v4/"); + + assertEquals("https://new.example/api/v4/", Main.endpoint()); + } + + @Test + public void Endpoint_Neither_Name() { + String expected = Main.configuredResourceKey() == null + ? "https://51degrees.com/starsign/api/" + : "https://cloud.51degrees.com/api/v4/"; + + assertEquals(expected, Main.endpoint()); + } + + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "RESOURCE_KEY"; + + @Test + public void ResourceKey_Old_Name_Alone() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + assumeTrue(System.getenv(OLD_KEY_NAME) == null); + String savedNew = System.clearProperty(NEW_KEY_NAME); + try { + System.setProperty(OLD_KEY_NAME, "old-name-key"); + + assertEquals("old-name-key", Main.configuredResourceKey()); + } finally { + System.clearProperty(OLD_KEY_NAME); + if (savedNew != null) { + System.setProperty(NEW_KEY_NAME, savedNew); + } + } + } + + @Test + public void ResourceKey_New_Name_Wins() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + assumeTrue(System.getenv(OLD_KEY_NAME) == null); + String savedNew = System.clearProperty(NEW_KEY_NAME); + try { + System.setProperty(NEW_KEY_NAME, "new-name-key"); + System.setProperty(OLD_KEY_NAME, "old-name-key"); + + assertEquals("new-name-key", Main.configuredResourceKey()); + } finally { + System.clearProperty(OLD_KEY_NAME); + System.clearProperty(NEW_KEY_NAME); + if (savedNew != null) { + System.setProperty(NEW_KEY_NAME, savedNew); + } + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java index 1057eb480..3eaf3c223 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java @@ -102,9 +102,11 @@ * variables: *

        *
      • {@code _51DEGREES_RESOURCE_KEY}, required. The resource key of the - * page, public by nature.
        • + * page, public by nature. Its deprecated name {@code RESOURCE_KEY} is read + * when it is not set. *
      • {@code _51DEGREES_LICENSE_KEY}, optional. A licence key of the same - * account, server side only.
        • + * account, server side only. Its deprecated name {@code LICENSE_KEY} is + * read when it is not set. *
      • {@code _51DEGREES_CLOUD_ENDPOINT}, optional. The cloud API base including * the {@code /api/v4/} segment, defaulting to * {@code https://cloud.51degrees.com/api/v4/}. This is the same variable @@ -119,9 +121,9 @@ public class CreatorContextDemoServer { static final String RESOURCES = "/fodid/creator-context/"; static final String RESOURCE = - env("_51DEGREES_RESOURCE_KEY"); + key("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); static final String LICENCE = - env("_51DEGREES_LICENSE_KEY"); + key("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); /** * The one client, built at start-up and shared by every request, @@ -462,6 +464,27 @@ static String env(String... names) { return null; } + /** + * The value of the named environment variable, or of its deprecated + * old name when the named one is not set, saying on standard error + * that the old name is deprecated when it is the one used. + */ + static String key(String name, String oldName) { + String value = env(name); + if (value == null) { + value = env(oldName); + if (value != null) { + System.err.println( + "The environment variable " + oldName + + " is deprecated and " + name + + " replaces it. Set " + name + + " to the same value, as a later release may no longer " + + "read " + oldName + "."); + } + } + return value; + } + static String envOr(String name, String fallback) { String value = env(name); return value == null ? fallback : value; diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java new file mode 100644 index 000000000..9aab1e55c --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java @@ -0,0 +1,196 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +import fiftyone.pipeline.did.DidClient; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests which environment variables the example takes its resource key and + * its licence key from. Each test starts {@link KeyProbe} in a Java process + * of its own with the variables it needs, so the example reads a real + * environment. + */ +public class DeprecatedKeyNameTests { + + private static final String RESOURCE_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_RESOURCE_KEY_NAME = "RESOURCE_KEY"; + private static final String LICENCE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; + private static final String OLD_LICENCE_KEY_NAME = "LICENSE_KEY"; + + private static final int PROBE_TIMEOUT_SECONDS = 120; + + @Test + public void oldNamesAlone_GiveTheirKeys() throws Exception { + List output = + probe(null, "old-name-key", null, "old-name-licence"); + + assertEquals("old-name-key", + value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); + assertEquals("old-name-licence", + value(output, KeyProbe.LICENCE_PREFIX), output.toString()); + } + + @Test + public void oldNamesAlone_AreEachReportedAsDeprecatedOnce() + throws Exception { + List output = + probe(null, "old-name-key", null, "old-name-licence"); + + assertEquals(1, deprecations(output, OLD_RESOURCE_KEY_NAME, + RESOURCE_KEY_NAME), output.toString()); + assertEquals(1, deprecations(output, OLD_LICENCE_KEY_NAME, + LICENCE_KEY_NAME), output.toString()); + } + + @Test + public void bothNames_NewNamesWin() throws Exception { + List output = probe( + "new-name-key", "old-name-key", + "new-name-licence", "old-name-licence"); + + assertEquals("new-name-key", + value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); + assertEquals("new-name-licence", + value(output, KeyProbe.LICENCE_PREFIX), output.toString()); + assertEquals(0, deprecations(output, OLD_RESOURCE_KEY_NAME, + RESOURCE_KEY_NAME), output.toString()); + assertEquals(0, deprecations(output, OLD_LICENCE_KEY_NAME, + LICENCE_KEY_NAME), output.toString()); + } + + @Test + public void noNames_GiveNoKeys() throws Exception { + List output = probe(null, null, null, null); + + assertEquals("null", + value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); + assertEquals("null", + value(output, KeyProbe.LICENCE_PREFIX), output.toString()); + } + + /** + * Runs the probe with each variable set to the value given, or absent + * where the value is null, and returns the lines it printed. + */ + private static List probe( + String resourceKey, + String oldResourceKey, + String licenceKey, + String oldLicenceKey) throws Exception { + String java = Paths.get( + System.getProperty("java.home"), "bin", "java").toString(); + ProcessBuilder builder = + new ProcessBuilder(java, KeyProbe.class.getName()); + Map environment = builder.environment(); + environment.put("CLASSPATH", classPath()); + set(environment, RESOURCE_KEY_NAME, resourceKey); + set(environment, OLD_RESOURCE_KEY_NAME, oldResourceKey); + set(environment, LICENCE_KEY_NAME, licenceKey); + set(environment, OLD_LICENCE_KEY_NAME, oldLicenceKey); + builder.redirectErrorStream(true); + + Process process = builder.start(); + List output = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + output.add(line); + } + } + assertTrue( + process.waitFor(PROBE_TIMEOUT_SECONDS, TimeUnit.SECONDS), + "The probe did not finish."); + assertEquals(0, process.exitValue(), output.toString()); + return output; + } + + private static void set( + Map environment, String name, String value) { + if (value == null) { + environment.remove(name); + } else { + environment.put(name, value); + } + } + + /** + * The places the probe loads its classes from, being the classes of + * the example, its test classes, the 51Did package and the JSON + * library. + */ + private static String classPath() throws Exception { + return location(CreatorContextDemoServer.class) + File.pathSeparator + + location(KeyProbe.class) + File.pathSeparator + + location(DidClient.class) + File.pathSeparator + + location(JSONObject.class); + } + + private static String location(Class type) throws Exception { + return Paths.get(type.getProtectionDomain().getCodeSource() + .getLocation().toURI()).toString(); + } + + /** The text after the prefix on the one line that starts with it. */ + private static String value(List output, String prefix) { + String found = null; + for (String line : output) { + if (line.startsWith(prefix)) { + assertNull(found, output.toString()); + found = line.substring(prefix.length()); + } + } + return found; + } + + /** + * How many lines say the old name is deprecated and name the new one. + */ + private static int deprecations( + List output, String oldName, String newName) { + int count = 0; + for (String line : output) { + if (line.contains(" " + oldName + " is deprecated") + && line.contains(newName)) { + count++; + } + } + return count; + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java new file mode 100644 index 000000000..074504798 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java @@ -0,0 +1,48 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +/** + * Prints the resource key and the licence key the example read from the + * environment of the process. {@link DeprecatedKeyNameTests} runs this in + * a Java process of its own, because the example reads both keys once, + * when its class loads, and a process cannot change its own environment + * variables. + */ +public final class KeyProbe { + + static final String RESOURCE_PREFIX = "RESOURCE "; + static final String LICENCE_PREFIX = "LICENCE "; + + private KeyProbe() { + } + + public static void main(String[] args) { + System.out.println( + RESOURCE_PREFIX + CreatorContextDemoServer.RESOURCE); + System.out.println( + LICENCE_PREFIX + CreatorContextDemoServer.LICENCE); + System.out.flush(); + System.exit(0); + } +} diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index cc4e28b92..9c2d067d2 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -59,7 +59,9 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; /** * The client for everything a server does with a 51Did against the @@ -114,7 +116,18 @@ public final class DidClient { * The environment variable read for the API base when the constructor * is given none, the same one the cloud request engine honours. */ - public static final String ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; + public static final String CLOUD_ENDPOINT_VARIABLE = + "_51DEGREES_CLOUD_ENDPOINT"; + + /** + * The old name of the environment variable read for the API base, read + * only when {@link #CLOUD_ENDPOINT_VARIABLE} is not set. + * + * @deprecated use {@link #CLOUD_ENDPOINT_VARIABLE}, and set the + * environment variable it names in place of this one. + */ + @Deprecated + public static final String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; /** * How old the whole held key list may be before a request for a key @@ -136,6 +149,13 @@ public final class DidClient { private static final String USER_AGENT = "pipeline.did/" + version(); + /** + * Set once the use of {@link #ENDPOINT_VARIABLE} has been logged, so + * that a process creating many clients logs it once. + */ + private static final AtomicBoolean OLD_VARIABLE_LOGGED = + new AtomicBoolean(false); + /** * Longest encoded identifier this client will take from a caller. The * figure is arbitrary and deliberately generous, far above anything the @@ -180,7 +200,7 @@ public enum SignatureCheck { /** * A client for the public cloud, or the host named by - * {@value #ENDPOINT_VARIABLE}, with no licence key. + * {@value #CLOUD_ENDPOINT_VARIABLE}, with no licence key. * * @param resourceKey the page's resource key, public by nature */ @@ -190,7 +210,7 @@ public DidClient(String resourceKey) { /** * A client for the public cloud, or the host named by - * {@value #ENDPOINT_VARIABLE}, with a licence key. + * {@value #CLOUD_ENDPOINT_VARIABLE}, with a licence key. * * @param resourceKey the page's resource key, public by nature * @param licenceKey a licence key of the same account, server side @@ -207,9 +227,9 @@ public DidClient(String resourceKey, String licenceKey) { * @param resourceKey the page's resource key, public by nature * @param licenceKey a licence key of the same account, or null * @param endpoint the API base including {@code /api/v4/}, or null - * to read {@value #ENDPOINT_VARIABLE} and fall back - * to {@link #DEFAULT_ENDPOINT}. A trailing slash is - * added where missing. + * to read {@value #CLOUD_ENDPOINT_VARIABLE} and fall + * back to {@link #DEFAULT_ENDPOINT}. A trailing + * slash is added where missing. */ public DidClient(String resourceKey, String licenceKey, String endpoint) { this(builder(resourceKey).licenceKey(licenceKey).endpoint(endpoint)); @@ -268,8 +288,8 @@ public Builder licenceKey(String licenceKey) { /** * @param endpoint the API base including {@code /api/v4/}, or null - * to read {@value #ENDPOINT_VARIABLE} and fall back - * to {@link #DEFAULT_ENDPOINT} + * to read {@value #CLOUD_ENDPOINT_VARIABLE} and + * fall back to {@link #DEFAULT_ENDPOINT} * @return this builder */ public Builder endpoint(String endpoint) { @@ -1152,14 +1172,29 @@ static String encode(String value) { } /** - * The API base to use: the argument, else {@value #ENDPOINT_VARIABLE}, - * else {@link #DEFAULT_ENDPOINT}, ending in exactly one slash so every - * route can be appended directly. + * The API base to use: the argument, else + * {@value #CLOUD_ENDPOINT_VARIABLE}, else the deprecated + * {@value #ENDPOINT_VARIABLE}, else {@link #DEFAULT_ENDPOINT}, ending + * in exactly one slash so every route can be appended directly. The + * first use of the deprecated variable in the process is logged as a + * warning. */ static String resolveEndpoint(String endpoint) { String value = blankToNull(endpoint); + if (value == null) { + value = blankToNull(System.getenv(CLOUD_ENDPOINT_VARIABLE)); + } if (value == null) { value = blankToNull(System.getenv(ENDPOINT_VARIABLE)); + if (value != null + && OLD_VARIABLE_LOGGED.compareAndSet(false, true)) { + Logger.getLogger(DidClient.class.getName()).warning( + "The environment variable " + ENDPOINT_VARIABLE + + " is deprecated and " + CLOUD_ENDPOINT_VARIABLE + + " replaces it. Set " + CLOUD_ENDPOINT_VARIABLE + + " to the same value, as a later release may no " + + "longer read " + ENDPOINT_VARIABLE + "."); + } } if (value == null) { value = DEFAULT_ENDPOINT; diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java new file mode 100644 index 000000000..562bbddb2 --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java @@ -0,0 +1,190 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import org.json.JSONObject; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests which environment variable {@link DidClient} takes its endpoint + * from. Each test starts {@link EndpointProbe} in a Java process of its own + * with the variables it needs, so the client reads a real environment. + */ +public class DidClientEnvironmentTests { + + private static final String NEW_NAME = "_51DEGREES_CLOUD_ENDPOINT"; + private static final String OLD_NAME = "FOD_CLOUD_API_URL"; + + private static final String NEW_ENDPOINT = "https://new.example/api/v4/"; + private static final String OLD_ENDPOINT = "https://old.example/api/v4/"; + + private static final int PROBE_TIMEOUT_SECONDS = 120; + + @Test + public void oldNameAlone_GivesItsEndpoint() throws Exception { + List output = probe(null, OLD_ENDPOINT); + + assertEquals( + Arrays.asList(OLD_ENDPOINT, OLD_ENDPOINT), endpoints(output)); + } + + @Test + public void oldNameAlone_IsLoggedAsDeprecatedOnce() throws Exception { + List logged = logged(probe(null, OLD_ENDPOINT)); + + assertEquals(logged.toString(), 1, logged.size()); + String record = logged.get(0); + assertTrue(record, record.startsWith("WARNING ")); + assertTrue(record, record.contains(OLD_NAME)); + assertTrue(record, record.contains(NEW_NAME)); + assertTrue(record, record.contains("deprecated")); + } + + @Test + public void newNameBlank_OldNameIsRead() throws Exception { + List output = probe(" ", OLD_ENDPOINT); + + assertEquals( + Arrays.asList(OLD_ENDPOINT, OLD_ENDPOINT), endpoints(output)); + } + + @Test + public void bothNames_NewNameWins() throws Exception { + List output = probe(NEW_ENDPOINT, OLD_ENDPOINT); + + assertEquals( + Arrays.asList(NEW_ENDPOINT, NEW_ENDPOINT), endpoints(output)); + assertEquals(output.toString(), 0, logged(output).size()); + } + + @Test + public void neitherName_GivesTheDefault() throws Exception { + List output = probe(null, null); + + assertEquals( + Arrays.asList( + DidClient.DEFAULT_ENDPOINT, DidClient.DEFAULT_ENDPOINT), + endpoints(output)); + assertEquals(output.toString(), 0, logged(output).size()); + } + + @Test + @SuppressWarnings("deprecation") + public void oldConstant_KeepsItsNameAndValue() throws Exception { + assertEquals(OLD_NAME, DidClient.ENDPOINT_VARIABLE); + assertTrue(DidClient.class.getField("ENDPOINT_VARIABLE") + .isAnnotationPresent(Deprecated.class)); + assertEquals( + NEW_NAME, + DidClient.class.getField("CLOUD_ENDPOINT_VARIABLE").get(null)); + } + + /** + * Runs the probe with each variable set to the value given, or absent + * where the value is null, and returns the lines it printed. + */ + private static List probe(String newValue, String oldValue) + throws Exception { + String java = Paths.get( + System.getProperty("java.home"), "bin", "java").toString(); + ProcessBuilder builder = + new ProcessBuilder(java, EndpointProbe.class.getName()); + Map environment = builder.environment(); + environment.put("CLASSPATH", classPath()); + set(environment, NEW_NAME, newValue); + set(environment, OLD_NAME, oldValue); + builder.redirectErrorStream(true); + + Process process = builder.start(); + List output = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + output.add(line); + } + } + assertTrue( + "The probe did not finish.", + process.waitFor(PROBE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals(output.toString(), 0, process.exitValue()); + return output; + } + + private static void set( + Map environment, String name, String value) { + if (value == null) { + environment.remove(name); + } else { + environment.put(name, value); + } + } + + /** + * The places the probe loads its classes from, being the classes of + * this module, its test classes and the JSON library the client uses. + */ + private static String classPath() throws Exception { + return location(DidClient.class) + File.pathSeparator + + location(EndpointProbe.class) + File.pathSeparator + + location(JSONObject.class); + } + + private static String location(Class type) throws Exception { + return Paths.get(type.getProtectionDomain().getCodeSource() + .getLocation().toURI()).toString(); + } + + private static List endpoints(List output) { + return withPrefix(output, EndpointProbe.ENDPOINT_PREFIX); + } + + private static List logged(List output) { + return withPrefix(output, EndpointProbe.LOG_PREFIX); + } + + private static List withPrefix( + List output, String prefix) { + List result = new ArrayList<>(); + for (String line : output) { + if (line.startsWith(prefix)) { + result.add(line.substring(prefix.length())); + } + } + return result; + } +} diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index cea3ec788..501a2cabf 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -129,8 +129,13 @@ public void endpoint_IsNormalisedToOneTrailingSlash() { } @Test + @SuppressWarnings("deprecation") public void endpoint_DefaultsToTheCloudOrTheEnvironment() { - String fromEnvironment = System.getenv(DidClient.ENDPOINT_VARIABLE); + String fromEnvironment = + System.getenv(DidClient.CLOUD_ENDPOINT_VARIABLE); + if (fromEnvironment == null || fromEnvironment.trim().isEmpty()) { + fromEnvironment = System.getenv(DidClient.ENDPOINT_VARIABLE); + } String expected = fromEnvironment == null || fromEnvironment.trim().isEmpty() ? DidClient.DEFAULT_ENDPOINT diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java new file mode 100644 index 000000000..016ee2eeb --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +/** + * Prints the endpoint two {@link DidClient} instances resolve from the + * environment of the process, and every record logged while they do. + * {@link DidClientEnvironmentTests} runs this in a Java process of its own, + * because a process cannot change its own environment variables. + */ +public final class EndpointProbe { + + static final String ENDPOINT_PREFIX = "ENDPOINT "; + static final String LOG_PREFIX = "LOG "; + + private EndpointProbe() { + } + + public static void main(String[] args) { + Logger root = Logger.getLogger(""); + for (Handler handler : root.getHandlers()) { + root.removeHandler(handler); + } + root.addHandler(new Handler() { + @Override + public void publish(LogRecord record) { + System.out.println(LOG_PREFIX + record.getLevel().getName() + + " " + record.getMessage()); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }); + + for (int i = 0; i < 2; i++) { + System.out.println( + ENDPOINT_PREFIX + new DidClient("resource").getEndpoint()); + } + System.out.flush(); + System.exit(0); + } +} diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java index 26061200e..88b51e27e 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java @@ -22,6 +22,10 @@ package fiftyone.pipeline.did; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + /** * Where the tests that call the live 51Degrees cloud service read their * resource key and licence key from, and what they say when there is @@ -65,18 +69,65 @@ final class LiveCloudKeys { private LiveCloudKeys() { } + /** + * The names that have a deprecated old name, each followed by it. The + * old name is read only when its new name is not set. + */ + private static final String[][] DEPRECATED_NAMES = { + {"_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"}, + {"_51DEGREES_LICENSE_KEY", "LICENSE_KEY"}, + }; + + /** The old names whose use has been reported, each reported once. */ + private static final Set REPORTED_OLD_NAMES = + Collections.synchronizedSet(new HashSet()); + /** * The first value set under any of the given names, or null when none - * of them is set anywhere. + * of them is set anywhere. A name with a deprecated old name falls + * back to it, saying so on standard error the first time. */ static String find(String[] names) { for (String name : names) { - String value = System.getenv(name); - if (value == null || value.trim().isEmpty()) { - value = System.getProperty(name); + String value = read(name); + if (value == null) { + String oldName = oldNameOf(name); + if (oldName != null) { + value = read(oldName); + if (value != null && REPORTED_OLD_NAMES.add(oldName)) { + System.err.println( + "The environment variable " + oldName + + " is deprecated and " + name + + " replaces it. Set " + name + + " to the same value, as a later release may no longer " + + "read " + oldName + "."); + } + } } - if (value != null && value.trim().isEmpty() == false) { - return value.trim(); + if (value != null) { + return value; + } + } + return null; + } + + /** + * The trimmed value of the environment variable of this name, or of + * the system property where that is not set or is blank, or null when + * neither holds a value. + */ + private static String read(String name) { + String value = System.getenv(name); + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + return value == null || value.trim().isEmpty() ? null : value.trim(); + } + + private static String oldNameOf(String name) { + for (String[] pair : DEPRECATED_NAMES) { + if (pair[0].equals(name)) { + return pair[1]; } } return null; diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java new file mode 100644 index 000000000..bf0f5a370 --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java @@ -0,0 +1,105 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assume.assumeTrue; + +/** + * Tests which names {@link LiveCloudKeys} reads a key from. System + * properties stand in for the environment variables, because the lookup + * reads either and a test cannot set an environment variable. + */ +public class LiveCloudKeysTests { + + private static final String RESOURCE_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_RESOURCE_KEY_NAME = "RESOURCE_KEY"; + private static final String LICENCE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; + private static final String OLD_LICENCE_KEY_NAME = "LICENSE_KEY"; + private static final String OTHER_NAME = "TestResourceKey"; + + private static final String[] NAMES = { + RESOURCE_KEY_NAME, OLD_RESOURCE_KEY_NAME, + LICENCE_KEY_NAME, OLD_LICENCE_KEY_NAME, OTHER_NAME, + }; + + private final String[] saved = new String[NAMES.length]; + + @Before + public void clearNames() { + for (int i = 0; i < NAMES.length; i++) { + // An environment variable would answer before the system + // properties set here, so the test cannot run. + assumeTrue(System.getenv(NAMES[i]) == null); + saved[i] = System.clearProperty(NAMES[i]); + } + } + + @After + public void restoreNames() { + for (int i = 0; i < NAMES.length; i++) { + if (saved[i] == null) { + System.clearProperty(NAMES[i]); + } else { + System.setProperty(NAMES[i], saved[i]); + } + } + } + + @Test + public void resourceKey_OldNameWhenNewIsNotSet() { + System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); + + assertEquals("old-name-key", + LiveCloudKeys.find(new String[] {RESOURCE_KEY_NAME})); + } + + @Test + public void licenceKey_OldNameWhenNewIsNotSet() { + System.setProperty(OLD_LICENCE_KEY_NAME, "old-name-licence"); + + assertEquals("old-name-licence", + LiveCloudKeys.find(new String[] {LICENCE_KEY_NAME})); + } + + @Test + public void newNameWinsOverOld() { + System.setProperty(RESOURCE_KEY_NAME, "new-name-key"); + System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); + + assertEquals("new-name-key", + LiveCloudKeys.find(new String[] {RESOURCE_KEY_NAME})); + } + + @Test + public void oldNameIsReadOnlyForItsNewName() { + System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); + + assertNull(LiveCloudKeys.find(new String[] {OTHER_NAME})); + } +} From ef34a03dc03d16f61a86935b3210e936c148cc3b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 23:19:50 +0000 Subject: [PATCH 92/95] Ask the IP Intelligence benchmark only for the property it reads (#117) * Test that the IP Intelligence benchmark asks only for the property it reads The properties move into benchmarkProperties, which the new test calls, with the behaviour unchanged, so the test fails here. * Ask the IP Intelligence benchmark only for the property it reads The benchmark asked for RegisteredName and AsnName, so that it ran with the free Asn data file. With the Enterprise data file each lookup then did the work for both properties. It now opens the data file with the low memory profile and asks for RegisteredName where the file has it, and AsnName otherwise. --- .../console/PerformanceBenchmark.java | 45 +++++++++++--- .../PerformanceBenchmarkPropertiesTest.java | 61 +++++++++++++++++++ 2 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index e9b5f5688..d3c03c4e2 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -35,8 +35,9 @@ * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). * * By default the example uses the free Asn IP Intelligence data file, which it downloads when it - * is missing, and is far smaller than the Lite file. It asks for RegisteredName and AsnName, so - * it runs with any of the data files. To obtain an Enterprise data file, please + * is missing, and is far smaller than the Lite file. It asks only for the property it reads, + * being RegisteredName, or AsnName with the free Asn data file, which has no RegisteredName, + * so it runs with any of the data files. To obtain an Enterprise data file, please * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). * * Required Maven Dependencies: @@ -51,6 +52,7 @@ import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.examples.shared.EvidenceHelper; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.IWeightedValue; @@ -103,6 +105,8 @@ public class PerformanceBenchmark { private int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; private List> evidence; private String dataFileLocation; + // the properties asked for when not asking for all of them + private List properties; private PrintWriter writer; // a default set of configurations: (profile, allProperties) @@ -147,6 +151,7 @@ protected void runBenchmarks(PerformanceConfiguration[] performanceConfiguration logger.info("Running Performance example"); this.dataFileLocation = getDataFileLocation(dataFilename); + this.properties = benchmarkProperties(dataFileLocation); File evidenceFile = getEvidenceFile(evidenceFilename); this.evidence = Collections.unmodifiableList( @@ -237,13 +242,37 @@ private void setPipelinePerformanceProperties( // Requesting properties from a single component // reduces processing time compared with requesting properties from multiple components. // If you don't specify any properties, then all properties are returned, - // here we choose "all properties" by specifying none, or just "RegisteredName" - // and "AsnName". The free Asn data file has AsnName, and the Lite and - // Enterprise data files have RegisteredName, so the benchmark runs - // with any of them. + // here we choose "all properties" by specifying none, or just the + // one property the benchmark reads, which benchmarkProperties + // chooses for the data file. if (BooleanUtils.isFalse(config.allProperties)) { - builder.setProperty("RegisteredName"); - builder.setProperty("AsnName"); + for (String property : properties) { + builder.setProperty(property); + } + } + } + + /** + * The properties the benchmark asks for when it does not ask for all of + * them, being only the one it reads. That is RegisteredName where the + * data file has it, as the Lite and Enterprise data files do, and + * AsnName for the free Asn data file, which has no RegisteredName. The + * data file is opened with the low memory profile, which reads little + * of it. + * @param dataFileLocation the data file the benchmark uses + * @return the one property to ask for + * @throws Exception if the data file cannot be opened + */ + static List benchmarkProperties(String dataFileLocation) + throws Exception { + try (IPIntelligenceOnPremiseEngine engine = + new IPIntelligenceOnPremiseEngineBuilder() + .setPerformanceProfile(LowMemory) + .setAutoUpdate(false) + .build(dataFileLocation, false)) { + return Collections.singletonList( + DataFileHelper.hasProperty(engine, "RegisteredName") ? + "RegisteredName" : "AsnName"); } } diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java new file mode 100644 index 000000000..5490a8c0d --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java @@ -0,0 +1,61 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * Checks that the performance benchmark asks only for the property it reads, + * being RegisteredName where the data file has it and AsnName for the free + * Asn data file, which has no RegisteredName. + */ +public class PerformanceBenchmarkPropertiesTest { + + @Test + public void registeredNameAloneWhenTheDataFileHasIt() throws Exception { + String dataFile = DataFileHelper.findAvailableDataFile( + DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH, + DataFileHelper.LITE_DATA_FILE_REL_PATH); + assumeTrue("Skipping test, no IP Intelligence data file with the " + + "RegisteredName property found", + dataFile != null); + assertEquals(Collections.singletonList("RegisteredName"), + PerformanceBenchmark.benchmarkProperties(dataFile)); + } + + @Test + public void asnNameAloneWithTheAsnDataFile() throws Exception { + String dataFile = DataFileHelper.findAvailableDataFile( + DataFileHelper.ASN_DATA_FILE_REL_PATH); + assumeTrue("Skipping test, no free Asn IP Intelligence data file found", + dataFile != null); + assertEquals(Collections.singletonList("AsnName"), + PerformanceBenchmark.benchmarkProperties(dataFile)); + } +} From 551d916535f37a1935952086290fdd05c8b064c7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 02:28:38 +0000 Subject: [PATCH 93/95] Usage sharing finds its values once a call, without copying the Evidence (#120) * Test that usage sharing finds its values without copying the Evidence * Count how often usage sharing reads all its values in one call Usage sharing finds the values it shares on every call, and most calls are repeats it does not share. A call that is not shared copies every Evidence value twice, once in the element and once in its tracker, and a call that is shared copies them three times. A web request is copied as often. * Find the usage sharing values once a call, without copying the Evidence Without a web request ShareUsageEvidence.find returns the Evidence's own read-only map, which is already keyed without regard to case and holds no null value, rather than a copy of every value. With a web request the element makes the one copy and gives it to its ShareUsageTracker, which makes its key from it once for both the look up and the store of its cache, rather than finding the values again for each. * Count the reads of a call that is not shared The read count tests assert what a call that is not shared costs, which is the call the tracker turns down, as most calls repeat one already shared. A call that is shared reads its values again to share them. * Find the usage sharing values only when they are needed The element no longer finds the values on every call. It finds them for the ignore filter when one is set, and otherwise only once the tracker allows the call to be shared. The tracker finds them for its key as it did, which without a web request copies nothing. Giving the element's copy to the tracker for each call measured slower than the copy it saved on a call with one Evidence value, so ShareUsageTracker is as it is on rc. --- .../fiftyone/data/ShareUsageEvidence.java | 8 +- .../fiftyone/flowelements/ShareUsageBase.java | 17 +- .../data/ShareUsageEvidenceTests.java | 117 +++++++++++ .../ShareUsageEvidenceReadTests.java | 187 ++++++++++++++++++ 4 files changed, 323 insertions(+), 6 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java index a48438286..d6e7d22d4 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java @@ -50,11 +50,17 @@ private ShareUsageEvidence() { * Every Evidence value, and every value of the web request when the * Evidence holds it, keyed without regard to case. A key in both keeps * the Evidence value. The web request itself is not one of the values. + * Without a web request the values found are the Evidence's own + * read-only map, as Evidence is already keyed without regard to case and + * holds no null value, so nothing is copied. * @param evidence the Evidence of the request - * @return the values found + * @return the values found, which the caller must not change */ public static Map find(Evidence evidence) { Map values = evidence.asKeyMap(); + if (values.containsKey(HTTP_REQUEST_KEY) == false) { + return values; + } Map found = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); Object request = null; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 9e417dcfe..1fd5bf088 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -645,12 +645,16 @@ public void addPipeline(Pipeline pipeline) { @Override protected void processInternal(FlowData flowData){ boolean ignoreData = false; - Map found = - ShareUsageEvidence.find(flowData.getEvidence()); + // The values are found here only for the ignore filter. Otherwise + // they are found once the tracker allows the call to be shared, as + // most calls repeat one already shared. + Map found = null; logger.debug("processInternal()"); - if (ignoreDataEvidenceFilter != null) { + if (ignoreDataEvidenceFilter != null && + ignoreDataEvidenceFilter.isEmpty() == false) { + found = ShareUsageEvidence.find(flowData.getEvidence()); for (Map.Entry entry : ignoreDataEvidenceFilter) { String key = entry.getKey(); if (found.containsKey(key)) { @@ -789,18 +793,21 @@ private boolean isLocalHost(String address) throws UnknownHostException { /** * Process the supplied request data * @param data the {@link FlowData} instance that provides the evidence - * @param found the values {@link ShareUsageEvidence#find} found for it + * @param found the values {@link ShareUsageEvidence#find} found for it, + * or null if they have not been found yet */ private void processData(FlowData data, Map found) { logger.debug("processData()"); if (random.nextDouble() <= sharePercentage) { // Check if the tracker will allow sharing of this data if (tracker.track(data)) { + Map shared = found != null ? found : + ShareUsageEvidence.find(data.getEvidence()); // Extract the data we want from the evidence and add // it to the collection. try { if (evidenceCollection.offer( - getDataFromEvidence(found), + getDataFromEvidence(shared), addTimeout, TimeUnit.MILLISECONDS)) { queueFullReported = false; diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java new file mode 100644 index 000000000..fee9b7269 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java @@ -0,0 +1,117 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * The usage sharing element finds the values it shares on every request, so + * with no web request finding them reads the Evidence rather than copying + * every value of it. + */ +public class ShareUsageEvidenceTests { + + private static Pipeline createPipeline() throws Exception { + return new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new EmptyEngineBuilder( + LoggerFactory.getILoggerFactory()).build()) + .setAutoCloseElements(true) + .build(); + } + + /** + * With no web request the values found are the Evidence itself, so a + * value added afterwards is among them. A copy would not hold it. + */ + @Test + public void Find_NoWebRequest_ReadsTheEvidenceWithoutCopying() + throws Exception { + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "evidence-ua"); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + data.addEvidence("header.x-later", "later"); + + assertEquals("later", found.get("header.x-later"), + "The Evidence was copied, so every request pays for a copy " + + "of every Evidence value."); + } + } + + /** + * With no web request the values found are still keyed without regard + * to case. + */ + @Test + public void Find_NoWebRequest_KeyedWithoutRegardToCase() + throws Exception { + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.User-Agent", "evidence-ua"); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + + assertEquals("evidence-ua", found.get("HEADER.USER-AGENT")); + assertEquals(1, found.size()); + } + } + + /** + * With a web request its values are added under their own keys, the + * Evidence value is kept for a key in both, and the web request itself + * is not one of the values. + */ + @Test + public void Find_WebRequest_AddsItsValuesAndKeepsTheEvidenceValue() + throws Exception { + Map request = new HashMap<>(); + request.put("header.User-Agent", "request-ua"); + request.put("header.X-Test", "shared"); + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "evidence-ua"); + data.addEvidence(ShareUsageEvidence.HTTP_REQUEST_KEY, + Collections.unmodifiableMap(request)); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + + assertEquals("evidence-ua", found.get("HEADER.USER-AGENT")); + assertEquals("shared", found.get("header.x-test")); + assertFalse(found.containsKey(ShareUsageEvidence.HTTP_REQUEST_KEY)); + assertEquals(2, found.size()); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java new file mode 100644 index 000000000..d3d919e73 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java @@ -0,0 +1,187 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.testhelpers.data.MockFlowData; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.AbstractMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is called for every request, and most calls repeat one + * already shared, which the tracker does not share again. Such a call must + * not copy every value of the Evidence, or of the web request, more than + * once. These tests count how often all the values of each are read in a + * call that is not shared. + */ +public class ShareUsageEvidenceReadTests { + + /** + * Enough values that copying them is most of the cost of a call. + */ + private static final int VALUE_COUNT = 100; + + /** + * A map keyed without regard to case and holding no null value, as + * Evidence and the web request are, that counts how often all its + * values are read. + */ + private static final class CountingMap + extends AbstractMap { + + private final Map values = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private int reads; + + /** + * @return the reads since this was last called + */ + int takeReads() { + int result = reads; + reads = 0; + return result; + } + + @Override + public Set> entrySet() { + reads++; + return values.entrySet(); + } + + @Override + public Object get(Object key) { + return values.get(key); + } + + @Override + public boolean containsKey(Object key) { + return values.containsKey(key); + } + + @Override + public Object put(String key, Object value) { + return values.put(key, value); + } + + @Override + public int size() { + return values.size(); + } + } + + private static CountingMap headers() { + CountingMap headers = new CountingMap(); + for (int i = 0; i < VALUE_COUNT; i++) { + headers.put("header.x-test-" + i, "value-" + i); + } + return headers; + } + + /** + * An element with the default tracker, sharing every call that is not + * a repeat, and sending nothing until 50 calls are queued. + */ + private static ShareUsageElement createElement() throws Exception { + return new ShareUsageBuilder( + LoggerFactory.getILoggerFactory(), + mock(DataUploader.class)) + .build(); + } + + /** + * With no web request a call that is not shared reads the Evidence once, + * to make the session tracking key, and does not copy it. + */ + @Test + public void NoWebRequest_EvidenceReadOnceWhenNotShared() + throws Exception { + CountingMap evidence = headers(); + ShareUsageElement element = createElement(); + try { + FlowData data = MockFlowData.createFromEvidence(evidence, false); + element.process(data); + assertEquals(1, element.evidenceCollection.size(), + "The first call was not shared."); + + evidence.takeReads(); + element.process(data); + int reads = evidence.takeReads(); + assertEquals(1, element.evidenceCollection.size(), + "The repeat was shared."); + + assertEquals(1, reads, + "A call that is not shared read all the Evidence more than " + + "once, when the session tracking key needs it once."); + } finally { + element.close(); + } + } + + /** + * With a web request a call that is not shared reads the Evidence and + * the web request once each, to make the one copy that the session + * tracking key is made from. + */ + @Test + public void WebRequest_EachCopiedOnceWhenNotShared() throws Exception { + CountingMap request = headers(); + CountingMap evidence = new CountingMap(); + evidence.put("header.user-agent", "evidence-ua"); + evidence.put(ShareUsageEvidence.HTTP_REQUEST_KEY, request); + ShareUsageElement element = createElement(); + try { + FlowData data = MockFlowData.createFromEvidence(evidence, false); + element.process(data); + assertEquals(1, element.evidenceCollection.size(), + "The first call was not shared."); + + evidence.takeReads(); + request.takeReads(); + element.process(data); + int evidenceReads = evidence.takeReads(); + int requestReads = request.takeReads(); + assertEquals(1, element.evidenceCollection.size(), + "The repeat was shared."); + + assertEquals(1, requestReads, + "A call that is not shared copied the web request more " + + "than once."); + assertEquals(1, evidenceReads, + "A call that is not shared copied the Evidence more than " + + "once."); + } finally { + element.close(); + } + } +} From 4f7be9b31dce83515057cb83660ce5dfc2e83b6c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 03:54:26 +0000 Subject: [PATCH 94/95] Run the file system watcher on its own setting (#119) * Test that the file system watcher runs on its own setting The watcher is created with automatic updates off and the watcher on, and not with the watcher off. Tests that set automatic updates only to get a watcher no longer do. * Run the file system watcher on its own setting The data update service creates the watcher when the watcher is on, whether or not automatic updates are on. Watching a file downloads nothing, so it needs neither automatic updates nor a license key. Polling, the update on start-up and the license key check are unchanged. * Test that the watcher alone registers a data file and closes its watch service The file system watcher runs on its own setting and needs no license key. An engine whose update source needs a key it does not have is built with automatic updates and the update on start-up off, and its data file should still be registered to be watched. The builder registers a data file only for the two options that download, so the new builder test fails until it also registers one for the watcher. Each watch service runs a thread of its own. Unregistering a watched data file should close its watch service, and a download should resume the watch on the same watch service rather than leave another one running. The watcher's task repeats, and a repeating task that fails is never run again, so it should do nothing while the watch is paused for a download. * Register a data file for the watcher alone and close its watch service The file system watcher runs on its own setting and needs no license key. The builder registered a data file with the data update service only for automatic updates or the update on start-up, so an engine with no license key, which must turn both off, was never watched. It now also registers a data file that is to be watched, when there is a data update service and a data file on disk. The data update service keeps each data file's watch service, as each runs a thread of its own. Unregistering the data file closes it, and the watch resumes on it after a download rather than on a new one, unless the data file was unregistered during the download. Unregistering stops the watcher's task even while the watch is paused, and the task does nothing while the watch is paused rather than fail, as a repeating task that fails is never run again. The notes on setDataFileSystemWatcher no longer say that automatic updates must be on for the watcher to work. * Test that the watcher's task needs a watch key and replaces an earlier one The watcher's task polls the watch key. With no watch key, as when the data file's folder does not exist, no task should be scheduled, as the task would otherwise run on doing nothing now that it no longer fails on a missing key. Registering a data file again that has no watch key should replace the watcher's task rather than leave the earlier one running beside it. The thread tests now follow the watch service threads started during each test, so a thread from an earlier test that ends late makes no difference. * Schedule the watcher's task only with a watch key, replacing an earlier one The watcher's task polls the watch key, so it is scheduled only when the watch key was created. Before, a task was scheduled whatever happened, and failed on its first run when there was no watch key. Now that a run that finds no watch key does nothing, that task would run on for nothing. A data file registered again with no watch key gets a new task, and the earlier task is cancelled rather than left running beside it. --- .../DataFileConfigurationBuilderBase.java | 5 +- .../OnPremiseAspectEngineBuilderBase.java | 16 +- .../SingleFileAspectEngineBuilderBase.java | 5 +- .../services/DataUpdateServiceDefault.java | 156 ++++++-- .../OnPremiseAspectEngineBuilderBaseTest.java | 96 +++++ .../DataUpdateServiceLoggingTests.java | 3 +- .../services/DataUpdateServiceTests.java | 19 +- .../DataUpdateServiceWatchedFileTests.java | 365 +++++++++++++++++- 8 files changed, 605 insertions(+), 60 deletions(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java index 61589448b..aeb9a1b29 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java @@ -148,8 +148,9 @@ public TBuilder setAutoUpdate(boolean enabled) { * disk and automatically refresh the engine as soon as the file is updated. * This setting enables/disables that feature. * - * The {@link #setAutoUpdate(boolean)} feature must also be enabled in order - * for the file system watcher to work. + * The file system watcher runs on its own setting, whether + * {@link #setAutoUpdate(boolean)} is on or off, and needs no license key, + * as watching a file downloads nothing. * If the engine is built from a byte[] then this setting does nothing. * @param enabled the cache configuration to use * @return this builder instance diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java index a0995b9fb..879f55f44 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java @@ -201,13 +201,27 @@ protected void preCreateEngine() { } if (dataFileConfig.getAutomaticUpdatesEnabled() || - dataFileConfig.getUpdateOnStartup()) { + dataFileConfig.getUpdateOnStartup() || + isWatched(dataFileConfig)) { dataUpdateService.registerDataFile(dataFile); } dataFiles.add(dataFile); } } + /** + * Whether the data file is to be watched for a new file on disk. The + * file system watcher runs on its own setting and needs neither + * automatic updates nor a license key, only a data update service to run + * it and a data file on disk. + */ + private boolean isWatched(DataFileConfiguration dataFileConfig) { + return dataUpdateService != null && + dataFileConfig.getFileSystemWatcherEnabled() && + dataFileConfig.getDataFilePath() != null && + dataFileConfig.getDataFilePath().isEmpty() == false; + } + /** * Refuse a data file configuration that needs a data update service or * a license key it does not have. Every data file is checked before any diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java index 4a0cf8667..84df5a4f9 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java @@ -216,8 +216,9 @@ public TBuilder setAutoUpdate(boolean enabled) { * disk and automatically refresh the engine as soon as the file is updated. * This setting enables/disables that feature. *

        - * The {@link #setAutoUpdate(boolean)} feature must also be enabled in order - * for the file system watcher to work. + * The file system watcher runs on its own setting, whether + * {@link #setAutoUpdate(boolean)} is on or off, and needs no license key, + * as watching a file downloads nothing. *

        * If the engine is built from a byte[] then this setting does nothing. *

        diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index b21f1a317..398bfd9b9 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -92,6 +92,13 @@ public class DataUpdateServiceDefault implements DataUpdateService { long watchedFileOpenTimeoutMillis = WATCHED_FILE_OPEN_TIMEOUT_MILLIS; final List configurations = new ArrayList<>(); + /** + * The watch service of each watched data file, held under the + * configuration lock. A watch service runs a thread of its own, so it is + * closed when its data file is unregistered. + */ + private final Map watchServices = + new IdentityHashMap<>(); private final Logger logger; private final boolean closeFutureFactory; private final List onUpdateCompleteList = new ArrayList<>(); @@ -548,11 +555,10 @@ public void run() { } } - // If file system watcher is enabled then set it up. The - // automatic update switch governs the watcher as well as the - // polling timer. - if (aspectDataFile.getAutomaticUpdatesEnabled() && - aspectDataFile.getConfiguration().getFileSystemWatcherEnabled() && + // If file system watcher is enabled then set it up. The watcher + // runs on its own setting. Watching a file downloads nothing, so + // it needs neither automatic updates nor a license key. + if (aspectDataFile.getConfiguration().getFileSystemWatcherEnabled() && aspectDataFile.getConfiguration().getWatchKey() == null && aspectDataFile.getDataFilePath() != null && aspectDataFile.getDataFilePath().isEmpty() == false) { @@ -568,9 +574,8 @@ public void run() { throw new IllegalStateException("Cannot find directory to watch"); } logger.debug("Getting a watcher using {}", aspectDataFilePath); - WatchService watcher = aspectDataFilePath - .getFileSystem() - .newWatchService(); + WatchService watcher = + watchService(aspectDataFile, aspectDataFilePath); logger.debug("Setting a watcher for {}", aspectDataFilePath.getParent()); aspectDataFile.getConfiguration().setWatchKey(aspectDataFilePath.getParent().register( watcher, @@ -579,33 +584,49 @@ public void run() { logger.error("File watcher for '{}' could not be initialised. " + e.getMessage(), aspectDataFile.getDataFilePath(), e); } - try { - long pollingInterval = - aspectDataFile.getConfiguration().getPollingIntervalSeconds() * 1000L; - logger.debug("Setting poll future {} millis", pollingInterval); - aspectDataFile.setPollFuture(futureFactory.scheduleRepeating( - new Runnable() { - @Override - public void run() { - logger.debug("Watcher is running"); - for (WatchEvent event : - aspectDataFile.getConfiguration().getWatchKey().pollEvents()) { - if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { - if (((Path) event.context()).endsWith(aspectDataFilePath.getFileName())) { - dataFileUpdated(aspectDataFile); + // The watcher's task polls the watch key, so it is scheduled + // only when there is one, and replaces any task scheduled + // before rather than run beside it. + if (aspectDataFile.getConfiguration().getWatchKey() != null) { + try { + if (aspectDataFile.getPollFuture() != null) { + aspectDataFile.getPollFuture().cancel(false); + } + long pollingInterval = + aspectDataFile.getConfiguration().getPollingIntervalSeconds() * 1000L; + logger.debug("Setting poll future {} millis", pollingInterval); + aspectDataFile.setPollFuture(futureFactory.scheduleRepeating( + new Runnable() { + @Override + public void run() { + logger.debug("Watcher is running"); + // The watch is paused while a downloaded + // file is copied into place. A repeating + // task that fails never runs again, so + // this run does nothing. + WatchKey watchKey = + aspectDataFile.getConfiguration().getWatchKey(); + if (watchKey == null) { + return; + } + for (WatchEvent event : watchKey.pollEvents()) { + if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { + if (((Path) event.context()).endsWith(aspectDataFilePath.getFileName())) { + dataFileUpdated(aspectDataFile); + } } } + // make sure to reset the watch key or we are not called again + watchKey.reset(); } - // make sure to reset the watch key or we are not called again - aspectDataFile.getConfiguration().getWatchKey().reset(); - } - }, - pollingInterval)); - logger.debug("Poll check scheduled for {} mins", - aspectDataFile.getPollFuture().getDelay(TimeUnit.MINUTES)); - } catch (Exception e) { - logger.error("Could not schedule watcher for {}.", - aspectDataFile.getDataFilePath(), e); + }, + pollingInterval)); + logger.debug("Poll check scheduled for {} mins", + aspectDataFile.getPollFuture().getDelay(TimeUnit.MINUTES)); + } catch (Exception e) { + logger.error("Could not schedule watcher for {}.", + aspectDataFile.getDataFilePath(), e); + } } } @@ -630,11 +651,13 @@ public void unregisterDataFile(AspectEngineDataFile dataFile) { } AspectEngineDataFileDefault aspectDataFile = (AspectEngineDataFileDefault) dataFile; + // The watcher's task is stopped even while the watch is + // paused for a download. + if (aspectDataFile.getPollFuture() != null) { + aspectDataFile.getPollFuture().cancel(true); + aspectDataFile.setPollFuture(null); + } if (aspectDataFile.getConfiguration().getWatchKey() != null) { - if (aspectDataFile.getPollFuture() != null) { - aspectDataFile.getPollFuture().cancel(true); - aspectDataFile.setPollFuture(null); - } aspectDataFile.getConfiguration().getWatchKey().cancel(); aspectDataFile.getConfiguration().setWatchKey(null); } @@ -645,12 +668,57 @@ public void unregisterDataFile(AspectEngineDataFile dataFile) { } } } + closeWatchService(dataFile); } if (dataFile != null) { dataFile.setDataUpdateService(null); } } + /** + * Get the data file's watch service, creating it the first time. The + * data file keeps the one watch service until it is unregistered, so the + * watch resumes on it after a download. + * + * @param dataFile the data file to watch + * @param path the absolute path of the data file + * @return the data file's watch service + * @throws IOException if a watch service cannot be created + */ + private WatchService watchService( + AspectEngineDataFile dataFile, + Path path) throws IOException { + synchronized (configLock) { + WatchService watcher = watchServices.get(dataFile); + if (watcher == null) { + watcher = path.getFileSystem().newWatchService(); + watchServices.put(dataFile, watcher); + } + return watcher; + } + } + + /** + * Close the data file's watch service, if it has one, which ends the + * watch service's thread. + * + * @param dataFile the data file no longer watched + */ + private void closeWatchService(AspectEngineDataFile dataFile) { + WatchService watcher; + synchronized (configLock) { + watcher = watchServices.remove(dataFile); + } + if (watcher != null) { + try { + watcher.close(); + } catch (IOException e) { + logger.warn("Could not close the file watcher for '{}'", + dataFile.getDataFilePath(), e); + } + } + } + /** * Get the polling interval for a data file plus a random number of * seconds, no greater than the configured maximum randomisation. @@ -1137,14 +1205,20 @@ private AutoUpdateStatus checkForUpdateFromUrl( dataFile.getDataFilePath(), ex); result = AutoUpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME; } - // Make sure to enable the file watcher again if needed. + // Make sure to enable the file watcher again if needed. The + // watch resumes on the data file's own watch service, unless + // the data file was unregistered during the download. if (watchKey != null) { final Path filePath = Paths.get(dataFile.getDataFilePath()).toAbsolutePath(); - WatchService watcher = filePath.getFileSystem().newWatchService(); - watchKey = filePath.getParent().register(watcher, - StandardWatchEventKinds.ENTRY_MODIFY); - dataFile.getConfiguration().setWatchKey(watchKey); + synchronized (configLock) { + if (configurations.contains(dataFile)) { + watchKey = filePath.getParent().register( + watchService(dataFile, filePath), + StandardWatchEventKinds.ENTRY_MODIFY); + dataFile.getConfiguration().setWatchKey(watchKey); + } + } } } return result; diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java index 196e6b9d3..efbd8ade3 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java @@ -26,7 +26,11 @@ import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.configuration.DataFileConfiguration; import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectEngineDataFile; import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClient; +import fiftyone.pipeline.engines.services.update.FutureFactory; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -42,14 +46,18 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @SuppressWarnings("rawtypes") public class OnPremiseAspectEngineBuilderBaseTest { @@ -177,6 +185,94 @@ protected boolean isLicenseKeyRequired( verify(updateService, never()).registerDataFile(any()); } + /** + * The file system watcher runs on its own setting and needs no license + * key. An engine whose update source needs a key it does not have is + * built with automatic updates and the update on start-up off, and its + * data file is still registered to be watched. With the watcher off as + * well, nothing is registered. + */ + @Test + public void WatcherAlone_RegistersTheDataFile() throws Exception { + String[] names = { + "watcher on, automatic updates off, no license key", + "watcher off, automatic updates off, no license key"}; + boolean[] watching = {true, false}; + for (int i = 0; i < names.length; i++) { + checkWatcherAlone(names[i], watching[i]); + } + } + + private static void checkWatcherAlone(String name, boolean watching) + throws Exception { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + DataUpdateService updateService = new DataUpdateServiceDefault( + null, mock(HttpClient.class), null, futureFactory); + OnPremiseAspectEngineBuilderBase builder = + keyRequiredBuilder(updateService); + + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Data".getBytes()); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setIdentifier("Watched"); + config.setDataFilePath(dataFile.toString()); + config.setAutomaticUpdatesEnabled(false); + config.setUpdateOnStartup(false); + config.setFileSystemWatcherEnabled(watching); + builder.addDataFile(config); + + try { + builder.buildEngine(); + AspectEngineDataFile file = + (AspectEngineDataFile) builder.dataFiles.get(0); + assertEquals(name + ", the data file is registered", + watching, file.getIsRegistered()); + assertEquals(name + ", a file system watcher was created", + watching, config.getWatchKey() != null); + } + finally { + for (Object file : builder.dataFiles) { + updateService.unregisterDataFile((AspectEngineDataFile) file); + } + if (config.getWatchKey() != null) { + config.getWatchKey().cancel(); + } + directory.toFile().deleteOnExit(); + if (dataFile.toFile().delete() == false) { + dataFile.toFile().deleteOnExit(); + } + directory.toFile().delete(); + } + } + + /** + * A builder whose update source serves nothing without a license key. + */ + private static OnPremiseAspectEngineBuilderBase keyRequiredBuilder( + DataUpdateService updateService) { + return new OnPremiseAspectEngineBuilderBase(null, updateService) { + @Override + public OnPremiseAspectEngineBuilderBase setPerformanceProfile( + Constants.PerformanceProfiles profile) { + return this; + } + + @Override + protected AspectEngine newEngine(List properties) { + return null; + } + + @Override + protected boolean isLicenseKeyRequired( + DataFileConfiguration dataFileConfig) { + return true; + } + }; + } + private static DataFileConfiguration autoUpdateConfiguration( String identifier, String licenseKey) { diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java index 15a2f9a1b..dcd972ae1 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -279,7 +279,8 @@ public void UpdateOnStartup_AutoUpdateOff_StartsNoTimer() throws Exception { @Test public void FileSystemWatcher_LogsCreatingFileSystemWatcher() { - AspectEngineDataFileDefault file = newDataFile(false, true, false); + // The watcher needs no automatic updates. + AspectEngineDataFileDefault file = newDataFile(false, false, false); file.getConfiguration().setFileSystemWatcherEnabled(true); dataUpdate.registerDataFile(file); diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index a1650c1da..b10b06a8a 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -374,12 +374,9 @@ public void DataUpdateService_Register_FileSystemWatcher() throws IOException { dataUpdate.registerDataFile(file); // Assert - // The automatic update switch governs the file system watcher. - if (autoUpdateEnabled) { - assertNotNull(config.getWatchKey()); - } else { - assertNull(config.getWatchKey()); - } + // The watcher runs on its own setting, whether or not automatic + // updates are on. + assertNotNull(config.getWatchKey()); } finally { // Make sure we tidy up the temp file. File tempFile = new File(tempData); @@ -398,7 +395,8 @@ public void DataUpdateService_UpdateFromWatcher() throws IOException, Interrupte // Arrange OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); DataFileConfigurationDefault config = new DataFileConfigurationDefault(); - config.setAutomaticUpdatesEnabled(true); + // The watcher needs no automatic updates. + config.setAutomaticUpdatesEnabled(false); config.setFileSystemWatcherEnabled(true); config.setPollingIntervalSeconds(1); @@ -424,8 +422,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { }); // Configure a flag to be set when the watcher has applied the - // update. The polling timer's own check also completes, with no - // update found, and must not release the flag. + // update. final Semaphore completeFlag = new Semaphore(1); completeFlag.acquire(); dataUpdate.onUpdateComplete(new OnUpdateComplete() { @@ -1028,8 +1025,8 @@ public void DataUpdateService_CheckForUpdate_FileUpdated() { /** * A programmatic update compares the data file on disk whether or not - * Automatic Update from File is enabled, and with automatic update off - * no file system watcher exists whatever the watcher flag says. + * Automatic Update from File is enabled, here with automatic update off + * and the watcher flag at its default. */ @Test public void DataUpdateService_CheckForUpdate_FileUpdated_AutoUpdateOffWatcherFlagDefault() { diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java index b57a16ca9..b3e6d13c1 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java @@ -37,12 +37,20 @@ import org.mockito.Answers; import org.slf4j.Logger; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.WatchKey; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -53,6 +61,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; import static org.mockito.ArgumentMatchers.any; @@ -60,6 +69,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; @@ -100,6 +110,10 @@ public class DataUpdateServiceWatchedFileTests { "File watchers are not well implemented in OS X, " + "so don't run this unit test as it is unlikely to pass."; + private static final String POLLER_MESSAGE = + "The watch service on OS X polls from a thread this test cannot " + + "pick out."; + @Test public void WatchedFileRefresh_PollingCarriesOn() throws Exception { assumeFalse(MAC_MESSAGE, isMac()); @@ -151,6 +165,283 @@ public void WatchedFileRefresh_PollingCarriesOn() throws Exception { } } + /** + * The file system watcher runs on its own setting. Watching a file + * downloads nothing, so the watcher does not wait for automatic updates + * to be on, and with its own setting off a replaced file is not loaded. + * Polling runs only with automatic updates on. + */ + @Test + public void WatcherRunsOnItsOwnSetting() throws Exception { + String[] names = { + "watcher on, automatic updates on", + "watcher on, automatic updates off", + "watcher off, automatic updates on"}; + boolean[] automatic = {true, false, true}; + boolean[] watching = {true, true, false}; + for (int i = 0; i < names.length; i++) { + checkWatcherSetting(names[i], automatic[i], watching[i]); + } + } + + private static void checkWatcherSetting( + String name, + boolean automatic, + boolean watching) throws Exception { + final List timers = new ArrayList<>(); + final AtomicReference watcher = new AtomicReference<>(); + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new FileWrapperDefault((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory(timers, watcher)); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine, dataFile, automatic, watching); + + try { + dataUpdate.registerDataFile(file); + assertEquals(name + ", a file system watcher was created", + watching, file.getConfiguration().getWatchKey() != null); + assertEquals(name + ", the watcher's task was scheduled", + watching, watcher.get() != null); + assertEquals(name + ", polling timers set", + automatic ? 1 : 0, timers.size()); + + // macOS watchers are too slow to wait for. The test holds the + // polling timers, so only a watcher can load the replaced file. + if (isMac() == false) { + Files.write(dataFile, "New".getBytes()); + Runnable task = watcher.get(); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (task != null && refreshCount(engine) == 0 && + System.currentTimeMillis() < end) { + task.run(); + Thread.sleep(RETRY_MILLIS); + } + assertEquals(name + ", the replaced file was loaded", + watching ? 1 : 0, refreshCount(engine)); + } + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * Each watch service runs a thread of its own. Unregistering a watched + * data file, as closing its engine does, closes the watch service so its + * thread ends. + */ + @Test + public void Unregistering_ClosesTheWatchService() throws Exception { + assumeFalse(POLLER_MESSAGE, isMac()); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), new AtomicReference<>())); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + Set before = watchServiceThreads(); + dataUpdate.registerDataFile(file); + assertEquals("Registering starts a watch service thread", + 1, waitForThreadsStartedSince(before, 1).size()); + + dataUpdate.unregisterDataFile(file); + assertEquals("Unregistering ends the watch service thread", + 0, waitForThreadsStartedSince(before, 0).size()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A downloaded data file is copied into place with the watch paused. + * The watch then resumes on the data file's own watch service, so no + * other watch service is left running, and unregistering ends the one + * there is. + */ + @Test + public void DownloadedFile_WatchResumesOnTheSameWatchService() + throws Exception { + assumeFalse(POLLER_MESSAGE, isMac()); + final List timers = new ArrayList<>(); + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()).thenReturn( + new ByteArrayInputStream("New".getBytes())); + HttpClient httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + httpClient, + fileWrapperFactory(), + timerFactory(timers)); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + Path tempDirectory = Files.createTempDirectory("download"); + AspectEngineDataFileDefault file = + watchedFile(engine, dataFile, true, true); + file.setTempDataDirPath(tempDirectory.toString()); + file.getConfiguration().setDataUpdateUrl("http://www.test.com"); + file.getConfiguration().setDecompressContent(false); + file.getConfiguration().setVerifyMd5(false); + file.getConfiguration().setVerifyModifiedSince(false); + + try { + Set before = watchServiceThreads(); + dataUpdate.registerDataFile(file); + assertEquals("Registering starts a watch service thread", + 1, waitForThreadsStartedSince(before, 1).size()); + + // The polling timer downloads the new file. + timers.get(0).run(); + assertEquals("The downloaded file refreshes the engine", + 1, refreshCount(engine)); + assertNotNull("The watch resumes after the download", + file.getConfiguration().getWatchKey()); + assertEquals("The watch resumes on the same watch service", + 1, waitForThreadsStartedSince(before, 1).size()); + + dataUpdate.unregisterDataFile(file); + assertEquals("Unregistering ends the watch service thread", + 0, waitForThreadsStartedSince(before, 0).size()); + } + finally { + cleanUp(file, directory, dataFile); + tempDirectory.toFile().delete(); + } + } + + /** + * The watcher's task repeats, and a repeating task that fails is never + * run again. So while the watch is paused, as it is when a downloaded + * file is copied into place, the task does nothing and carries on. + * Unregistering the data file while the watch is paused still stops the + * task. + */ + @Test + public void WatchPaused_WatcherTaskCarriesOn() throws Exception { + final AtomicReference watcher = new AtomicReference<>(); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), watcher)); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + dataUpdate.registerDataFile(file); + WatchKey watchKey = file.getConfiguration().getWatchKey(); + assertNotNull("A file system watcher was created", watchKey); + ScheduledFuture task = file.getPollFuture(); + + file.getConfiguration().setWatchKey(null); + watcher.get().run(); + file.getConfiguration().setWatchKey(watchKey); + watcher.get().run(); + + file.getConfiguration().setWatchKey(null); + dataUpdate.unregisterDataFile(file); + verify(task).cancel(true); + watchKey.cancel(); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * The watcher's task polls the watch key, so with no watch key, here + * because the data file's folder does not exist, no task is scheduled. + */ + @Test + public void NoWatchKey_NoWatcherTask() throws Exception { + final AtomicReference watcher = new AtomicReference<>(); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), watcher)); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("missing").resolve("data.dat"); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + dataUpdate.registerDataFile(file); + assertNull("No watch key without the data file's folder", + file.getConfiguration().getWatchKey()); + assertNull("No watcher task without a watch key", + watcher.get()); + dataUpdate.unregisterDataFile(file); + } + finally { + directory.toFile().delete(); + } + } + + /** + * Registering a data file again that has no watch key, as it has while + * the watch is paused for a download, creates the watch key again and + * replaces the watcher's task rather than leave the old one running. + */ + @Test + public void RegisteredAgain_WatcherTaskReplaced() throws Exception { + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), new AtomicReference<>())); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + dataUpdate.registerDataFile(file); + ScheduledFuture first = file.getPollFuture(); + assertNotNull("The watcher's task was scheduled", first); + + file.getConfiguration().setWatchKey(null); + dataUpdate.registerDataFile(file); + assertNotNull("The watch key is created again", + file.getConfiguration().getWatchKey()); + verify(first).cancel(false); + dataUpdate.unregisterDataFile(file); + } + finally { + cleanUp(file, directory, dataFile); + } + } + /** * A watched file that cannot be opened is tried, with a pause between * tries, until the wait ends. The update then reports @@ -351,6 +642,67 @@ private static FutureFactory futureFactory( return futureFactory; } + /** + * Keeps each one-off timer's task for the test to run itself. + */ + private static FutureFactory timerFactory(final List timers) { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + timers.add((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + return futureFactory; + } + + private static FileWrapperFactory fileWrapperFactory() { + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new FileWrapperDefault((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + return fileWrapperFactory; + } + + /** + * The watch service threads running, found by the watch service's + * poller in each thread's stack. + */ + private static Set watchServiceThreads() { + Set threads = new HashSet<>(); + for (Map.Entry thread : + Thread.getAllStackTraces().entrySet()) { + if (Arrays.stream(thread.getValue()).anyMatch(frame -> + frame.getClassName().endsWith("WatchService$Poller"))) { + threads.add(thread.getKey()); + } + } + return threads; + } + + /** + * Wait for the watch service threads running that were not running + * before to number the count given, as a thread starts and ends a moment + * after its watch service is created or closed. Threads running before + * are left out, so one of those ending makes no difference. + */ + private static Set waitForThreadsStartedSince( + Set before, + int count) throws InterruptedException { + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + Set started = watchServiceThreads(); + started.removeAll(before); + while (started.size() != count && + System.currentTimeMillis() < end) { + Thread.sleep(RETRY_MILLIS); + started = watchServiceThreads(); + started.removeAll(before); + } + return started; + } + @SuppressWarnings("unchecked") private static OnPremiseAspectEngine engine() { @@ -361,9 +713,18 @@ private static AspectEngineDataFileDefault watchedFile( OnPremiseAspectEngine engine, Path dataFile) { + return watchedFile(engine, dataFile, true, true); + } + + private static AspectEngineDataFileDefault watchedFile( + OnPremiseAspectEngine engine, + Path dataFile, + boolean automatic, + boolean watching) { DataFileConfigurationDefault config = new DataFileConfigurationDefault(); - config.setAutomaticUpdatesEnabled(true); - config.setFileSystemWatcherEnabled(true); + config.setAutomaticUpdatesEnabled(automatic); + config.setFileSystemWatcherEnabled(watching); config.setPollingIntervalSeconds(1); config.setDataFilePath(dataFile.toString()); AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); From d99b7acf60e3f15a3c7f4f7b277a63c04dfb22eb Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:13:37 +0100 Subject: [PATCH 95/95] Update submodules (#121) Co-authored-by: Automation51D <51DCI@51Degrees.com> --- cxx | 2 +- .../resources/fiftyone/pipeline/javascriptbuilder/templates | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cxx b/cxx index 7c4ac436e..7992af701 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 7c4ac436e47cd15beed304cad2b8151d50a1efd1 +Subproject commit 7992af7010b7654c115510938a2f7626204bf936 diff --git a/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates b/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates index c773ce221..41e7561f8 160000 --- a/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates +++ b/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates @@ -1 +1 @@ -Subproject commit c773ce221ccec6fcb67fbdd5ad2175a66fab11d6 +Subproject commit 41e7561f82dd4a3a498e6b16d81d2c5cdbcdda8a