From 8291a3b662813018ede3ad139c5c3b27fcc01566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Kr=C3=BCger?= Date: Wed, 26 Nov 2025 15:01:27 +0100 Subject: [PATCH] feat: rewrite CivitAI and HuggingFace download scripts with curl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete rewrite of both model download scripts with: - Beautiful colorful CLI output with progress indicators - Pure bash/curl downloads (no Python dependencies for downloading) - yq-based YAML parsing (consistent with arty.sh) - Three commands: download, link, verify - Filtering by --category and --repo-id (comma-separated) - --dry-run mode for previewing operations - Respects format field for file extensions (.safetensors, .pt, etc.) - Uses type field for output subdirectories (checkpoints, embeddings, loras) ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- artifact_civitai_download.sh | Bin 22536 -> 17843 bytes artifact_huggingface_download.sh | 1924 ++++++++---------------------- 2 files changed, 513 insertions(+), 1411 deletions(-) diff --git a/artifact_civitai_download.sh b/artifact_civitai_download.sh index 7fea697facfc91a998730094707852da4616d053..14d23bfbd804478bd70854dee03b42771f7b2a2d 100755 GIT binary patch literal 17843 zcmdU1UvC>pa)0Ni=pE4(wd)Zjdy@_1)!}H0k_k(qKuLBC+eeJZp)@CQhMpNp=E?-_ zAxQF;T(Xw{hXBcA9`XSa7F5}KehxUI^dRUc2!q(RoCBcFMU%z z3Bu}$mzc~kJP`HWrZ|eC0U-38 zL;!3^V2S9*(J({hg+0-YeJ}MBkxYhzAUsb*8i{C}j>f6z1+m{vqd4#rrrVA<#p3emuT{WPES#@nw*Ju_14~AZGY>7c-J~OYPR+dKS=tc;SdWZ8=y=G4S|mt zotj2a(Dk}!en!AVRC<2j8xPVI1|~uYkpAjSzagI)l;E?ROc?yakAwb1g1?rZGwy@1 z$UR^H)f%Ni6e{A~?i1m<-6-q_r((Og+mN`38Xg-{$W8o_2M!Feu|JC3peLGJ>ra|n3LaPt3YyWuj71oroo8+u zo%^9UYQ1XgYmT*2*AALcJp3#f4rzaT@x+bCp`ObBJv@sp#pRioipyv`=!p|3Nfi1L z>WrYy{J|&(f-ek3KR}eF#7~9mk0UV(Mo_9?Py+PkJ^le*y|vqFi-X#3622qT2b-zoO+Kt1;(dP2|wP(-Xtqqrnx9uqQPvdBe!>$%>H@5VY=g-zL zYp304?3XpG;ZE^Pwc@qV-3+G>dU%UZiu zKWelO?|Y3>{bi&63S72dZ92dF)89L#db@Ram<{}+Q);*Bub^1Y|NQeGf9I68T5tAu zTeU6B{24{?#ef+U?exPP@7D5^MbOmw)F#Hh=)5$Bey0?qbuy4ofe&>1a=gU2VVZw2${U``#e&OR&_9ofd`w0ihF2yK&I! zG`Fk~P`M>NW9BU3tzmI`ehS|$+T-vb>WL^8gQ)8bNy|`E9O;6Arwxzuuef1 zB#+on$8jju7?P-`L6)W<%oJWEn-w0VUm!HKVfa-_CO3Q#N56$;BvrW9QPisvwbnFC z%8P&T0t;+KQq+fzX?g^rDA1w5ChvazVd^}Gr_QKUJJ828J!=l@M7ZQs#s>6V|MAFX zxpIooLpZ-F?Bk}B!~FDD)t-M*4ab9l_$<1JZQb7T=CW}6PhAY82%>vp*;&p$JIjvv zOh1hMm5gXw+7ca8uz+fsIzv)fLget##GHg_fc_w9`62trfzI>+?A~r`X9uQQY#;B} zIbgqc+c%1XFzuXpVd%%%DNvI2KczTb>rw~h*k#a5&o+OMDsQHB27b6%zFJBFGUeTo zzm2?JF9=Ve3>8sfSQtQ>L;x$IDxQiJrKNuNEE2G!w6*-%^+{9h$YDxfwXd0$D@=P| zEI(RXP2hNzMY)prAH}*T^HM7-`T18Ifhn7~Xo+9`@OumKw+WCcCdBeGBSe5h@d}7x zZi}-k8J`-?;&{bEq9T_mt#~Wa^4xqq_t7<_7V?5UMn*VI3AiE7(sYz;RI8^!dNw{;1wGYaaE?4B z_Rgx4j|STR6r^IERRi%3$2L1C&#+{`#yCtx)K_J}(y5W6FB#A3P;%wX7$wl^ zU!cyiyb47`9B`VREtC9tnj>-%;b$2OIBQB9hz4FspwJYAJ^z!nct7w5y&D#nBr#;s zA~Lu!iE<_g9m_O`KjHXl@}1RpWybSiwalQRS{3h|_YN!bEHlz@EQz*1j4pifaU4-` z7iuwrA)~~GgvM6UDf3!%P7@!|Z&Lk@Q>{Kvs!q9DU6#VAlYSAj+>N1^LgP?G=F(wO zQr{^DK`GHVPsx<32^x_oL8)q^+!&?@`4}&h6(S`<{>=)w3mNg8&|9v+WT`Bg*rmVb z_TlF8lV!x(NYmWdqoQXbe*E!A=tL1OuS-Cb(_o%TX9!|8lc9O|1tL?siZ8UMpS<`q zSrK9hRxlW%W*I;blx}C(2~u(gwN#tpz(A0e5{_I}PLuVYBa$fg7DM#DF0XxsKbQb(8mu~YPsnu!Xieb(m{zu zHgh^74>>L-XC*cYe{paa3ze1#w{+1ms1;5dDwYK3$W16lv*Itaq>hKI$gQKn(d#01 z(Z(Tg!@Qy{zt%PrP7`x1Q1y_CJJwdvK^Bi67iirwdqqgB`E!?(oh+KEauREovUp!@ zincZ|VcviY`a~d*NYaG2OgRiHEm=6`SR2W2A`?I1)D$YMNPdr*Hv%Df&|h{^=!>y{ zTdG|}uF@jJDXX$B8wN7kvnEzjB7-Q*c2NZ05O27_rK!Qjr$(fUIvRYTs{Uqsy`_4);u6suYG?7l(t3Q`bac3<>cJfEd844JY1j4Q8+A@E zMd7mDs9s$~#A&Tc3nB zbhN$EXb`~X;&4=dn?zx8b`9N2w84Mk85`8ii_tlSs^%-g`BhwD(s8g=gH8G*C(KlZh!wCpI8<7q2rzF_U{NVw(PGmTlON=cTGlEcA|kCiL% z*pAXVq1rs%>jB?10-jHN2wKGLvjZ6?0i;$Z>6S+@AXU^j?yV~Cx_?sQ>ArohL zC5u|iD;b5_EHK_L%b7{!v%c&bsi4-{3%!{t8VW~JPMMm|NqwLC|JOUT zEDu+7J&Z%+yE`}901~tPWK1I}RFm;EQ7dgqzAB8|@*-Ze{ROd)AGwBauHeWZ|&F!}jXgCoxrP*MwNWQ$Vxz`YD>}W2YT7+q)%Vd~7iRr?g z6-j|_LZjKw^ukJyTt+R!poO*5|S*8u@Wmrl!74!VZLl~UBT615# zX|-Q%z0jm9KI(DCc z>Cez`TO0<$LZfTXw?(4oTPO^Zw?#s^goQw*5w_-CvlVi-1#a#9Z3(bIfGFG3B~e2$ z02jhki$m@-7wW$eTr2kz-S2>4v{3Ld+m=F?jW8ri;n0>7NIUKV4U`~$;;LyGYZ5ar4jVcw^aaU2YvOo15OaVyu|05Hb27B)E^S zP8tn&sZ;2brG;^DuCqO*Uv*+`w3kNdPyZsV957gWYUwL}P-4d?TX<9_L2rwQG5raN zT8QV*b5xLzn5BR{2di*?b)(g^LwfZ)EC=!tEzT||$LBFTcgY+J#F@pFpLkuPz0xFO z2WdNS%l)HRo6R){gE5t)l2ihvI#OPt^vz6~Pk+jg0ziJs%~rmjTTD!^t&fTEsa53J#PbBH?P}>9_!#dPtSY&VhcZs5W(RZ-@&t=n zA+6+bs<_sZxIxT!L+Cj{)@8uiM=wzx{%o{*eBv@({tEDsE!CrarZxlyT6*+aPr}Uh1IT^e}b9BJ>RKjMA4H zpSxLI|+5_S!D0uy_Zgt`f@ji)e5oBjN3K5?0#h+1x zTeT#DufxwrNGZ3CmzSNf2hc4|FG}uw)Neroo0#j^Q>Mdg;mvF*%X1x8UBZOv!u^+v zz`qm`?t^h3=N43JMi&SM8>il=L8n{W)Cd@Qfw|tQOBje8aCHg{AxNTv04{7vm%Su* z>D2&V3!iyAK-2T@2Q<)DkcG=1yjonk!#AQ$B!mk_}F=M{k;3fqYQe5||hqAiU@s;#n)4 zvA)-;eee4)^Y@D2*WMuL;pR*}gS0k8T{z}`AKq$aDW=7h`xExd&TeM|Cb_w)*M!cY z!6fVJ`fcZ`I7SHUcB}ersENk&7u~GoRc-(42%rkFF7Vr!cpcKCIm=6?I3P;U7?f;T z4GRm8l>`v7A|&~D8&I3&>%m(5bk%}07WsNW74#4!hifKhQBSHYaNYoY49R!!TOZ4W zS&|?v;D^id29G}bb1EA0I7r3GcYxD+WTv!f@pmfk*CJKTjez z&Tf-lcG;o9Vk~s0ae{W1P}^W`Y1(c3apK2Suge!K^SCS(A_7&=ZD`FjiSWPj`G$(T za9!}~8*3YXHbqBYA8Wac&UTd*Qt3C{kHg-ul=RJ$;%z;gS9AMB?pR&jrc+sbrYii@ z6Y5bwVJQspqZcY%E1R=U8zCKT(=1vhkHA_*spWyJ(7}$=1x#GbQ5SdlXO*(6+8^5} zsh!$dZ}T%^9aItFcFI+B2${9V8(@lEU^L5D+2F(6g08i9j`#397!RTOCiK16M{%hV zeKN?ayksSn>X{kWBg*S`A;Rr`)}}e1lv~CQ1y$AHrY)tvj*RC$zz!|(-oRc4b%n^*oXz-scHJmP{mxjW_%N}!^gJZ2T% zZ02rzWE!wY{@PJ|rWp6L3`+BOv!}lLm7CTg)~rQ%-80ZhiBld~R(#08v2~WUaE;t5 z61m^zapVG;dcjBkvWM^z72R>Hm1ElW!{%}I0v}8?CIq)z2FK<-g`v3p{ literal 22536 zcmcg!&2k$_a<)&7z{Lk2e649w18{~QNJ+Eaa2m9ie@!fCWFEi_}n;=Kha+eb`gRZX1%D>9W%*yIVUo|e`Y2zZy zt}2fzkJN7bKF(Vmb(jpJvD!|!!s$>A(kRTMOl6D7IG(=CRGz5aWHMTuchoRWqd}geag;G0=UI3e z?WoO0m}16okoO0KSs3?+8mG=@@wy zrIUCXf}_beNi$VDnWwXH)HnqB(>zpXSJ5N_o)3!=;PoO9Ug@<@+h@({o2{o$UvEvS zMEZV`Mwe+ahrk4<%JXh(_eay|`=`{e*WPmt+h{oGwcAIo=?R)%wY%Npm#*z8+MajM z+pgsqS`J$W?W41nYy1w4yRTXvuwS9&<%`alH}db$1YDmd<6-6bad)rpc_tgD!t>~AF=d;=D!4$6Ra;R13dh1P}w_vdDJ~_?KOkH`s<)_dV=xY_GvSy1(p3y zk2*aDur7u-ga1R1eduSqhvxqYD$LoS3Gk<9Ewr5km6xsFQRnEO8T<$Oc8>Oso564J ztJUirzwGxq2QPrbztgPe&%5nDH#dX-3ZSmP>v!e#U&zyocI&5C&EPNPNe1u~tsY?f z@Sj9S=ZHx9Z%pve>?9+4wBI>6@3qc4$43th9`H^FX*|pAPPC|;?)Ew-XZ^hnIJs6E z3>E&1eQFv`BK*90-a38JKRrJ0?Y3WUys%YW|erP zUcXlL5h3;B_^`cfoTSO`v7Kg(bUvLW!-nk0W((b1gp+Zg-oE`_5 z2Un3(Dwv&NltVVEH=mv)LzSdzo{2QaMv~C9#&wdu%VyyqDhUcZw+xJ?@8dL?PT(d8 ziY*u%t*ke@=3%GqAZc+lv?sL;c--;%*~z)l@*DS57^&Sf4FMx9XCA386mhIR6&<5n zn*(CZC*xv<-9n_qZE^f6+AnMdXwXs*>R(`cstsnkxBt%CJY4d!r-ht*~k#^p&}eN;K(fT5}eRr@=r z2I?cdn8QnjEIx~Z)yC?NqvSrAe`<=nrS``VaSP0!5$`Z=JP_&@(idTm74x9p7+k=! zPPcRRO6{K??UDyU9w4GCc!8JkK)ncOvuK*B7f~8L^xS}VlE%|qjpow+6NrYd0Kp>fPrHrc+fnfu>ca0Japo097Wn zh~#~vR@3;$;`tzK2`bBq)`<*W16;#4NSg%Bp)MWyA*N>OuUfGx>L z`qx_|0wY+5@_aCevdp4v(bLFr^a^>oPzi`_uK*OKY2ubc84SW=0%8NK0uc4vub((3 zUWe%vi{nm=Bz6Wg9XAy~IWX~bl-vo!vhIfBtveuOc{H;SWGxToX$o(yyu|Y)565VH z;;(yc3!biCv=rQsPv>q#8!-;Dzx0 zwTRPLLGD9P-v6s~5!M zr^CkpVAX>F)*cTVkB4sE!;zd>RtI>}tfrsLWmYFBK$s%pdDS}Xs#+FLD5~ZEQYgbA z?RsTKUHTNKmQ;b7`V<_OR|$gQ=rTzcvQUKlMShh`pJK}k*6cXOOZN@b4?nbzPY9C= zFi6r|Q9tuLTV$1r9jKZ}TxXkMdinnK_S=e?&u@P6wo=K{#SYKTTYLhiqGmyCsA^hW zS0Qra(uZ_T;(makZkt&+iu#mtuZ{Hn43FB))l^l49porvRR_W}0f<1`yo~ai>$#zB zmZIP^A!|h~upwgeW@{O2iVK2`Tc8UBj?wzac~pVA=ax!7!31yBb-jE%=q`@D2YfQm zXY+go{9=^13O>?et!}s7g-0UkveSF!fi{I$(qnsekb$1<6<8y;)JE3pu!DHG-EF

PB%QUQS65oeNHIF(}L`n!+mcU%WxBsD|S;S zdB<@BNaS3^;FDlGp!)pADAVieEo9!l)|f5cvaS;vL=AUTm`*@Akko<6Y|1N9FKQvp zsygMQj_;#1gQL@rhfrvSB;8&6ia~;7d5CFUF<=;FobaP;TVWdS#fo7WG6Ma}i{cR7 zi0)fOQn1azs3DI~>~EVfVn0NZsW?io&KI+&IE-2dB_vJ;?`8=Cg{;InS=&{t=#Mws z`1c9^eTsjd;otA@@2{3sv(kzA8y>GWgw^2s22xa(HycV5x1lW7rYqs2mpf|fOF z+BBP4Pk5tIPq{zr^;cNRd;qEH@6}B2LSO$@HHOjq#&kX&dx=+?!Uz6svli$ffsjlY zF6w1w!#U;ZiQ?H)^*OVk{~c4L(a&=n$PSgJ40*_RaGx?auc4|wP=U8~2I|Rg@1jz8 zt{rd{oGjP_piEjk&ETBE=h`qo!RpP;P4R|^cF_FPyaPvRt5o%$86CqO=KBh?h?XeJ zv(G@y4B<52XCi0E-bN3ZJA~oBuwVbfTSVlSx!Njb7+AeTNE4{>IDrk+z1$Zih^OcR_XsG?PpOF3>Vw=+&lRlcLyw}Zr@ip#R zQ+hP5s$KtRLoX|wx4`2^ijiX;vtSrs!IMHPtV~%*L~Fr!Mz|+O|2AgPaCHK53ZKtA z(*m0zbkbYq_j8+1jzva>G6n8&4H>9%?gBoctW8^s4o4F{z|f0F@XW@ z#aZ4w9vcXpP0u^ncH!BRi^5q`Hj4(a2%pB7RRyC{%6=!)@j_VxLA$=Gu~1ax!37rD z{8@Ve%_nhitww;vo(jwu&T1ope7gr{?r$#(EniBJmoQr)V@JIaT!|Q+$KO;Z<0wQ> z0{53rgyF$Z7E~k)3cYgz&nP+QqN@iOyh!G$^3ND{)K#9(vYm$UmN&ti29A8vMmQKi zVRE--3E-!zh_2KzlgVnF!)nuFk&wK^p`ERr?X4vYs|n;}YERb1!z$KJVfe#}537z& zQ*8)xujsTPpyDAiCLEKv12rPnWgaZV(t04y(% z>J>{-Q&fccY0#XkjyZU8jJR7^RQdGT>&!R8<*ACOx=^pgbl}Ra*`BW0QXzcF=_SH_ zc5n*?VijW$%prB3``FDzyfE@v%Yna(3d%V$&x^$?ohlPR+NVYHGreyJy^`fp!4W1R zD!r7;+6~QKBC$B?Ckb3sXOG4@#PqgY?7ZiEZ0hYHyR!Q!;tq?E%~EPQmIu$>l4eD4 zb$K{mre!0Z9KW+pl1_h^5ZdHD5|k83HU!?b`I*O)$ow`wPK8VSs7hfRUr-K*GAJih zw1D&oyl;x^5y4ckS)23dWwjE{;y#_g%8Q~w#%sqm>EWL7l0+cQEDSCJ+vT#tvR$YU z9yN=``X(>^8d3oW$g<`c4lEJTe2BBWpSSf4?sT<<3o@EI)0lV!mqY&keIF%sM zA1#~BY#brk??EWe3|Jt39!Z!-qbvbw7$Xk~iS~sHCi0|Cj}TMmDN1A6P2rUxCAV5> z_F@Px1Nw-$g%Orsnh|=1_365hh(+3)1-2O-^uSRL!b?0rSXD~N5(=mlkZe;Keox#YP=KD6%o;#U5gAd$%%T+OHN z*x(s8jEvEoD#pYwg5Gb1f>-Hwy)7}`&EwoER-r3w9EgwU@ zu;(?a5FaUZLCq{=tz#=xk;~$uYpS%zf?#`mOP%i2&|BK6^M-cck!KjQY_lgRomDAC zpVl@V8iU5@-gN^JnBDRR!|=6K_J~kHJdbSb$%+quB9I6K`8V&2gP%Jq$m<&Wp1Eiv=fbxI&Tz)U9NY%#~7v zTA-or<472b8uvJiTCM`-1E8F(^*4DM@ShT6e~%iJpYP6tWhQH7)ECqKvU^UP2etEb zs?E-dU`%HQZj%D|>}0z5T!XS%HAy^+`>8Lruqg^9z^PE+^?>n=I~I^|hS|sBMOvCp z=T*?fjHF$8La~C2i@g{PwEzwfRR)-<4F-pOP3RH3tFrlpcyKscCEJMP$S{erX*HLv zjt5ChmE&kM;?q2!=M|FD9Nvt=PLaCK1GteZ5&=SBDG6Y4Dw62{mEtr~1TXe?{mL1T z{TMkc##Z)qa%MPAuZ@%B9ZT2uA-Gc}%TKrP#A|3*bH zTHz6mHFR%29(sL6j!tL)lw8hmsOqW8kTx5@(Z!4Jp8=5Yu-xKB1L+RcBzYgnLZDS=Y=Ep2arO#B zG&x-$WGB3X7@8rSNOs!wEkLuC_jDb>^|_rhC0O()OUbIjK8c~|xT+duM3JYFD|+j+ zygd2^jQcM+R~)&xCYL%kZPuh{pamMnQ&rDO@+nYL=v_2+Yd3V?Wy$L1XFV;#&bAp` z=>NnitFx%v@t?m%Re+sQrvr!B*pVsu0ciOvO_V_-%G@_F??2G78(Ejp0Dt}YmuygK z**aUAKhXw96N=%|vwX@nhhilv2f98T;U+CGDD$G{t@N{&&4Eu^jh^2nw~oj4ClN<`4<&(J@j;vCBxdSg_Ae$yYk z0Glwx2Aj~`E(EECF~R{OB&@+)NkOC*le^ioX0`_;I*Iqcw3i#d-=rzmZpzDGDRz;X zCC%y~Stzj(azHvIE5V^MAg6iMQ}2!Xr=?c=ekJXo4d;_JgF2HnC4cvmPlX8&OlsJ7 zhg8wyE8hM;QD@kk_GzWwr6)A+-MT_=Jw}xteNDO_cb5VT#%<;ht>TCug z8EsE4w5gV}h(f2w(#aEoTqsh3$-Ush@8JnPN72?EmzCYE{eo_iFOAe3uZbbvAe(CV zg6kp$P_{~*KRU2HBS*)6V+3Bl!N6#QQ6FKu6N}y3wAUpXX)#_o%|7dsn}QJ2{1C~a!ePk-fP;mzTvY1}cXA@hE;kEI4+1y~=dm;CuOuzOldJuHb8_|x35=g(S0 zmlM3vhhpCK0@gpvau201P!fg(GfBBZUkrDp4ehk{CX~?>^GSuJq6#?elRL#R_8gzN z885y9UADhro-uC)n|pgc%Orc1=rlp@*aQKrYkul;Y@Lpo5q6t% zjt_h_@(|x`u=gS#{PyD{jLVnSoV{bQx<^c3EHxT(%jTu>i0>UE@&@c9c`4{)O2 z$2jXp+Qloj^XlS^_aggW2~YP3UAiZ(cl~foTBgu@dD{ao6UYnNV;3~V#P_~%o&S6% zDPq^G_frVKeVgYpttKl&CpiX%1ykcfRvWKssuqQ?Q$l(33gu+tJ#lP{@|hXFxy2tr zpxRDcELQwY3Z3yLaTkQJF!>oqfnFQH<^_bpivxb9PU+CSwG4}+nl<_j)wrZ@3+U%J z7jJ5tZ`K=c=5My~Ld%K3mQzmTw_14ax0%(;+~OkmOcymHXH9iEHO?6!isid#Tm!p7 zb)40qn!*bum83EYWV8qQ9L~seKDod+mnSTX4ena0$vWSj|E%%F`qJ1rOJ`hHqEz^6 zd`0G~c_RWPk!4zj%Dd;r=>30c3*Cd{+cBUVAqrsQHdau;UF+LOY;M_;{1;jvs=694`3n@>`#&>I0ArTN`TV zYuXc>%rn|?=_qW0Z$emu{+4}R)WGew={tgIN9^w&w_Z1 zN8BLaPufv;-e+A+^J-vSI$hXN|J*2k&uBG>dfhEdN1_hm{=_6=9ngn1b&2X~{C/dev/null | cut -d'=' -f2- | tr -d '"' | tr -d "'" || true) + if [[ -n "$token" ]]; then + HF_TOKEN="$token" + return 0 + fi + fi + done +} # ============================================================================ -# UTILITY FUNCTIONS - The Magic Happens Here +# LOGGING FUNCTIONS # ============================================================================ -# Print functions with beautiful formatting print_banner() { local text="$1" - local width=80 - local padding=$(( (width - ${#text} - 2) / 2 )) + local width=70 + local text_len=${#text} + local padding=$(( (width - text_len) / 2 )) - echo -e "" - echo -e "${BOLD_CYAN}${BOX_DOUBLE}$(printf '%.0s'"${BOX_DOUBLE}" $(seq 1 $width))${BOX_DOUBLE}${RESET}" - echo -e "${BOLD_CYAN}${BOX_DOUBLE}$(printf '%.0s ' $(seq 1 $padding))${BOLD_MAGENTA}${text}$(printf '%.0s ' $(seq 1 $padding))${BOLD_CYAN}${BOX_DOUBLE}${RESET}" - echo -e "${BOLD_CYAN}${BOX_DOUBLE}$(printf '%.0s'"${BOX_DOUBLE}" $(seq 1 $width))${BOX_DOUBLE}${RESET}" - echo -e "" + echo "" + echo -e "${BOLD_CYAN}${BOX_DOUBLE}$(printf '%0.sโ•' $(seq 1 $width))${BOX_DOUBLE}${RESET}" + echo -e "${BOLD_CYAN}โ•‘$(printf '%*s' $padding '')${BOLD_MAGENTA}${text}$(printf '%*s' $((width - padding - text_len)) '')${BOLD_CYAN}โ•‘${RESET}" + echo -e "${BOLD_CYAN}${BOX_DOUBLE}$(printf '%0.sโ•' $(seq 1 $width))${BOX_DOUBLE}${RESET}" + echo "" } print_section() { local text="$1" - echo -e "\n${BOLD_YELLOW}${DOUBLE_ARROW} ${text}${RESET}" - echo -e "${CYAN}$(printf '%.0s'"${BOX_LIGHT}" $(seq 1 80))${RESET}" + echo -e "\n${BOLD_CYAN}ยป ${text}${RESET}" + echo -e "${CYAN}$(printf '%0.sโ”€' $(seq 1 70))${RESET}" } print_success() { @@ -203,1382 +160,555 @@ print_step() { local current="$1" local total="$2" local text="$3" - echo -e "${BOLD_BLUE}[${current}/${total}]${RESET} ${MAGENTA}${DOWNLOAD}${RESET} ${text}" + echo -e "${BOLD_BLUE}[${current}/${total}]${RESET} ${CYAN}${PACKAGE}${RESET} ${text}" } print_detail() { echo -e " ${DIM}${CYAN}${ARROW_RIGHT} $1${RESET}" } -# Progress bar function show_progress() { local current="$1" local total="$2" - local width=50 + local width=40 local percentage=$((current * 100 / total)) local filled=$((current * width / total)) local empty=$((width - filled)) printf "\r ${BOLD_CYAN}Progress: ${RESET}[" - printf "${BG_GREEN}${BOLD_WHITE}%${filled}s${RESET}" | tr ' ' 'โ–ˆ' + printf "${BG_CYAN}${BOLD_WHITE}%${filled}s${RESET}" | tr ' ' 'โ–ˆ' printf "${DIM}%${empty}s${RESET}" | tr ' ' 'โ–‘' printf "] ${BOLD_YELLOW}%3d%%${RESET} ${DIM}(%d/%d)${RESET}" "$percentage" "$current" "$total" } -# Parse YAML (simple implementation) -parse_yaml() { - local yaml_file="$1" - local category="$2" +# ============================================================================ +# YAML PARSING (using yq) +# ============================================================================ - python3 - "$yaml_file" "$category" < /dev/null; then - missing_deps+=("python3") - fi - - # Check pip - if ! command -v pip3 &> /dev/null; then - missing_deps+=("pip3") - fi - - # Check required Python packages - if ! python3 -c "import yaml" 2>/dev/null; then - print_warning "PyYAML not installed, installing..." - pip3 install pyyaml -q - fi - - if ! python3 -c "import huggingface_hub" 2>/dev/null; then - print_warning "huggingface_hub not installed, installing..." - pip3 install huggingface_hub -q - fi - - if [[ ${#missing_deps[@]} -gt 0 ]]; then - print_error "Missing dependencies: ${missing_deps[*]}" +check_yq() { + if ! command -v yq &>/dev/null; then + print_error "yq is not installed. Please install yq first." + print_info "Install: https://github.com/mikefarah/yq" exit 1 fi - - print_success "All dependencies satisfied" } -# Validate configuration -validate_config() { - print_section "Validating Configuration" +# Get total count of models +get_model_count() { + local config="$1" + yq eval '. | length' "$config" 2>/dev/null || echo "0" +} - # Show current command - print_info "Command: ${BOLD_CYAN}${COMMAND}${RESET}" +# Get model field at index +get_model_field() { + local config="$1" + local index="$2" + local field="$3" + local value + value=$(yq eval ".[$index].$field // \"\"" "$config" 2>/dev/null) + echo "$value" | sed 's/^"//;s/"$//' +} - if [[ -n "$CONFIG_FILE" ]]; then - if [[ ! -f "$CONFIG_FILE" ]]; then - print_error "Configuration file not found: $CONFIG_FILE" - exit 1 - fi - print_success "Configuration file found: ${CYAN}${CONFIG_FILE}${RESET}" - else - print_warning "No configuration file specified" - fi +# Get files array length for a model +get_files_count() { + local config="$1" + local index="$2" + yq eval ".[$index].files | length" "$config" 2>/dev/null || echo "0" +} - # HF_TOKEN only required for download and both commands - if [[ "$COMMAND" == "download" ]] || [[ "$COMMAND" == "both" ]]; then - if [[ -z "$HF_TOKEN" ]]; then - print_error "HF_TOKEN not set. Please set it in .env file or environment." - exit 1 - fi - print_success "HuggingFace token configured: ${DIM}${HF_TOKEN:0:10}...${RESET}" - elif [[ "$COMMAND" == "verify" ]]; then - print_info "Verify mode: HuggingFace token not required" - fi +# Get file mapping at index +get_file_field() { + local config="$1" + local model_index="$2" + local file_index="$3" + local field="$4" + local value + value=$(yq eval ".[$model_index].files[$file_index].$field // \"\"" "$config" 2>/dev/null) + echo "$value" | sed 's/^"//;s/"$//' +} - # Validate flag combinations - if [[ "$CLEANUP_MODE" == true ]] && [[ ! "$COMMAND" =~ ^(link|both)$ ]]; then - print_error "--cleanup can only be used with 'link' or 'both' commands" - exit 1 - fi +# Check if model matches filters +matches_filters() { + local repo_id="$1" + local category="$2" - # Cache directory - if [[ "$COMMAND" == "download" ]] || [[ "$COMMAND" == "both" ]]; then - if [[ ! -d "$CACHE_DIR" ]]; then - print_info "Creating cache directory: ${CYAN}${CACHE_DIR}${RESET}" - mkdir -p "$CACHE_DIR" - fi - print_success "Cache directory ready: ${CYAN}${CACHE_DIR}${RESET}" - else - # For link and verify commands, just show the directory - if [[ -d "$CACHE_DIR" ]]; then - print_success "Cache directory found: ${CYAN}${CACHE_DIR}${RESET}" - else - print_warning "Cache directory not found: ${CYAN}${CACHE_DIR}${RESET}" - fi - fi - - # ComfyUI directory - if [[ "$COMMAND" == "link" ]] || [[ "$COMMAND" == "both" ]] || [[ "$COMMAND" == "verify" ]]; then - if [[ -d "$COMFYUI_DIR" ]]; then - print_success "ComfyUI directory found: ${CYAN}${COMFYUI_DIR}${RESET}" - else - if [[ "$COMMAND" == "verify" ]]; then - print_warning "ComfyUI directory not found: ${CYAN}${COMFYUI_DIR}${RESET}" - else - print_info "ComfyUI directory: ${CYAN}${COMFYUI_DIR}${RESET}" + # Check category filter + if [[ -n "$CATEGORY_FILTER" ]]; then + local match=false + IFS=',' read -ra cats <<< "$CATEGORY_FILTER" + for cat in "${cats[@]}"; do + cat=$(echo "$cat" | xargs) + if [[ "$category" == "$cat" ]]; then + match=true + break fi - fi - fi -} - -# Find model files in HuggingFace cache -find_model_files() { - local repo_id="$1" - local filename_filter="$2" - - python3 - "$CACHE_DIR" "$repo_id" "$filename_filter" </dev/null | grep "/$filename_only$" | head -n1) - - if [[ -z "$source_file" ]] || [[ ! -f "$source_file" ]]; then - print_warning "Source file not found: ${source_pattern}" - continue - fi - - # Construct full link path with directory included in dest_path - local link_path="${COMFYUI_DIR}/${dest_path}" - local link_dir=$(dirname "$link_path") - - # Ensure directory exists - if [[ ! -d "$link_dir" ]]; then - mkdir -p "$link_dir" - fi - - # Remove existing symlink or file if it exists - if [[ -L "$link_path" ]]; then - rm -f "$link_path" - elif [[ -e "$link_path" ]]; then - print_warning "File already exists (not a symlink): ${dest_path}" - continue - fi - - # Create symlink - ln -s "$source_file" "$link_path" - print_detail "${LINK} Linked: ${DIM}${dest_path}${RESET}" - linked_count=$((linked_count+1)) - done <<< "$file_mappings" - else - # Fallback: use automatic prefixing for files without explicit mappings - print_detail "No file mappings found, using automatic prefixing" - - # Extract model name from repo_id for prefixing filenames - # e.g., "facebook/musicgen-medium" -> "musicgen-medium" - local model_name=$(echo "$repo_id" | sed 's/.*\///') - - while IFS= read -r source_file; do - if [[ -f "$source_file" ]]; then - local filename=$(basename "$source_file") - - # Add model name prefix to filename for better organization - # e.g., "pytorch_model.bin" -> "musicgen-medium-pytorch_model.bin" - local prefixed_filename="${model_name}-${filename}" - local link_path="${target_dir}/${prefixed_filename}" - - # Remove existing symlink or file if it exists - if [[ -L "$link_path" ]]; then - rm -f "$link_path" - elif [[ -e "$link_path" ]]; then - print_warning "File already exists (not a symlink): ${prefixed_filename}" - continue - fi - - # Create symlink - ln -s "$source_file" "$link_path" - print_detail "${LINK} Linked: ${DIM}${prefixed_filename}${RESET}" - linked_count=$((linked_count+1)) - fi - done <<< "$model_files" - fi - - if [[ $linked_count -gt 0 ]]; then - print_success "Linked ${linked_count} file(s) for ${BOLD_WHITE}${repo_id}${RESET}" - return 0 - else - print_error "Failed to link files for ${repo_id}" - return 1 - fi -} - -# Cleanup unused cache files that aren't symlinked -cleanup_unused_cache_files() { - local repo_id="$1" - local cache_dir="$2" - local comfyui_dir="$3" - local file_mappings="$4" - - # Find the latest snapshot directory for this repo - local repo_cache_dir="${cache_dir}/models--${repo_id//\//--}" - if [[ ! -d "$repo_cache_dir" ]]; then - print_warning "Cache directory not found for $repo_id" - return 1 - fi - - print_info "Analyzing cache for ${BOLD_WHITE}${repo_id}${RESET}..." - - # Use Python to clean up old snapshots AND non-whitelisted files in latest snapshot - local cleanup_result - cleanup_result=$(python3 - "$repo_cache_dir" "$comfyui_dir" "$file_mappings" <<'EOPYCLEANUP' -import os -import sys -import shutil -from pathlib import Path - -repo_cache = Path(sys.argv[1]) -comfyui_dir = Path(sys.argv[2]) -file_mappings_str = sys.argv[3] if len(sys.argv) > 3 else "" - -# Parse whitelist from file_mappings (format: "source|dest\nsource|dest\n...") -whitelist_sources = set() -if file_mappings_str: - for line in file_mappings_str.strip().split('\n'): - if '|' in line: - source = line.split('|')[0].strip() - if source: - whitelist_sources.add(source) - -# Essential HuggingFace metadata files to always preserve -ESSENTIAL_FILES = { - '.gitattributes', - 'README.md', - 'model_index.json', - '.huggingface', - 'config.json' -} - -# Find latest snapshot -snapshots_dir = repo_cache / 'snapshots' -if not snapshots_dir.exists(): - sys.exit(0) - -snapshots = sorted(snapshots_dir.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True) -if not snapshots: - sys.exit(0) - -latest_snapshot = snapshots[0] -old_snapshots = snapshots[1:] # All snapshots except the latest - -# Calculate size of old snapshot directories -old_snapshot_size = 0 -old_snapshot_paths = [] -for old_snap in old_snapshots: - try: - # Calculate size of old snapshot directory - for file_path in old_snap.rglob('*'): - if file_path.is_file(): - old_snapshot_size += file_path.stat().st_size - old_snapshot_paths.append(str(old_snap)) - except Exception: - pass - -# Find non-whitelisted files in latest snapshot -unwanted_files = [] -unwanted_size = 0 -if whitelist_sources: - for item in latest_snapshot.rglob('*'): - if item.is_file(): - # Get relative path from snapshot root - rel_path = str(item.relative_to(latest_snapshot)) - - # Check if this file is in whitelist or is essential - is_whitelisted = False - - # Check exact match first - if rel_path in whitelist_sources: - is_whitelisted = True - else: - # Check if any whitelist entry matches this file - # (handles cases where whitelist has paths like "split_files/diffusion_models/file.safetensors") - for whitelisted in whitelist_sources: - if rel_path == whitelisted or rel_path.endswith('/' + whitelisted): - is_whitelisted = True - break - - # Check if it's an essential file - if item.name in ESSENTIAL_FILES: - is_whitelisted = True - - # If not whitelisted, mark for deletion - if not is_whitelisted: - unwanted_files.append(str(item)) - unwanted_size += item.stat().st_size - -# Output results: old_snapshot_count|old_snapshot_size|unwanted_files_count|unwanted_size -print(f"{len(old_snapshot_paths)}|{old_snapshot_size}|{len(unwanted_files)}|{unwanted_size}") -for snap in old_snapshot_paths: - print(snap) -for unwanted_file in unwanted_files: - print(unwanted_file) -EOPYCLEANUP -) - - # Parse results - local first_line - first_line=$(echo "$cleanup_result" | head -n 1) - local snapshot_count - snapshot_count=$(echo "$first_line" | cut -d'|' -f1) - local snapshot_bytes - snapshot_bytes=$(echo "$first_line" | cut -d'|' -f2) - local unwanted_count - unwanted_count=$(echo "$first_line" | cut -d'|' -f3) - local unwanted_bytes - unwanted_bytes=$(echo "$first_line" | cut -d'|' -f4) - - # Check if there's anything to clean - if [[ "$snapshot_count" -eq 0 ]] && [[ "$unwanted_count" -eq 0 ]]; then - print_success "No cleanup needed - cache is optimal" - return 0 - fi - - # Convert bytes to MB - local snapshot_mb - snapshot_mb=$(echo "scale=2; $snapshot_bytes / 1048576" | bc) - local unwanted_mb - unwanted_mb=$(echo "scale=2; $unwanted_bytes / 1048576" | bc) - local total_mb - total_mb=$(echo "scale=2; ($snapshot_bytes + $unwanted_bytes) / 1048576" | bc) - - # Get list of items to delete (skip first line which is summary) - # First snapshot_count lines are old snapshots, remaining lines are unwanted files - local all_items - all_items=$(echo "$cleanup_result" | tail -n +2) - - local snapshots_to_delete - if [[ "$snapshot_count" -gt 0 ]]; then - snapshots_to_delete=$(echo "$all_items" | head -n "$snapshot_count") - else - snapshots_to_delete="" - fi - - local files_to_delete - if [[ "$unwanted_count" -gt 0 ]]; then - files_to_delete=$(echo "$all_items" | tail -n "$unwanted_count") - else - files_to_delete="" - fi - - if [[ "$DRY_RUN" == true ]]; then - if [[ "$snapshot_count" -gt 0 ]]; then - print_warning "DRY-RUN: Would clean up ${BOLD_YELLOW}${snapshot_count}${RESET} old snapshot(s) (~${snapshot_mb} MB)" - if [[ -n "$snapshots_to_delete" ]]; then - print_detail "Old snapshots that would be deleted:" - while IFS= read -r snapshot; do - local basename - basename=$(basename "$snapshot") - print_detail " ${CROSS_MARK} Would delete snapshot: ${DIM}${basename}${RESET}" - done <<< "$snapshots_to_delete" - fi - fi - - if [[ "$unwanted_count" -gt 0 ]]; then - print_warning "DRY-RUN: Would clean up ${BOLD_YELLOW}${unwanted_count}${RESET} non-whitelisted file(s) (~${unwanted_mb} MB)" - if [[ -n "$files_to_delete" ]]; then - print_detail "Non-whitelisted files that would be deleted (showing first 10):" - echo "$files_to_delete" | head -n 10 | while IFS= read -r file; do - local basename - basename=$(basename "$file") - print_detail " ${CROSS_MARK} Would delete file: ${DIM}${basename}${RESET}" - done - if [[ "$unwanted_count" -gt 10 ]]; then - print_detail " ${DIM}... and $((unwanted_count - 10)) more${RESET}" - fi - fi - fi - - print_info "Total space that would be freed: ~${total_mb} MB" - return 0 - fi - - # Actually delete items - local deleted_snapshots=0 - local deleted_files=0 - - # Delete old snapshot directories - if [[ "$snapshot_count" -gt 0 ]]; then - print_warning "Cleaning up ${BOLD_YELLOW}${snapshot_count}${RESET} old snapshot(s) (~${snapshot_mb} MB)..." - while IFS= read -r snapshot; do - if [[ -d "$snapshot" ]]; then - rm -rf "$snapshot" && deleted_snapshots=$((deleted_snapshots+1)) - fi - done <<< "$snapshots_to_delete" - fi - - # Delete non-whitelisted files - if [[ "$unwanted_count" -gt 0 ]]; then - print_warning "Cleaning up ${BOLD_YELLOW}${unwanted_count}${RESET} non-whitelisted file(s) (~${unwanted_mb} MB)..." - while IFS= read -r file; do - if [[ -f "$file" ]]; then - rm -f "$file" && deleted_files=$((deleted_files+1)) - fi - done <<< "$files_to_delete" - fi - - # Report results - local success=true - if [[ "$snapshot_count" -gt 0 ]] && [[ $deleted_snapshots -eq $snapshot_count ]]; then - print_success "Cleaned up ${deleted_snapshots} old snapshot(s), freed ~${snapshot_mb} MB" - elif [[ "$snapshot_count" -gt 0 ]]; then - print_warning "Cleaned up ${deleted_snapshots}/${snapshot_count} old snapshots" - success=false - fi - - if [[ "$unwanted_count" -gt 0 ]] && [[ $deleted_files -eq $unwanted_count ]]; then - print_success "Cleaned up ${deleted_files} non-whitelisted file(s), freed ~${unwanted_mb} MB" - elif [[ "$unwanted_count" -gt 0 ]]; then - print_warning "Cleaned up ${deleted_files}/${unwanted_count} non-whitelisted files" - success=false - fi - - if [[ "$snapshot_count" -gt 0 ]] || [[ "$unwanted_count" -gt 0 ]]; then - print_info "Total space freed: ~${total_mb} MB" - fi - - if $success; then - return 0 - else - return 1 - fi -} - -# ============================================================================ -# VERIFICATION FUNCTIONS - Model Status Checking -# ============================================================================ - -# Get actual disk usage for a model's files -get_model_disk_usage() { - local model_files="$1" - - if [[ -z "$model_files" ]]; then - echo "0" - return - fi - - local total_bytes=0 - while IFS= read -r file_path; do - if [[ -f "$file_path" ]] || [[ -L "$file_path" ]]; then - local file_size - # Use -L to follow symlinks (HuggingFace uses symlinks to blobs) - # Try Linux stat first, then macOS stat - if stat -L -c "%s" "$file_path" >/dev/null 2>&1; then - file_size=$(stat -L -c "%s" "$file_path" 2>/dev/null) - elif stat -L -f "%z" "$file_path" >/dev/null 2>&1; then - file_size=$(stat -L -f "%z" "$file_path" 2>/dev/null) - else - file_size=0 - fi - total_bytes=$((total_bytes + file_size)) - fi - done <<< "$model_files" - - echo "$total_bytes" -} - -# Format bytes to human-readable size -format_bytes() { - local bytes="$1" - - if (( bytes < 1024 )); then - echo "${bytes} B" - elif (( bytes < 1048576 )); then - echo "$(( bytes / 1024 )) KB" - elif (( bytes < 1073741824 )); then - printf "%.2f MB" "$(bc <<< "scale=2; $bytes / 1048576")" - else - printf "%.2f GB" "$(bc <<< "scale=2; $bytes / 1073741824")" - fi -} - -# Verify if model is downloaded -verify_model_download() { - local repo_id="$1" - local expected_size_gb="$2" - local filename_filter="$3" - - # Find model files in cache - # Capture both stdout (file paths) and stderr (error messages) - local find_output - find_output=$(find_model_files "$repo_id" "$filename_filter" 2>&1) - - # Separate file paths from error/debug messages - local model_files - model_files=$(echo "$find_output" | grep -v "^ERROR:" | grep -v "^WARN:" | grep -v "^DEBUG:") - - # Extract error messages for logging - local error_msgs - error_msgs=$(echo "$find_output" | grep "^ERROR:\|^WARN:\|^DEBUG:") - - if [[ -z "$model_files" ]]; then - # Log error messages to stderr if they exist - if [[ -n "$error_msgs" ]]; then - echo "$error_msgs" >&2 - fi - echo "NOT_FOUND|0|0|" - return 1 - fi - - # Count files - local file_count - file_count=$(echo "$model_files" | wc -l | tr -d ' ') - - # Get actual size - local actual_bytes - actual_bytes=$(get_model_disk_usage "$model_files") - - # Get cache path (first file's directory) - local cache_path - cache_path=$(echo "$model_files" | head -n1 | xargs dirname) - - # Get modification time of first file - local mod_time="Unknown" - if [[ -n "$model_files" ]]; then - local first_file - first_file=$(echo "$model_files" | head -n1) - if [[ -f "$first_file" ]]; then - # Try Linux stat first (most common), then macOS stat - if stat -c "%y" "$first_file" >/dev/null 2>&1; then - mod_time=$(stat -c "%y" "$first_file" 2>/dev/null | cut -d'.' -f1) - elif stat -f "%Sm" "$first_file" >/dev/null 2>&1; then - mod_time=$(stat -f "%Sm" -t "%Y-%m-%d %H:%M:%S" "$first_file" 2>/dev/null) - fi - fi - fi - - echo "FOUND|${actual_bytes}|${file_count}|${cache_path}|${mod_time}" - return 0 -} - -# Verify model symlinks -verify_model_links() { - local repo_id="$1" - local filename_filter="$2" - local file_mappings="$3" - - local total_links=0 - local valid_links=0 - local broken_links=0 - local link_details="" - - # Check if explicit file mappings exist - if [[ -n "$file_mappings" ]]; then - # Process each file mapping to verify links - while IFS='|' read -r source_pattern dest_path; do - if [[ -z "$source_pattern" ]] || [[ -z "$dest_path" ]]; then - continue - fi - - total_links=$((total_links + 1)) - local link_path="${COMFYUI_DIR}/${dest_path}" - - if [[ -L "$link_path" ]]; then - # Symlink exists, check if it's valid - if [[ -e "$link_path" ]]; then - valid_links=$((valid_links + 1)) - local link_target - link_target=$(readlink "$link_path") - link_details="${link_details}VALID|${dest_path}|${link_target}\n" - else - broken_links=$((broken_links + 1)) - local link_target - link_target=$(readlink "$link_path") - link_details="${link_details}BROKEN|${dest_path}|${link_target}\n" - fi - else - link_details="${link_details}MISSING|${dest_path}|\n" - fi - done <<< "$file_mappings" - else - # No explicit mappings, check automatic prefixed filenames - local model_files - model_files=$(find_model_files "$repo_id" "$filename_filter" 2>/dev/null) - - if [[ -z "$model_files" ]]; then - echo "NOT_DOWNLOADED|0|0|0" + if [[ "$match" == false ]]; then + return 1 + fi + fi + + # Check repo_id filter + if [[ -n "$REPO_ID_FILTER" ]]; then + local match=false + IFS=',' read -ra repos <<< "$REPO_ID_FILTER" + for repo in "${repos[@]}"; do + repo=$(echo "$repo" | xargs) + if [[ "$repo_id" == "$repo" ]]; then + match=true + break + fi + done + if [[ "$match" == false ]]; then return 1 fi - - local model_name - model_name=$(echo "$repo_id" | sed 's/.*\///') - - while IFS= read -r source_file; do - if [[ -f "$source_file" ]]; then - local filename - filename=$(basename "$source_file") - local prefixed_filename="${model_name}-${filename}" - - total_links=$((total_links + 1)) - local link_path="${target_dir}/${prefixed_filename}" - - if [[ -L "$link_path" ]]; then - if [[ -e "$link_path" ]]; then - valid_links=$((valid_links + 1)) - local link_target - link_target=$(readlink "$link_path") - link_details="${link_details}VALID|${prefixed_filename}|${link_target}\n" - else - broken_links=$((broken_links + 1)) - local link_target - link_target=$(readlink "$link_path") - link_details="${link_details}BROKEN|${prefixed_filename}|${link_target}\n" - fi - else - link_details="${link_details}MISSING|${prefixed_filename}|\n" - fi - fi - done <<< "$model_files" fi - echo -e "CHECKED|${total_links}|${valid_links}|${broken_links}\n${link_details}" return 0 } -# Verify models by category -verify_category() { - local category="$1" - local category_display="$2" +# ============================================================================ +# DOWNLOAD FUNCTIONS +# ============================================================================ - print_section "${category_display}" +download_file() { + local repo_id="$1" + local source="$2" - # Get models for this category - local models_data - models_data=$(parse_yaml "$CONFIG_FILE" "$category") + # Convert repo_id to cache path (replace / with --) + local cache_repo_dir="${CACHE_DIR}/${repo_id}" + local source_dir + source_dir=$(dirname "$source") + local output_dir="${cache_repo_dir}" + if [[ "$source_dir" != "." ]]; then + output_dir="${cache_repo_dir}/${source_dir}" + fi + local filename + filename=$(basename "$source") + local output_path="${output_dir}/${filename}" - if [[ -z "$models_data" ]]; then - print_warning "No models found in category: ${category}" + print_detail "File: ${BOLD_WHITE}${source}${RESET}" + print_detail "Output: ${CYAN}${output_path}${RESET}" + + # Check if already exists + if [[ -f "$output_path" ]]; then + local size + size=$(du -h "$output_path" | cut -f1) + print_success "Already downloaded: ${filename} (${size})" return 0 fi - local total_models - total_models=$(echo "$models_data" | wc -l) - local current=0 - local models_downloaded=0 - local models_missing=0 - local models_linked=0 - local models_broken=0 - local models_not_linked=0 - local total_size_bytes=0 - local expected_size_bytes=0 + # Dry-run mode + if [[ "$DRY_RUN" == true ]]; then + print_info "DRY-RUN: Would download ${BOLD_WHITE}${source}${RESET}" + return 0 + fi - while IFS='|' read -r repo_id description size_gb essential filename; do - current=$((current+1)) + # Create output directory + mkdir -p "$output_dir" - echo "" - print_step "$current" "$total_models" "${BOLD_MAGENTA}${description}${RESET}" - print_detail "Repository: ${BOLD_WHITE}${repo_id}${RESET}" - print_detail "Category: ${CYAN}${category}${RESET}" - print_detail "Expected Size: ${BOLD_YELLOW}${size_gb} GB${RESET}" + # Build download URL + local url="https://huggingface.co/${repo_id}/resolve/main/${source}" + print_detail "Downloading from HuggingFace..." - expected_size_bytes=$((expected_size_bytes + $(echo "$size_gb * 1073741824" | bc | cut -d'.' -f1))) + # Download with curl (with resume support) + local curl_args=(-L -C - --progress-bar -o "$output_path") + if [[ -n "$HF_TOKEN" ]]; then + curl_args+=(-H "Authorization: Bearer ${HF_TOKEN}") + fi - # Verify download status - echo "" - local download_result - download_result=$(verify_model_download "$repo_id" "$size_gb" "$filename") - local download_status - download_status=$(echo "$download_result" | cut -d'|' -f1) - - if [[ "$download_status" == "FOUND" ]]; then - local actual_bytes - actual_bytes=$(echo "$download_result" | cut -d'|' -f2) - local file_count - file_count=$(echo "$download_result" | cut -d'|' -f3) - local cache_path - cache_path=$(echo "$download_result" | cut -d'|' -f4) - local mod_time - mod_time=$(echo "$download_result" | cut -d'|' -f5) - - total_size_bytes=$((total_size_bytes + actual_bytes)) - local actual_size_human - actual_size_human=$(format_bytes "$actual_bytes") - - print_success "Download Status: ${BOLD_GREEN}DOWNLOADED${RESET}" - print_detail "${DIM}Path: ${cache_path}${RESET}" - print_detail "${DIM}Actual Size: ${actual_size_human} (${actual_bytes} bytes)${RESET}" - print_detail "${DIM}Files: ${file_count} file(s)${RESET}" - print_detail "${DIM}Modified: ${mod_time}${RESET}" - - # Check for size mismatch - local expected_bytes - expected_bytes=$(echo "$size_gb * 1073741824" | bc | cut -d'.' -f1) - local size_diff_pct - size_diff_pct=$(echo "scale=2; (($actual_bytes - $expected_bytes) / $expected_bytes) * 100" | bc | sed 's/^\./0./') - local abs_size_diff_pct - abs_size_diff_pct=${size_diff_pct#-} - - if (( $(echo "$abs_size_diff_pct > 10" | bc -l) )); then - print_warning "Size mismatch: ${size_diff_pct}% difference from expected" - fi - - models_downloaded=$((models_downloaded+1)) - - # Verify link status - echo "" - local file_mappings - file_mappings=$(parse_file_mappings "$CONFIG_FILE" "$category" "$repo_id") - local link_result - link_result=$(verify_model_links "$repo_id" "$filename" "$file_mappings") - local first_line - first_line=$(echo -e "$link_result" | head -n1) - local link_status - link_status=$(echo "$first_line" | cut -d'|' -f1) - - if [[ "$link_status" == "CHECKED" ]]; then - local total_links - total_links=$(echo "$first_line" | cut -d'|' -f2) - local valid_links - valid_links=$(echo "$first_line" | cut -d'|' -f3) - local broken_links - broken_links=$(echo "$first_line" | cut -d'|' -f4) - - if [[ $broken_links -gt 0 ]]; then - print_warning "Link Status: ${BOLD_YELLOW}${broken_links} BROKEN LINK(S)${RESET}" - models_broken=$((models_broken+1)) - elif [[ $valid_links -eq $total_links ]] && [[ $total_links -gt 0 ]]; then - print_success "Link Status: ${BOLD_GREEN}LINKED${RESET} (${valid_links}/${total_links})" - models_linked=$((models_linked+1)) - else - print_warning "Link Status: ${BOLD_YELLOW}PARTIALLY LINKED${RESET} (${valid_links}/${total_links})" - models_not_linked=$((models_not_linked+1)) - fi - - # Show link details - local link_details - link_details=$(echo -e "$link_result" | tail -n +2) - if [[ -n "$link_details" ]]; then - while IFS='|' read -r link_state link_name link_target; do - if [[ -z "$link_state" ]]; then - continue - fi - - case "$link_state" in - VALID) - print_detail "${LINK} ${BOLD_GREEN}โœ“${RESET} ${DIM}${link_name}${RESET}" - ;; - BROKEN) - print_detail "${LINK} ${BOLD_RED}โœ—${RESET} ${DIM}${link_name}${RESET} ${BOLD_RED}(BROKEN)${RESET}" - ;; - MISSING) - print_detail "${LINK} ${BOLD_YELLOW}โ—‹${RESET} ${DIM}${link_name}${RESET} ${BOLD_YELLOW}(NOT LINKED)${RESET}" - ;; - esac - done <<< "$link_details" - fi - else - print_error "Link Status: ${BOLD_RED}NOT LINKED${RESET}" - models_not_linked=$((models_not_linked+1)) - fi - else - print_error "Download Status: ${BOLD_RED}NOT DOWNLOADED${RESET}" - models_missing=$((models_missing+1)) - echo "" - print_info "Link Status: ${DIM}N/A (model not downloaded)${RESET}" + if curl "${curl_args[@]}" "$url" 2>&1; then + if [[ -f "$output_path" ]] && [[ -s "$output_path" ]]; then + local size + size=$(du -h "$output_path" | cut -f1) + print_success "Downloaded ${BOLD_WHITE}${filename}${RESET} (${size})" + return 0 fi + fi - show_progress "$current" "$total_models" - done <<< "$models_data" - - echo -e "\n" - - # Category summary - local total_size_human - total_size_human=$(format_bytes "$total_size_bytes") - local expected_size_human - expected_size_human=$(format_bytes "$expected_size_bytes") - - print_info "Category Summary:" - echo -e " ${BOLD_WHITE}Total Models:${RESET} ${total_models}" - echo -e " ${BOLD_GREEN}โœ“ Downloaded:${RESET} ${models_downloaded} ($(( models_downloaded * 100 / total_models ))%)" - echo -e " ${BOLD_RED}โœ— Missing:${RESET} ${models_missing} ($(( models_missing * 100 / total_models ))%)" - echo -e " ${BOLD_GREEN}โœ“ Properly Linked:${RESET} ${models_linked}" - echo -e " ${BOLD_YELLOW}โš  Broken Links:${RESET} ${models_broken}" - echo -e " ${BOLD_YELLOW}โ—‹ Not Linked:${RESET} ${models_not_linked}" - echo -e " ${BOLD_CYAN}๐Ÿ“Š Disk Usage:${RESET} ${total_size_human} / ${expected_size_human} expected" - - # Return statistics for global summary (format: downloaded|missing|linked|broken|not_linked|total_size|expected_size) - echo "${models_downloaded}|${models_missing}|${models_linked}|${models_broken}|${models_not_linked}|${total_size_bytes}|${expected_size_bytes}" > /tmp/verify_stats_${category} + print_error "Failed to download ${source}" + rm -f "$output_path" 2>/dev/null || true + return 1 } -# Process models by category -process_category() { - local category="$1" - local category_display="$2" +download_model() { + local config="$1" + local index="$2" + local repo_id="$3" + local description="$4" - print_section "${category_display}" + print_detail "Repository: ${BOLD_WHITE}${repo_id}${RESET}" + [[ -n "$description" ]] && print_detail "Description: ${description}" - # Get models for this category - local models_data - models_data=$(parse_yaml "$CONFIG_FILE" "$category") + local files_count + files_count=$(get_files_count "$config" "$index") - if [[ -z "$models_data" ]]; then - print_warning "No models found in category: ${category}" - return 0 + if [[ "$files_count" == "0" ]]; then + print_warning "No files defined for ${repo_id}" + return 1 fi - local total_models - total_models=$(echo "$models_data" | wc -l) - local current=0 local succeeded=0 local failed=0 - while IFS='|' read -r repo_id description size_gb essential filename; do - current=$((current+1)) + for ((f=0; f 0 ? total_downloaded * 100 / total_models : 0 ))%)" - echo -e " ${BOLD_RED}โœ— Missing:${RESET} ${total_missing} ($(( total_models > 0 ? total_missing * 100 / total_models : 0 ))%)" - echo "" - echo -e " ${BOLD_GREEN}โœ“ Properly Linked:${RESET} ${total_linked} ($(( total_models > 0 ? total_linked * 100 / total_models : 0 ))%)" - echo -e " ${BOLD_YELLOW}โš  Broken Links:${RESET} ${total_broken} ($(( total_models > 0 ? total_broken * 100 / total_models : 0 ))%)" - echo -e " ${BOLD_YELLOW}โ—‹ Not Linked:${RESET} ${total_not_linked} ($(( total_models > 0 ? total_not_linked * 100 / total_models : 0 ))%)" - echo "" - echo -e " ${BOLD_CYAN}๐Ÿ“Š Disk Space Used:${RESET} ${total_size_human} / ${expected_size_human} expected" - echo -e " ${BOLD_WHITE}Cache Directory:${RESET} ${CYAN}${CACHE_DIR}${RESET}" - echo -e " ${BOLD_WHITE}ComfyUI Directory:${RESET} ${CYAN}${COMFYUI_DIR}${RESET}" - echo -e " ${BOLD_WHITE}Duration:${RESET} ${BOLD_YELLOW}${minutes}m ${seconds}s${RESET}" - echo -e "${CYAN}$(printf '%.0s'"${BOX_DOUBLE}" $(seq 1 80))${RESET}" + # Create symlink + ln -s "$source_path" "$link_path" + print_success "Linked: ${LINK_ICON} ${dest}" + return 0 +} - # Provide actionable suggestions - if [[ $total_missing -gt 0 ]] || [[ $total_broken -gt 0 ]] || [[ $total_not_linked -gt 0 ]]; then - echo -e "\n${BOLD_YELLOW}${WARNING} Issues Found - Suggested Actions:${RESET}\n" +link_model() { + local config="$1" + local index="$2" + local repo_id="$3" - if [[ $total_missing -gt 0 ]]; then - echo -e " ${BOLD_RED}โœ—${RESET} ${total_missing} model(s) not downloaded" - echo -e " ${DIM}Fix:${RESET} ${CYAN}$0 download -c ${CONFIG_FILE}${RESET}" + print_detail "Repository: ${BOLD_WHITE}${repo_id}${RESET}" + + local files_count + files_count=$(get_files_count "$config" "$index") + + if [[ "$files_count" == "0" ]]; then + print_warning "No files defined for ${repo_id}" + return 1 + fi + + local succeeded=0 + local failed=0 + + for ((f=0; f